Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/Transforms/IPO/FunctionImport.cpp
Line
Count
Source (jump to first uncovered line)
1
//===- FunctionImport.cpp - ThinLTO Summary-based Function Import ---------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This file implements Function import based on summaries.
10
//
11
//===----------------------------------------------------------------------===//
12
13
#include "llvm/Transforms/IPO/FunctionImport.h"
14
#include "llvm/ADT/ArrayRef.h"
15
#include "llvm/ADT/STLExtras.h"
16
#include "llvm/ADT/SetVector.h"
17
#include "llvm/ADT/SmallVector.h"
18
#include "llvm/ADT/Statistic.h"
19
#include "llvm/ADT/StringMap.h"
20
#include "llvm/ADT/StringRef.h"
21
#include "llvm/ADT/StringSet.h"
22
#include "llvm/Bitcode/BitcodeReader.h"
23
#include "llvm/IR/AutoUpgrade.h"
24
#include "llvm/IR/Constants.h"
25
#include "llvm/IR/Function.h"
26
#include "llvm/IR/GlobalAlias.h"
27
#include "llvm/IR/GlobalObject.h"
28
#include "llvm/IR/GlobalValue.h"
29
#include "llvm/IR/GlobalVariable.h"
30
#include "llvm/IR/Metadata.h"
31
#include "llvm/IR/Module.h"
32
#include "llvm/IR/ModuleSummaryIndex.h"
33
#include "llvm/IRReader/IRReader.h"
34
#include "llvm/Linker/IRMover.h"
35
#include "llvm/Object/ModuleSymbolTable.h"
36
#include "llvm/Object/SymbolicFile.h"
37
#include "llvm/Pass.h"
38
#include "llvm/Support/Casting.h"
39
#include "llvm/Support/CommandLine.h"
40
#include "llvm/Support/Debug.h"
41
#include "llvm/Support/Error.h"
42
#include "llvm/Support/ErrorHandling.h"
43
#include "llvm/Support/FileSystem.h"
44
#include "llvm/Support/SourceMgr.h"
45
#include "llvm/Support/raw_ostream.h"
46
#include "llvm/Transforms/IPO/Internalize.h"
47
#include "llvm/Transforms/Utils/Cloning.h"
48
#include "llvm/Transforms/Utils/FunctionImportUtils.h"
49
#include "llvm/Transforms/Utils/ValueMapper.h"
50
#include <cassert>
51
#include <memory>
52
#include <set>
53
#include <string>
54
#include <system_error>
55
#include <tuple>
56
#include <utility>
57
58
using namespace llvm;
59
60
#define DEBUG_TYPE "function-import"
61
62
STATISTIC(NumImportedFunctionsThinLink,
63
          "Number of functions thin link decided to import");
64
STATISTIC(NumImportedHotFunctionsThinLink,
65
          "Number of hot functions thin link decided to import");
66
STATISTIC(NumImportedCriticalFunctionsThinLink,
67
          "Number of critical functions thin link decided to import");
68
STATISTIC(NumImportedGlobalVarsThinLink,
69
          "Number of global variables thin link decided to import");
70
STATISTIC(NumImportedFunctions, "Number of functions imported in backend");
71
STATISTIC(NumImportedGlobalVars,
72
          "Number of global variables imported in backend");
73
STATISTIC(NumImportedModules, "Number of modules imported from");
74
STATISTIC(NumDeadSymbols, "Number of dead stripped symbols in index");
75
STATISTIC(NumLiveSymbols, "Number of live symbols in index");
76
77
/// Limit on instruction count of imported functions.
78
static cl::opt<unsigned> ImportInstrLimit(
79
    "import-instr-limit", cl::init(100), cl::Hidden, cl::value_desc("N"),
80
    cl::desc("Only import functions with less than N instructions"));
81
82
static cl::opt<int> ImportCutoff(
83
    "import-cutoff", cl::init(-1), cl::Hidden, cl::value_desc("N"),
84
    cl::desc("Only import first N functions if N>=0 (default -1)"));
85
86
static cl::opt<float>
87
    ImportInstrFactor("import-instr-evolution-factor", cl::init(0.7),
88
                      cl::Hidden, cl::value_desc("x"),
89
                      cl::desc("As we import functions, multiply the "
90
                               "`import-instr-limit` threshold by this factor "
91
                               "before processing newly imported functions"));
92
93
static cl::opt<float> ImportHotInstrFactor(
94
    "import-hot-evolution-factor", cl::init(1.0), cl::Hidden,
95
    cl::value_desc("x"),
96
    cl::desc("As we import functions called from hot callsite, multiply the "
97
             "`import-instr-limit` threshold by this factor "
98
             "before processing newly imported functions"));
99
100
static cl::opt<float> ImportHotMultiplier(
101
    "import-hot-multiplier", cl::init(10.0), cl::Hidden, cl::value_desc("x"),
102
    cl::desc("Multiply the `import-instr-limit` threshold for hot callsites"));
103
104
static cl::opt<float> ImportCriticalMultiplier(
105
    "import-critical-multiplier", cl::init(100.0), cl::Hidden,
106
    cl::value_desc("x"),
107
    cl::desc(
108
        "Multiply the `import-instr-limit` threshold for critical callsites"));
109
110
// FIXME: This multiplier was not really tuned up.
111
static cl::opt<float> ImportColdMultiplier(
112
    "import-cold-multiplier", cl::init(0), cl::Hidden, cl::value_desc("N"),
113
    cl::desc("Multiply the `import-instr-limit` threshold for cold callsites"));
114
115
static cl::opt<bool> PrintImports("print-imports", cl::init(false), cl::Hidden,
116
                                  cl::desc("Print imported functions"));
117
118
static cl::opt<bool> PrintImportFailures(
119
    "print-import-failures", cl::init(false), cl::Hidden,
120
    cl::desc("Print information for functions rejected for importing"));
121
122
static cl::opt<bool> ComputeDead("compute-dead", cl::init(true), cl::Hidden,
123
                                 cl::desc("Compute dead symbols"));
124
125
static cl::opt<bool> EnableImportMetadata(
126
    "enable-import-metadata", cl::init(
127
#if !defined(NDEBUG)
128
                                  true /*Enabled with asserts.*/
129
#else
130
                                  false
131
#endif
132
                                  ),
133
    cl::Hidden, cl::desc("Enable import metadata like 'thinlto_src_module'"));
134
135
/// Summary file to use for function importing when using -function-import from
136
/// the command line.
137
static cl::opt<std::string>
138
    SummaryFile("summary-file",
139
                cl::desc("The summary file to use for function importing."));
140
141
/// Used when testing importing from distributed indexes via opt
142
// -function-import.
143
static cl::opt<bool>
144
    ImportAllIndex("import-all-index",
145
                   cl::desc("Import all external functions in index."));
146
147
// Load lazily a module from \p FileName in \p Context.
148
static std::unique_ptr<Module> loadFile(const std::string &FileName,
149
20
                                        LLVMContext &Context) {
150
20
  SMDiagnostic Err;
151
20
  LLVM_DEBUG(dbgs() << "Loading '" << FileName << "'\n");
152
20
  // Metadata isn't loaded until functions are imported, to minimize
153
20
  // the memory overhead.
154
20
  std::unique_ptr<Module> Result =
155
20
      getLazyIRFileModule(FileName, Err, Context,
156
20
                          /* ShouldLazyLoadMetadata = */ true);
157
20
  if (!Result) {
158
0
    Err.print("function-import", errs());
159
0
    report_fatal_error("Abort");
160
0
  }
161
20
162
20
  return Result;
163
20
}
164
165
/// Given a list of possible callee implementation for a call site, select one
166
/// that fits the \p Threshold.
167
///
168
/// FIXME: select "best" instead of first that fits. But what is "best"?
169
/// - The smallest: more likely to be inlined.
170
/// - The one with the least outgoing edges (already well optimized).
171
/// - One from a module already being imported from in order to reduce the
172
///   number of source modules parsed/linked.
173
/// - One that has PGO data attached.
174
/// - [insert you fancy metric here]
175
static const GlobalValueSummary *
176
selectCallee(const ModuleSummaryIndex &Index,
177
             ArrayRef<std::unique_ptr<GlobalValueSummary>> CalleeSummaryList,
178
             unsigned Threshold, StringRef CallerModulePath,
179
             FunctionImporter::ImportFailureReason &Reason,
180
398
             GlobalValue::GUID GUID) {
181
398
  Reason = FunctionImporter::ImportFailureReason::None;
182
398
  auto It = llvm::find_if(
183
398
      CalleeSummaryList,
184
401
      [&](const std::unique_ptr<GlobalValueSummary> &SummaryPtr) {
185
401
        auto *GVSummary = SummaryPtr.get();
186
401
        if (!Index.isGlobalValueLive(GVSummary)) {
187
3
          Reason = FunctionImporter::ImportFailureReason::NotLive;
188
3
          return false;
189
3
        }
190
398
191
398
        // For SamplePGO, in computeImportForFunction the OriginalId
192
398
        // may have been used to locate the callee summary list (See
193
398
        // comment there).
194
398
        // The mapping from OriginalId to GUID may return a GUID
195
398
        // that corresponds to a static variable. Filter it out here.
196
398
        // This can happen when
197
398
        // 1) There is a call to a library function which is not defined
198
398
        // in the index.
199
398
        // 2) There is a static variable with the  OriginalGUID identical
200
398
        // to the GUID of the library function in 1);
201
398
        // When this happens, the logic for SamplePGO kicks in and
202
398
        // the static variable in 2) will be found, which needs to be
203
398
        // filtered out.
204
398
        if (GVSummary->getSummaryKind() == GlobalValueSummary::GlobalVarKind) {
205
2
          Reason = FunctionImporter::ImportFailureReason::GlobalVar;
206
2
          return false;
207
2
        }
208
396
        if (GlobalValue::isInterposableLinkage(GVSummary->linkage())) {
209
32
          Reason = FunctionImporter::ImportFailureReason::InterposableLinkage;
210
32
          // There is no point in importing these, we can't inline them
211
32
          return false;
212
32
        }
213
364
214
364
        auto *Summary = cast<FunctionSummary>(GVSummary->getBaseObject());
215
364
216
364
        // If this is a local function, make sure we import the copy
217
364
        // in the caller's module. The only time a local function can
218
364
        // share an entry in the index is if there is a local with the same name
219
364
        // in another module that had the same source file name (in a different
220
364
        // directory), where each was compiled in their own directory so there
221
364
        // was not distinguishing path.
222
364
        // However, do the import from another module if there is only one
223
364
        // entry in the list - in that case this must be a reference due
224
364
        // to indirect call profile data, since a function pointer can point to
225
364
        // a local in another module.
226
364
        if (GlobalValue::isLocalLinkage(Summary->linkage()) &&
227
364
            
CalleeSummaryList.size() > 125
&&
228
364
            
Summary->modulePath() != CallerModulePath6
) {
229
3
          Reason =
230
3
              FunctionImporter::ImportFailureReason::LocalLinkageNotInModule;
231
3
          return false;
232
3
        }
233
361
234
361
        if (Summary->instCount() > Threshold) {
235
52
          Reason = FunctionImporter::ImportFailureReason::TooLarge;
236
52
          return false;
237
52
        }
238
309
239
309
        // Skip if it isn't legal to import (e.g. may reference unpromotable
240
309
        // locals).
241
309
        if (Summary->notEligibleToImport()) {
242
11
          Reason = FunctionImporter::ImportFailureReason::NotEligible;
243
11
          return false;
244
11
        }
245
298
246
298
        // Don't bother importing if we can't inline it anyway.
247
298
        if (Summary->fflags().NoInline) {
248
4
          Reason = FunctionImporter::ImportFailureReason::NoInline;
249
4
          return false;
250
4
        }
251
294
252
294
        return true;
253
294
      });
254
398
  if (It == CalleeSummaryList.end())
255
104
    return nullptr;
256
294
257
294
  return cast<GlobalValueSummary>(It->get());
258
294
}
259
260
namespace {
261
262
using EdgeInfo = std::tuple<const FunctionSummary *, unsigned /* Threshold */,
263
                            GlobalValue::GUID>;
264
265
} // anonymous namespace
266
267
static ValueInfo
268
885
updateValueInfoForIndirectCalls(const ModuleSummaryIndex &Index, ValueInfo VI) {
269
885
  if (!VI.getSummaryList().empty())
270
787
    return VI;
271
98
  // For SamplePGO, the indirect call targets for local functions will
272
98
  // have its original name annotated in profile. We try to find the
273
98
  // corresponding PGOFuncName as the GUID.
274
98
  // FIXME: Consider updating the edges in the graph after building
275
98
  // it, rather than needing to perform this mapping on each walk.
276
98
  auto GUID = Index.getGUIDFromOriginalID(VI.getGUID());
277
98
  if (GUID == 0)
278
87
    return ValueInfo();
279
11
  return Index.getValueInfo(GUID);
280
11
}
281
282
static void computeImportForReferencedGlobals(
283
    const FunctionSummary &Summary, const GVSummaryMapTy &DefinedGVSummaries,
284
    FunctionImporter::ImportMapTy &ImportList,
285
1.12k
    StringMap<FunctionImporter::ExportSetTy> *ExportLists) {
286
1.12k
  for (auto &VI : Summary.refs()) {
287
223
    if (DefinedGVSummaries.count(VI.getGUID())) {
288
118
      LLVM_DEBUG(
289
118
          dbgs() << "Ref ignored! Target already in destination module.\n");
290
118
      continue;
291
118
    }
292
105
293
105
    LLVM_DEBUG(dbgs() << " ref -> " << VI << "\n");
294
105
295
105
    // If this is a local variable, make sure we import the copy
296
105
    // in the caller's module. The only time a local variable can
297
105
    // share an entry in the index is if there is a local with the same name
298
105
    // in another module that had the same source file name (in a different
299
105
    // directory), where each was compiled in their own directory so there
300
105
    // was not distinguishing path.
301
105
    auto LocalNotInModule = [&](const GlobalValueSummary *RefSummary) -> bool {
302
71
      return GlobalValue::isLocalLinkage(RefSummary->linkage()) &&
303
71
             
RefSummary->modulePath() != Summary.modulePath()44
;
304
71
    };
305
105
306
105
    for (auto &RefSummary : VI.getSummaryList())
307
106
      if (isa<GlobalVarSummary>(RefSummary.get()) &&
308
106
          
canImportGlobalVar(RefSummary.get())90
&&
309
106
          
!LocalNotInModule(RefSummary.get())71
) {
310
66
        auto ILI = ImportList[RefSummary->modulePath()].insert(VI.getGUID());
311
66
        // Only update stat if we haven't already imported this variable.
312
66
        if (ILI.second)
313
61
          NumImportedGlobalVarsThinLink++;
314
66
        if (ExportLists)
315
65
          (*ExportLists)[RefSummary->modulePath()].insert(VI.getGUID());
316
66
        break;
317
66
      }
318
105
  }
319
1.12k
}
320
321
static const char *
322
1
getFailureName(FunctionImporter::ImportFailureReason Reason) {
323
1
  switch (Reason) {
324
1
  case FunctionImporter::ImportFailureReason::None:
325
0
    return "None";
326
1
  case FunctionImporter::ImportFailureReason::GlobalVar:
327
0
    return "GlobalVar";
328
1
  case FunctionImporter::ImportFailureReason::NotLive:
329
0
    return "NotLive";
330
1
  case FunctionImporter::ImportFailureReason::TooLarge:
331
1
    return "TooLarge";
332
1
  case FunctionImporter::ImportFailureReason::InterposableLinkage:
333
0
    return "InterposableLinkage";
334
1
  case FunctionImporter::ImportFailureReason::LocalLinkageNotInModule:
335
0
    return "LocalLinkageNotInModule";
336
1
  case FunctionImporter::ImportFailureReason::NotEligible:
337
0
    return "NotEligible";
338
1
  case FunctionImporter::ImportFailureReason::NoInline:
339
0
    return "NoInline";
340
0
  }
341
0
  llvm_unreachable("invalid reason");
342
0
}
343
344
/// Compute the list of functions to import for a given caller. Mark these
345
/// imported functions and the symbols they reference in their source module as
346
/// exported from their source module.
347
static void computeImportForFunction(
348
    const FunctionSummary &Summary, const ModuleSummaryIndex &Index,
349
    const unsigned Threshold, const GVSummaryMapTy &DefinedGVSummaries,
350
    SmallVectorImpl<EdgeInfo> &Worklist,
351
    FunctionImporter::ImportMapTy &ImportList,
352
    StringMap<FunctionImporter::ExportSetTy> *ExportLists,
353
1.12k
    FunctionImporter::ImportThresholdsTy &ImportThresholds) {
354
1.12k
  computeImportForReferencedGlobals(Summary, DefinedGVSummaries, ImportList,
355
1.12k
                                    ExportLists);
356
1.12k
  static int ImportCount = 0;
357
1.12k
  for (auto &Edge : Summary.calls()) {
358
500
    ValueInfo VI = Edge.first;
359
500
    LLVM_DEBUG(dbgs() << " edge -> " << VI << " Threshold:" << Threshold
360
500
                      << "\n");
361
500
362
500
    if (ImportCutoff >= 0 && 
ImportCount >= ImportCutoff0
) {
363
0
      LLVM_DEBUG(dbgs() << "ignored! import-cutoff value of " << ImportCutoff
364
0
                        << " reached.\n");
365
0
      continue;
366
0
    }
367
500
368
500
    VI = updateValueInfoForIndirectCalls(Index, VI);
369
500
    if (!VI)
370
31
      continue;
371
469
372
469
    if (DefinedGVSummaries.count(VI.getGUID())) {
373
66
      LLVM_DEBUG(dbgs() << "ignored! Target already in destination module.\n");
374
66
      continue;
375
66
    }
376
403
377
403
    auto GetBonusMultiplier = [](CalleeInfo::HotnessType Hotness) -> float {
378
403
      if (Hotness == CalleeInfo::HotnessType::Hot)
379
44
        return ImportHotMultiplier;
380
359
      if (Hotness == CalleeInfo::HotnessType::Cold)
381
19
        return ImportColdMultiplier;
382
340
      if (Hotness == CalleeInfo::HotnessType::Critical)
383
0
        return ImportCriticalMultiplier;
384
340
      return 1.0;
385
340
    };
386
403
387
403
    const auto NewThreshold =
388
403
        Threshold * GetBonusMultiplier(Edge.second.getHotness());
389
403
390
403
    auto IT = ImportThresholds.insert(std::make_pair(
391
403
        VI.getGUID(), std::make_tuple(NewThreshold, nullptr, nullptr)));
392
403
    bool PreviouslyVisited = !IT.second;
393
403
    auto &ProcessedThreshold = std::get<0>(IT.first->second);
394
403
    auto &CalleeSummary = std::get<1>(IT.first->second);
395
403
    auto &FailureInfo = std::get<2>(IT.first->second);
396
403
397
403
    bool IsHotCallsite =
398
403
        Edge.second.getHotness() == CalleeInfo::HotnessType::Hot;
399
403
    bool IsCriticalCallsite =
400
403
        Edge.second.getHotness() == CalleeInfo::HotnessType::Critical;
401
403
402
403
    const FunctionSummary *ResolvedCalleeSummary = nullptr;
403
403
    if (CalleeSummary) {
404
5
      assert(PreviouslyVisited);
405
5
      // Since the traversal of the call graph is DFS, we can revisit a function
406
5
      // a second time with a higher threshold. In this case, it is added back
407
5
      // to the worklist with the new threshold (so that its own callee chains
408
5
      // can be considered with the higher threshold).
409
5
      if (NewThreshold <= ProcessedThreshold) {
410
5
        LLVM_DEBUG(
411
5
            dbgs() << "ignored! Target was already imported with Threshold "
412
5
                   << ProcessedThreshold << "\n");
413
5
        continue;
414
5
      }
415
0
      // Update with new larger threshold.
416
0
      ProcessedThreshold = NewThreshold;
417
0
      ResolvedCalleeSummary = cast<FunctionSummary>(CalleeSummary);
418
398
    } else {
419
398
      // If we already rejected importing a callee at the same or higher
420
398
      // threshold, don't waste time calling selectCallee.
421
398
      if (PreviouslyVisited && 
NewThreshold <= ProcessedThreshold2
) {
422
0
        LLVM_DEBUG(
423
0
            dbgs() << "ignored! Target was already rejected with Threshold "
424
0
            << ProcessedThreshold << "\n");
425
0
        if (PrintImportFailures) {
426
0
          assert(FailureInfo &&
427
0
                 "Expected FailureInfo for previously rejected candidate");
428
0
          FailureInfo->Attempts++;
429
0
        }
430
0
        continue;
431
0
      }
432
398
433
398
      FunctionImporter::ImportFailureReason Reason;
434
398
      CalleeSummary = selectCallee(Index, VI.getSummaryList(), NewThreshold,
435
398
                                   Summary.modulePath(), Reason, VI.getGUID());
436
398
      if (!CalleeSummary) {
437
104
        // Update with new larger threshold if this was a retry (otherwise
438
104
        // we would have already inserted with NewThreshold above). Also
439
104
        // update failure info if requested.
440
104
        if (PreviouslyVisited) {
441
1
          ProcessedThreshold = NewThreshold;
442
1
          if (PrintImportFailures) {
443
0
            assert(FailureInfo &&
444
0
                   "Expected FailureInfo for previously rejected candidate");
445
0
            FailureInfo->Reason = Reason;
446
0
            FailureInfo->Attempts++;
447
0
            FailureInfo->MaxHotness =
448
0
                std::max(FailureInfo->MaxHotness, Edge.second.getHotness());
449
0
          }
450
103
        } else if (PrintImportFailures) {
451
1
          assert(!FailureInfo &&
452
1
                 "Expected no FailureInfo for newly rejected candidate");
453
1
          FailureInfo = llvm::make_unique<FunctionImporter::ImportFailureInfo>(
454
1
              VI, Edge.second.getHotness(), Reason, 1);
455
1
        }
456
104
        LLVM_DEBUG(
457
104
            dbgs() << "ignored! No qualifying callee with summary found.\n");
458
104
        continue;
459
104
      }
460
294
461
294
      // "Resolve" the summary
462
294
      CalleeSummary = CalleeSummary->getBaseObject();
463
294
      ResolvedCalleeSummary = cast<FunctionSummary>(CalleeSummary);
464
294
465
294
      assert(ResolvedCalleeSummary->instCount() <= NewThreshold &&
466
294
             "selectCallee() didn't honor the threshold");
467
294
468
294
      auto ExportModulePath = ResolvedCalleeSummary->modulePath();
469
294
      auto ILI = ImportList[ExportModulePath].insert(VI.getGUID());
470
294
      // We previously decided to import this GUID definition if it was already
471
294
      // inserted in the set of imports from the exporting module.
472
294
      bool PreviouslyImported = !ILI.second;
473
294
      if (!PreviouslyImported) {
474
294
        NumImportedFunctionsThinLink++;
475
294
        if (IsHotCallsite)
476
28
          NumImportedHotFunctionsThinLink++;
477
294
        if (IsCriticalCallsite)
478
0
          NumImportedCriticalFunctionsThinLink++;
479
294
      }
480
294
481
294
      // Make exports in the source module.
482
294
      if (ExportLists) {
483
238
        auto &ExportList = (*ExportLists)[ExportModulePath];
484
238
        ExportList.insert(VI.getGUID());
485
238
        if (!PreviouslyImported) {
486
238
          // This is the first time this function was exported from its source
487
238
          // module, so mark all functions and globals it references as exported
488
238
          // to the outside if they are defined in the same source module.
489
238
          // For efficiency, we unconditionally add all the referenced GUIDs
490
238
          // to the ExportList for this module, and will prune out any not
491
238
          // defined in the module later in a single pass.
492
238
          for (auto &Edge : ResolvedCalleeSummary->calls()) {
493
42
            auto CalleeGUID = Edge.first.getGUID();
494
42
            ExportList.insert(CalleeGUID);
495
42
          }
496
238
          for (auto &Ref : ResolvedCalleeSummary->refs()) {
497
75
            auto GUID = Ref.getGUID();
498
75
            ExportList.insert(GUID);
499
75
          }
500
238
        }
501
238
      }
502
294
    }
503
403
504
403
    
auto GetAdjustedThreshold = [](unsigned Threshold, bool IsHotCallsite) 294
{
505
294
      // Adjust the threshold for next level of imported functions.
506
294
      // The threshold is different for hot callsites because we can then
507
294
      // inline chains of hot calls.
508
294
      if (IsHotCallsite)
509
28
        return Threshold * ImportHotInstrFactor;
510
266
      return Threshold * ImportInstrFactor;
511
266
    };
512
294
513
294
    const auto AdjThreshold = GetAdjustedThreshold(Threshold, IsHotCallsite);
514
294
515
294
    ImportCount++;
516
294
517
294
    // Insert the newly imported function to the worklist.
518
294
    Worklist.emplace_back(ResolvedCalleeSummary, AdjThreshold, VI.getGUID());
519
294
  }
520
1.12k
}
521
522
/// Given the list of globals defined in a module, compute the list of imports
523
/// as well as the list of "exports", i.e. the list of symbols referenced from
524
/// another module (that may require promotion).
525
static void ComputeImportForModule(
526
    const GVSummaryMapTy &DefinedGVSummaries, const ModuleSummaryIndex &Index,
527
    StringRef ModName, FunctionImporter::ImportMapTy &ImportList,
528
529
    StringMap<FunctionImporter::ExportSetTy> *ExportLists = nullptr) {
529
529
  // Worklist contains the list of function imported in this module, for which
530
529
  // we will analyse the callees and may import further down the callgraph.
531
529
  SmallVector<EdgeInfo, 128> Worklist;
532
529
  FunctionImporter::ImportThresholdsTy ImportThresholds;
533
529
534
529
  // Populate the worklist with the import for the functions in the current
535
529
  // module
536
1.10k
  for (auto &GVSummary : DefinedGVSummaries) {
537
#ifndef NDEBUG
538
    // FIXME: Change the GVSummaryMapTy to hold ValueInfo instead of GUID
539
    // so this map look up (and possibly others) can be avoided.
540
    auto VI = Index.getValueInfo(GVSummary.first);
541
#endif
542
1.10k
    if (!Index.isGlobalValueLive(GVSummary.second)) {
543
105
      LLVM_DEBUG(dbgs() << "Ignores Dead GUID: " << VI << "\n");
544
105
      continue;
545
105
    }
546
1.00k
    auto *FuncSummary =
547
1.00k
        dyn_cast<FunctionSummary>(GVSummary.second->getBaseObject());
548
1.00k
    if (!FuncSummary)
549
169
      // Skip import for global variables
550
169
      continue;
551
834
    LLVM_DEBUG(dbgs() << "Initialize import for " << VI << "\n");
552
834
    computeImportForFunction(*FuncSummary, Index, ImportInstrLimit,
553
834
                             DefinedGVSummaries, Worklist, ImportList,
554
834
                             ExportLists, ImportThresholds);
555
834
  }
556
529
557
529
  // Process the newly imported functions and add callees to the worklist.
558
823
  while (!Worklist.empty()) {
559
294
    auto FuncInfo = Worklist.pop_back_val();
560
294
    auto *Summary = std::get<0>(FuncInfo);
561
294
    auto Threshold = std::get<1>(FuncInfo);
562
294
563
294
    computeImportForFunction(*Summary, Index, Threshold, DefinedGVSummaries,
564
294
                             Worklist, ImportList, ExportLists,
565
294
                             ImportThresholds);
566
294
  }
567
529
568
529
  // Print stats about functions considered but rejected for importing
569
529
  // when requested.
570
529
  if (PrintImportFailures) {
571
4
    dbgs() << "Missed imports into module " << ModName << "\n";
572
4
    for (auto &I : ImportThresholds) {
573
2
      auto &ProcessedThreshold = std::get<0>(I.second);
574
2
      auto &CalleeSummary = std::get<1>(I.second);
575
2
      auto &FailureInfo = std::get<2>(I.second);
576
2
      if (CalleeSummary)
577
1
        continue; // We are going to import.
578
1
      assert(FailureInfo);
579
1
      FunctionSummary *FS = nullptr;
580
1
      if (!FailureInfo->VI.getSummaryList().empty())
581
1
        FS = dyn_cast<FunctionSummary>(
582
1
            FailureInfo->VI.getSummaryList()[0]->getBaseObject());
583
1
      dbgs() << FailureInfo->VI
584
1
             << ": Reason = " << getFailureName(FailureInfo->Reason)
585
1
             << ", Threshold = " << ProcessedThreshold
586
1
             << ", Size = " << (FS ? (int)FS->instCount() : 
-10
)
587
1
             << ", MaxHotness = " << getHotnessName(FailureInfo->MaxHotness)
588
1
             << ", Attempts = " << FailureInfo->Attempts << "\n";
589
1
    }
590
4
  }
591
529
}
592
593
#ifndef NDEBUG
594
static bool isGlobalVarSummary(const ModuleSummaryIndex &Index,
595
                               GlobalValue::GUID G) {
596
  if (const auto &VI = Index.getValueInfo(G)) {
597
    auto SL = VI.getSummaryList();
598
    if (!SL.empty())
599
      return SL[0]->getSummaryKind() == GlobalValueSummary::GlobalVarKind;
600
  }
601
  return false;
602
}
603
604
static GlobalValue::GUID getGUID(GlobalValue::GUID G) { return G; }
605
606
template <class T>
607
static unsigned numGlobalVarSummaries(const ModuleSummaryIndex &Index,
608
                                      T &Cont) {
609
  unsigned NumGVS = 0;
610
  for (auto &V : Cont)
611
    if (isGlobalVarSummary(Index, getGUID(V)))
612
      ++NumGVS;
613
  return NumGVS;
614
}
615
#endif
616
617
/// Compute all the import and export for every module using the Index.
618
void llvm::ComputeCrossModuleImport(
619
    const ModuleSummaryIndex &Index,
620
    const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,
621
    StringMap<FunctionImporter::ImportMapTy> &ImportLists,
622
278
    StringMap<FunctionImporter::ExportSetTy> &ExportLists) {
623
278
  // For each module that has function defined, compute the import/export lists.
624
512
  for (auto &DefinedGVSummaries : ModuleToDefinedGVSummaries) {
625
512
    auto &ImportList = ImportLists[DefinedGVSummaries.first()];
626
512
    LLVM_DEBUG(dbgs() << "Computing import for Module '"
627
512
                      << DefinedGVSummaries.first() << "'\n");
628
512
    ComputeImportForModule(DefinedGVSummaries.second, Index,
629
512
                           DefinedGVSummaries.first(), ImportList,
630
512
                           &ExportLists);
631
512
  }
632
278
633
278
  // When computing imports we added all GUIDs referenced by anything
634
278
  // imported from the module to its ExportList. Now we prune each ExportList
635
278
  // of any not defined in that module. This is more efficient than checking
636
278
  // while computing imports because some of the summary lists may be long
637
278
  // due to linkonce (comdat) copies.
638
278
  for (auto &ELI : ExportLists) {
639
159
    const auto &DefinedGVSummaries =
640
159
        ModuleToDefinedGVSummaries.lookup(ELI.first());
641
495
    for (auto EI = ELI.second.begin(); EI != ELI.second.end();) {
642
336
      if (!DefinedGVSummaries.count(*EI))
643
18
        EI = ELI.second.erase(EI);
644
318
      else
645
318
        ++EI;
646
336
    }
647
159
  }
648
278
649
#ifndef NDEBUG
650
  LLVM_DEBUG(dbgs() << "Import/Export lists for " << ImportLists.size()
651
                    << " modules:\n");
652
  for (auto &ModuleImports : ImportLists) {
653
    auto ModName = ModuleImports.first();
654
    auto &Exports = ExportLists[ModName];
655
    unsigned NumGVS = numGlobalVarSummaries(Index, Exports);
656
    LLVM_DEBUG(dbgs() << "* Module " << ModName << " exports "
657
                      << Exports.size() - NumGVS << " functions and " << NumGVS
658
                      << " vars. Imports from " << ModuleImports.second.size()
659
                      << " modules.\n");
660
    for (auto &Src : ModuleImports.second) {
661
      auto SrcModName = Src.first();
662
      unsigned NumGVSPerMod = numGlobalVarSummaries(Index, Src.second);
663
      LLVM_DEBUG(dbgs() << " - " << Src.second.size() - NumGVSPerMod
664
                        << " functions imported from " << SrcModName << "\n");
665
      LLVM_DEBUG(dbgs() << " - " << NumGVSPerMod
666
                        << " global vars imported from " << SrcModName << "\n");
667
    }
668
  }
669
#endif
670
}
671
672
#ifndef NDEBUG
673
static void dumpImportListForModule(const ModuleSummaryIndex &Index,
674
                                    StringRef ModulePath,
675
                                    FunctionImporter::ImportMapTy &ImportList) {
676
  LLVM_DEBUG(dbgs() << "* Module " << ModulePath << " imports from "
677
                    << ImportList.size() << " modules.\n");
678
  for (auto &Src : ImportList) {
679
    auto SrcModName = Src.first();
680
    unsigned NumGVSPerMod = numGlobalVarSummaries(Index, Src.second);
681
    LLVM_DEBUG(dbgs() << " - " << Src.second.size() - NumGVSPerMod
682
                      << " functions imported from " << SrcModName << "\n");
683
    LLVM_DEBUG(dbgs() << " - " << NumGVSPerMod << " vars imported from "
684
                      << SrcModName << "\n");
685
  }
686
}
687
#endif
688
689
/// Compute all the imports for the given module in the Index.
690
void llvm::ComputeCrossModuleImportForModule(
691
    StringRef ModulePath, const ModuleSummaryIndex &Index,
692
17
    FunctionImporter::ImportMapTy &ImportList) {
693
17
  // Collect the list of functions this module defines.
694
17
  // GUID -> Summary
695
17
  GVSummaryMapTy FunctionSummaryMap;
696
17
  Index.collectDefinedFunctionsForModule(ModulePath, FunctionSummaryMap);
697
17
698
17
  // Compute the import list for this module.
699
17
  LLVM_DEBUG(dbgs() << "Computing import for Module '" << ModulePath << "'\n");
700
17
  ComputeImportForModule(FunctionSummaryMap, Index, ModulePath, ImportList);
701
17
702
#ifndef NDEBUG
703
  dumpImportListForModule(Index, ModulePath, ImportList);
704
#endif
705
}
706
707
// Mark all external summaries in Index for import into the given module.
708
// Used for distributed builds using a distributed index.
709
void llvm::ComputeCrossModuleImportForModuleFromIndex(
710
    StringRef ModulePath, const ModuleSummaryIndex &Index,
711
4
    FunctionImporter::ImportMapTy &ImportList) {
712
18
  for (auto &GlobalList : Index) {
713
18
    // Ignore entries for undefined references.
714
18
    if (GlobalList.second.SummaryList.empty())
715
2
      continue;
716
16
717
16
    auto GUID = GlobalList.first;
718
16
    assert(GlobalList.second.SummaryList.size() == 1 &&
719
16
           "Expected individual combined index to have one summary per GUID");
720
16
    auto &Summary = GlobalList.second.SummaryList[0];
721
16
    // Skip the summaries for the importing module. These are included to
722
16
    // e.g. record required linkage changes.
723
16
    if (Summary->modulePath() == ModulePath)
724
10
      continue;
725
6
    // Add an entry to provoke importing by thinBackend.
726
6
    ImportList[Summary->modulePath()].insert(GUID);
727
6
  }
728
#ifndef NDEBUG
729
  dumpImportListForModule(Index, ModulePath, ImportList);
730
#endif
731
}
732
733
void llvm::computeDeadSymbols(
734
    ModuleSummaryIndex &Index,
735
    const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
736
533
    function_ref<PrevailingType(GlobalValue::GUID)> isPrevailing) {
737
533
  assert(!Index.withGlobalValueDeadStripping());
738
533
  if (!ComputeDead)
739
4
    return;
740
529
  if (GUIDPreservedSymbols.empty())
741
74
    // Don't do anything when nothing is live, this is friendly with tests.
742
74
    return;
743
455
  unsigned LiveSymbols = 0;
744
455
  SmallVector<ValueInfo, 128> Worklist;
745
455
  Worklist.reserve(GUIDPreservedSymbols.size() * 2);
746
768
  for (auto GUID : GUIDPreservedSymbols) {
747
768
    ValueInfo VI = Index.getValueInfo(GUID);
748
768
    if (!VI)
749
397
      continue;
750
371
    for (auto &S : VI.getSummaryList())
751
372
      S->setLive(true);
752
371
  }
753
455
754
455
  // Add values flagged in the index as live roots to the worklist.
755
773
  for (const auto &Entry : Index) {
756
773
    auto VI = Index.getValueInfo(Entry);
757
773
    for (auto &S : Entry.second.SummaryList)
758
679
      if (S->isLive()) {
759
373
        LLVM_DEBUG(dbgs() << "Live root: " << VI << "\n");
760
373
        Worklist.push_back(VI);
761
373
        ++LiveSymbols;
762
373
        break;
763
373
      }
764
773
  }
765
455
766
455
  // Make value live and add it to the worklist if it was not live before.
767
455
  auto visit = [&](ValueInfo VI) {
768
385
    // FIXME: If we knew which edges were created for indirect call profiles,
769
385
    // we could skip them here. Any that are live should be reached via
770
385
    // other edges, e.g. reference edges. Otherwise, using a profile collected
771
385
    // on a slightly different binary might provoke preserving, importing
772
385
    // and ultimately promoting calls to functions not linked into this
773
385
    // binary, which increases the binary size unnecessarily. Note that
774
385
    // if this code changes, the importer needs to change so that edges
775
385
    // to functions marked dead are skipped.
776
385
    VI = updateValueInfoForIndirectCalls(Index, VI);
777
385
    if (!VI)
778
56
      return;
779
329
780
329
    if (llvm::any_of(VI.getSummaryList(),
781
332
                     [](const std::unique_ptr<llvm::GlobalValueSummary> &S) {
782
332
                       return S->isLive();
783
332
                     }))
784
126
      return;
785
203
786
203
    // We only keep live symbols that are known to be non-prevailing if any are
787
203
    // available_externally, linkonceodr, weakodr. Those symbols are discarded
788
203
    // later in the EliminateAvailableExternally pass and setting them to
789
203
    // not-live could break downstreams users of liveness information (PR36483)
790
203
    // or limit optimization opportunities.
791
203
    if (isPrevailing(VI.getGUID()) == PrevailingType::No) {
792
16
      bool KeepAliveLinkage = false;
793
16
      bool Interposable = false;
794
17
      for (auto &S : VI.getSummaryList()) {
795
17
        if (S->linkage() == GlobalValue::AvailableExternallyLinkage ||
796
17
            
S->linkage() == GlobalValue::WeakODRLinkage14
||
797
17
            
S->linkage() == GlobalValue::LinkOnceODRLinkage13
)
798
6
          KeepAliveLinkage = true;
799
11
        else if (GlobalValue::isInterposableLinkage(S->linkage()))
800
3
          Interposable = true;
801
17
      }
802
16
803
16
      if (!KeepAliveLinkage)
804
10
        return;
805
6
806
6
      if (Interposable)
807
1
        report_fatal_error(
808
1
          "Interposable and available_externally/linkonce_odr/weak_odr symbol");
809
192
    }
810
192
811
192
    for (auto &S : VI.getSummaryList())
812
194
      S->setLive(true);
813
192
    ++LiveSymbols;
814
192
    Worklist.push_back(VI);
815
192
  };
816
455
817
1.02k
  while (!Worklist.empty()) {
818
565
    auto VI = Worklist.pop_back_val();
819
574
    for (auto &Summary : VI.getSummaryList()) {
820
574
      if (auto *AS = dyn_cast<AliasSummary>(Summary.get())) {
821
18
        // If this is an alias, visit the aliasee VI to ensure that all copies
822
18
        // are marked live and it is added to the worklist for further
823
18
        // processing of its references.
824
18
        visit(AS->getAliaseeVI());
825
18
        continue;
826
18
      }
827
556
828
556
      Summary->setLive(true);
829
556
      for (auto Ref : Summary->refs())
830
149
        visit(Ref);
831
556
      if (auto *FS = dyn_cast<FunctionSummary>(Summary.get()))
832
444
        for (auto Call : FS->calls())
833
218
          visit(Call.first);
834
556
    }
835
565
  }
836
455
  Index.setWithGlobalValueDeadStripping();
837
455
838
455
  unsigned DeadSymbols = Index.size() - LiveSymbols;
839
455
  LLVM_DEBUG(dbgs() << LiveSymbols << " symbols Live, and " << DeadSymbols
840
455
                    << " symbols Dead \n");
841
455
  NumDeadSymbols += DeadSymbols;
842
455
  NumLiveSymbols += LiveSymbols;
843
455
}
844
845
// Compute dead symbols and propagate constants in combined index.
846
void llvm::computeDeadSymbolsWithConstProp(
847
    ModuleSummaryIndex &Index,
848
    const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
849
    function_ref<PrevailingType(GlobalValue::GUID)> isPrevailing,
850
533
    bool ImportEnabled) {
851
533
  computeDeadSymbols(Index, GUIDPreservedSymbols, isPrevailing);
852
533
  if (ImportEnabled) {
853
520
    Index.propagateAttributes(GUIDPreservedSymbols);
854
520
  } else {
855
13
    // If import is disabled we should drop read/write-only attribute
856
13
    // from all summaries to prevent internalization.
857
13
    for (auto &P : Index)
858
12
      for (auto &S : P.second.SummaryList)
859
12
        if (auto *GVS = dyn_cast<GlobalVarSummary>(S.get())) {
860
1
          GVS->setReadOnly(false);
861
1
          GVS->setWriteOnly(false);
862
1
        }
863
13
  }
864
533
}
865
866
/// Compute the set of summaries needed for a ThinLTO backend compilation of
867
/// \p ModulePath.
868
void llvm::gatherImportedSummariesForModule(
869
    StringRef ModulePath,
870
    const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,
871
    const FunctionImporter::ImportMapTy &ImportList,
872
66
    std::map<std::string, GVSummaryMapTy> &ModuleToSummariesForIndex) {
873
66
  // Include all summaries from the importing module.
874
66
  ModuleToSummariesForIndex[ModulePath] =
875
66
      ModuleToDefinedGVSummaries.lookup(ModulePath);
876
66
  // Include summaries for imports.
877
66
  for (auto &ILI : ImportList) {
878
20
    auto &SummariesForIndex = ModuleToSummariesForIndex[ILI.first()];
879
20
    const auto &DefinedGVSummaries =
880
20
        ModuleToDefinedGVSummaries.lookup(ILI.first());
881
28
    for (auto &GI : ILI.second) {
882
28
      const auto &DS = DefinedGVSummaries.find(GI);
883
28
      assert(DS != DefinedGVSummaries.end() &&
884
28
             "Expected a defined summary for imported global value");
885
28
      SummariesForIndex[GI] = DS->second;
886
28
    }
887
20
  }
888
66
}
889
890
/// Emit the files \p ModulePath will import from into \p OutputFilename.
891
std::error_code llvm::EmitImportsFiles(
892
    StringRef ModulePath, StringRef OutputFilename,
893
40
    const std::map<std::string, GVSummaryMapTy> &ModuleToSummariesForIndex) {
894
40
  std::error_code EC;
895
40
  raw_fd_ostream ImportsOS(OutputFilename, EC, sys::fs::OpenFlags::F_None);
896
40
  if (EC)
897
2
    return EC;
898
38
  for (auto &ILI : ModuleToSummariesForIndex)
899
49
    // The ModuleToSummariesForIndex map includes an entry for the current
900
49
    // Module (needed for writing out the index files). We don't want to
901
49
    // include it in the imports file, however, so filter it out.
902
49
    if (ILI.first != ModulePath)
903
11
      ImportsOS << ILI.first << "\n";
904
38
  return std::error_code();
905
38
}
906
907
90
bool llvm::convertToDeclaration(GlobalValue &GV) {
908
90
  LLVM_DEBUG(dbgs() << "Converting to a declaration: `" << GV.getName()
909
90
                    << "\n");
910
90
  if (Function *F = dyn_cast<Function>(&GV)) {
911
45
    F->deleteBody();
912
45
    F->clearMetadata();
913
45
    F->setComdat(nullptr);
914
45
  } else if (GlobalVariable *V = dyn_cast<GlobalVariable>(&GV)) {
915
38
    V->setInitializer(nullptr);
916
38
    V->setLinkage(GlobalValue::ExternalLinkage);
917
38
    V->clearMetadata();
918
38
    V->setComdat(nullptr);
919
38
  } else {
920
7
    GlobalValue *NewGV;
921
7
    if (GV.getValueType()->isFunctionTy())
922
2
      NewGV =
923
2
          Function::Create(cast<FunctionType>(GV.getValueType()),
924
2
                           GlobalValue::ExternalLinkage, GV.getAddressSpace(),
925
2
                           "", GV.getParent());
926
5
    else
927
5
      NewGV =
928
5
          new GlobalVariable(*GV.getParent(), GV.getValueType(),
929
5
                             /*isConstant*/ false, GlobalValue::ExternalLinkage,
930
5
                             /*init*/ nullptr, "",
931
5
                             /*insertbefore*/ nullptr, GV.getThreadLocalMode(),
932
5
                             GV.getType()->getAddressSpace());
933
7
    NewGV->takeName(&GV);
934
7
    GV.replaceAllUsesWith(NewGV);
935
7
    return false;
936
7
  }
937
83
  return true;
938
83
}
939
940
/// Fixup prevailing symbol linkages in \p TheModule based on summary analysis.
941
void llvm::thinLTOResolvePrevailingInModule(
942
365
    Module &TheModule, const GVSummaryMapTy &DefinedGlobals) {
943
1.01k
  auto updateLinkage = [&](GlobalValue &GV) {
944
1.01k
    // See if the global summary analysis computed a new resolved linkage.
945
1.01k
    const auto &GS = DefinedGlobals.find(GV.getGUID());
946
1.01k
    if (GS == DefinedGlobals.end())
947
343
      return;
948
676
    auto NewLinkage = GS->second->linkage();
949
676
    if (NewLinkage == GV.getLinkage())
950
522
      return;
951
154
952
154
    // Switch the linkage to weakany if asked for, e.g. we do this for
953
154
    // linker redefined symbols (via --wrap or --defsym).
954
154
    // We record that the visibility should be changed here in `addThinLTO`
955
154
    // as we need access to the resolution vectors for each input file in
956
154
    // order to find which symbols have been redefined.
957
154
    // We may consider reorganizing this code and moving the linkage recording
958
154
    // somewhere else, e.g. in thinLTOResolvePrevailingInIndex.
959
154
    if (NewLinkage == GlobalValue::WeakAnyLinkage) {
960
28
      GV.setLinkage(NewLinkage);
961
28
      return;
962
28
    }
963
126
964
126
    if (GlobalValue::isLocalLinkage(GV.getLinkage()) ||
965
126
        // In case it was dead and already converted to declaration.
966
126
        
GV.isDeclaration()124
)
967
10
      return;
968
116
    // Check for a non-prevailing def that has interposable linkage
969
116
    // (e.g. non-odr weak or linkonce). In that case we can't simply
970
116
    // convert to available_externally, since it would lose the
971
116
    // interposable property and possibly get inlined. Simply drop
972
116
    // the definition in that case.
973
116
    if (GlobalValue::isAvailableExternallyLinkage(NewLinkage) &&
974
116
        
GlobalValue::isInterposableLinkage(GV.getLinkage())16
) {
975
2
      if (!convertToDeclaration(GV))
976
2
        // FIXME: Change this to collect replaced GVs and later erase
977
2
        // them from the parent module once thinLTOResolvePrevailingGUID is
978
2
        // changed to enable this for aliases.
979
2
        
llvm_unreachable0
("Expected GV to be converted");
980
114
    } else {
981
114
      // If all copies of the original symbol had global unnamed addr and
982
114
      // linkonce_odr linkage, it should be an auto hide symbol. In that case
983
114
      // the thin link would have marked it as CanAutoHide. Add hidden visibility
984
114
      // to the symbol to preserve the property.
985
114
      if (NewLinkage == GlobalValue::WeakODRLinkage &&
986
114
          
GS->second->canAutoHide()40
) {
987
3
        assert(GV.hasLinkOnceODRLinkage() && GV.hasGlobalUnnamedAddr());
988
3
        GV.setVisibility(GlobalValue::HiddenVisibility);
989
3
      }
990
114
991
114
      LLVM_DEBUG(dbgs() << "ODR fixing up linkage for `" << GV.getName()
992
114
                        << "` from " << GV.getLinkage() << " to " << NewLinkage
993
114
                        << "\n");
994
114
      GV.setLinkage(NewLinkage);
995
114
    }
996
116
    // Remove declarations from comdats, including available_externally
997
116
    // as this is a declaration for the linker, and will be dropped eventually.
998
116
    // It is illegal for comdats to contain declarations.
999
116
    auto *GO = dyn_cast_or_null<GlobalObject>(&GV);
1000
116
    if (GO && 
GO->isDeclarationForLinker()94
&&
GO->hasComdat()16
)
1001
7
      GO->setComdat(nullptr);
1002
116
  };
1003
365
1004
365
  // Process functions and global now
1005
365
  for (auto &GV : TheModule)
1006
750
    updateLinkage(GV);
1007
365
  for (auto &GV : TheModule.globals())
1008
160
    updateLinkage(GV);
1009
365
  for (auto &GV : TheModule.aliases())
1010
110
    updateLinkage(GV);
1011
365
}
1012
1013
/// Run internalization on \p TheModule based on symmary analysis.
1014
void llvm::thinLTOInternalizeModule(Module &TheModule,
1015
334
                                    const GVSummaryMapTy &DefinedGlobals) {
1016
334
  // Declare a callback for the internalize pass that will ask for every
1017
334
  // candidate GlobalValue if it can be internalized or not.
1018
428
  auto MustPreserveGV = [&](const GlobalValue &GV) -> bool {
1019
428
    // Lookup the linkage recorded in the summaries during global analysis.
1020
428
    auto GS = DefinedGlobals.find(GV.getGUID());
1021
428
    if (GS == DefinedGlobals.end()) {
1022
22
      // Must have been promoted (possibly conservatively). Find original
1023
22
      // name so that we can access the correct summary and see if it can
1024
22
      // be internalized again.
1025
22
      // FIXME: Eventually we should control promotion instead of promoting
1026
22
      // and internalizing again.
1027
22
      StringRef OrigName =
1028
22
          ModuleSummaryIndex::getOriginalNameBeforePromote(GV.getName());
1029
22
      std::string OrigId = GlobalValue::getGlobalIdentifier(
1030
22
          OrigName, GlobalValue::InternalLinkage,
1031
22
          TheModule.getSourceFileName());
1032
22
      GS = DefinedGlobals.find(GlobalValue::getGUID(OrigId));
1033
22
      if (GS == DefinedGlobals.end()) {
1034
0
        // Also check the original non-promoted non-globalized name. In some
1035
0
        // cases a preempted weak value is linked in as a local copy because
1036
0
        // it is referenced by an alias (IRLinker::linkGlobalValueProto).
1037
0
        // In that case, since it was originally not a local value, it was
1038
0
        // recorded in the index using the original name.
1039
0
        // FIXME: This may not be needed once PR27866 is fixed.
1040
0
        GS = DefinedGlobals.find(GlobalValue::getGUID(OrigName));
1041
0
        assert(GS != DefinedGlobals.end());
1042
0
      }
1043
22
    }
1044
428
    return !GlobalValue::isLocalLinkage(GS->second->linkage());
1045
428
  };
1046
334
1047
334
  // FIXME: See if we can just internalize directly here via linkage changes
1048
334
  // based on the index, rather than invoking internalizeModule.
1049
334
  internalizeModule(TheModule, MustPreserveGV);
1050
334
}
1051
1052
/// Make alias a clone of its aliasee.
1053
30
static Function *replaceAliasWithAliasee(Module *SrcModule, GlobalAlias *GA) {
1054
30
  Function *Fn = cast<Function>(GA->getBaseObject());
1055
30
1056
30
  ValueToValueMapTy VMap;
1057
30
  Function *NewFn = CloneFunction(Fn, VMap);
1058
30
  // Clone should use the original alias's linkage, visibility and name, and we
1059
30
  // ensure all uses of alias instead use the new clone (casted if necessary).
1060
30
  NewFn->setLinkage(GA->getLinkage());
1061
30
  NewFn->setVisibility(GA->getVisibility());
1062
30
  GA->replaceAllUsesWith(ConstantExpr::getBitCast(NewFn, GA->getType()));
1063
30
  NewFn->takeName(GA);
1064
30
  return NewFn;
1065
30
}
1066
1067
// Internalize values that we marked with specific attribute
1068
// in processGlobalForThinLTO.
1069
406
static void internalizeGVsAfterImport(Module &M) {
1070
406
  for (auto &GV : M.globals())
1071
209
    // Skip GVs which have been converted to declarations
1072
209
    // by dropDeadSymbols.
1073
209
    if (!GV.isDeclaration() && 
GV.hasAttribute("thinlto-internalize")151
) {
1074
44
      GV.setLinkage(GlobalValue::InternalLinkage);
1075
44
      GV.setVisibility(GlobalValue::DefaultVisibility);
1076
44
    }
1077
406
}
1078
1079
// Automatically import functions in Module \p DestModule based on the summaries
1080
// index.
1081
Expected<bool> FunctionImporter::importFunctions(
1082
403
    Module &DestModule, const FunctionImporter::ImportMapTy &ImportList) {
1083
403
  LLVM_DEBUG(dbgs() << "Starting import for Module "
1084
403
                    << DestModule.getModuleIdentifier() << "\n");
1085
403
  unsigned ImportedCount = 0, ImportedGVCount = 0;
1086
403
1087
403
  IRMover Mover(DestModule);
1088
403
  // Do the actual import of functions now, one Module at a time
1089
403
  std::set<StringRef> ModuleNameOrderedList;
1090
403
  for (auto &FunctionsToImportPerModule : ImportList) {
1091
159
    ModuleNameOrderedList.insert(FunctionsToImportPerModule.first());
1092
159
  }
1093
403
  for (auto &Name : ModuleNameOrderedList) {
1094
159
    // Get the module for the import
1095
159
    const auto &FunctionsToImportPerModule = ImportList.find(Name);
1096
159
    assert(FunctionsToImportPerModule != ImportList.end());
1097
159
    Expected<std::unique_ptr<Module>> SrcModuleOrErr = ModuleLoader(Name);
1098
159
    if (!SrcModuleOrErr)
1099
0
      return SrcModuleOrErr.takeError();
1100
159
    std::unique_ptr<Module> SrcModule = std::move(*SrcModuleOrErr);
1101
159
    assert(&DestModule.getContext() == &SrcModule->getContext() &&
1102
159
           "Context mismatch");
1103
159
1104
159
    // If modules were created with lazy metadata loading, materialize it
1105
159
    // now, before linking it (otherwise this will be a noop).
1106
159
    if (Error Err = SrcModule->materializeMetadata())
1107
0
      return std::move(Err);
1108
159
1109
159
    auto &ImportGUIDs = FunctionsToImportPerModule->second;
1110
159
    // Find the globals to import
1111
159
    SetVector<GlobalValue *> GlobalsToImport;
1112
536
    for (Function &F : *SrcModule) {
1113
536
      if (!F.hasName())
1114
0
        continue;
1115
536
      auto GUID = F.getGUID();
1116
536
      auto Import = ImportGUIDs.count(GUID);
1117
536
      LLVM_DEBUG(dbgs() << (Import ? "Is" : "Not") << " importing function "
1118
536
                        << GUID << " " << F.getName() << " from "
1119
536
                        << SrcModule->getSourceFileName() << "\n");
1120
536
      if (Import) {
1121
215
        if (Error Err = F.materialize())
1122
0
          return std::move(Err);
1123
215
        if (EnableImportMetadata) {
1124
2
          // Add 'thinlto_src_module' metadata for statistics and debugging.
1125
2
          F.setMetadata(
1126
2
              "thinlto_src_module",
1127
2
              MDNode::get(DestModule.getContext(),
1128
2
                          {MDString::get(DestModule.getContext(),
1129
2
                                         SrcModule->getSourceFileName())}));
1130
2
        }
1131
215
        GlobalsToImport.insert(&F);
1132
215
      }
1133
536
    }
1134
159
    for (GlobalVariable &GV : SrcModule->globals()) {
1135
149
      if (!GV.hasName())
1136
0
        continue;
1137
149
      auto GUID = GV.getGUID();
1138
149
      auto Import = ImportGUIDs.count(GUID);
1139
149
      LLVM_DEBUG(dbgs() << (Import ? "Is" : "Not") << " importing global "
1140
149
                        << GUID << " " << GV.getName() << " from "
1141
149
                        << SrcModule->getSourceFileName() << "\n");
1142
149
      if (Import) {
1143
48
        if (Error Err = GV.materialize())
1144
0
          return std::move(Err);
1145
48
        ImportedGVCount += GlobalsToImport.insert(&GV);
1146
48
      }
1147
149
    }
1148
159
    for (GlobalAlias &GA : SrcModule->aliases()) {
1149
84
      if (!GA.hasName())
1150
0
        continue;
1151
84
      auto GUID = GA.getGUID();
1152
84
      auto Import = ImportGUIDs.count(GUID);
1153
84
      LLVM_DEBUG(dbgs() << (Import ? "Is" : "Not") << " importing alias "
1154
84
                        << GUID << " " << GA.getName() << " from "
1155
84
                        << SrcModule->getSourceFileName() << "\n");
1156
84
      if (Import) {
1157
30
        if (Error Err = GA.materialize())
1158
0
          return std::move(Err);
1159
30
        // Import alias as a copy of its aliasee.
1160
30
        GlobalObject *Base = GA.getBaseObject();
1161
30
        if (Error Err = Base->materialize())
1162
0
          return std::move(Err);
1163
30
        auto *Fn = replaceAliasWithAliasee(SrcModule.get(), &GA);
1164
30
        LLVM_DEBUG(dbgs() << "Is importing aliasee fn " << Base->getGUID()
1165
30
                          << " " << Base->getName() << " from "
1166
30
                          << SrcModule->getSourceFileName() << "\n");
1167
30
        if (EnableImportMetadata) {
1168
2
          // Add 'thinlto_src_module' metadata for statistics and debugging.
1169
2
          Fn->setMetadata(
1170
2
              "thinlto_src_module",
1171
2
              MDNode::get(DestModule.getContext(),
1172
2
                          {MDString::get(DestModule.getContext(),
1173
2
                                         SrcModule->getSourceFileName())}));
1174
2
        }
1175
30
        GlobalsToImport.insert(Fn);
1176
30
      }
1177
84
    }
1178
159
1179
159
    // Upgrade debug info after we're done materializing all the globals and we
1180
159
    // have loaded all the required metadata!
1181
159
    UpgradeDebugInfo(*SrcModule);
1182
159
1183
159
    // Link in the specified functions.
1184
159
    if (renameModuleForThinLTO(*SrcModule, Index, &GlobalsToImport))
1185
0
      return true;
1186
159
1187
159
    if (PrintImports) {
1188
6
      for (const auto *GV : GlobalsToImport)
1189
7
        dbgs() << DestModule.getSourceFileName() << ": Import " << GV->getName()
1190
7
               << " from " << SrcModule->getSourceFileName() << "\n";
1191
6
    }
1192
159
1193
159
    if (Mover.move(std::move(SrcModule), GlobalsToImport.getArrayRef(),
1194
159
                   [](GlobalValue &, IRMover::ValueAdder) 
{}70
,
1195
159
                   /*IsPerformingImport=*/true))
1196
0
      report_fatal_error("Function Import: link error");
1197
159
1198
159
    ImportedCount += GlobalsToImport.size();
1199
159
    NumImportedModules++;
1200
159
  }
1201
403
1202
403
  internalizeGVsAfterImport(DestModule);
1203
403
1204
403
  NumImportedFunctions += (ImportedCount - ImportedGVCount);
1205
403
  NumImportedGlobalVars += ImportedGVCount;
1206
403
1207
403
  LLVM_DEBUG(dbgs() << "Imported " << ImportedCount - ImportedGVCount
1208
403
                    << " functions for Module "
1209
403
                    << DestModule.getModuleIdentifier() << "\n");
1210
403
  LLVM_DEBUG(dbgs() << "Imported " << ImportedGVCount
1211
403
                    << " global variables for Module "
1212
403
                    << DestModule.getModuleIdentifier() << "\n");
1213
403
  return ImportedCount;
1214
403
}
1215
1216
21
static bool doImportingForModule(Module &M) {
1217
21
  if (SummaryFile.empty())
1218
0
    report_fatal_error("error: -function-import requires -summary-file\n");
1219
21
  Expected<std::unique_ptr<ModuleSummaryIndex>> IndexPtrOrErr =
1220
21
      getModuleSummaryIndexForFile(SummaryFile);
1221
21
  if (!IndexPtrOrErr) {
1222
0
    logAllUnhandledErrors(IndexPtrOrErr.takeError(), errs(),
1223
0
                          "Error loading file '" + SummaryFile + "': ");
1224
0
    return false;
1225
0
  }
1226
21
  std::unique_ptr<ModuleSummaryIndex> Index = std::move(*IndexPtrOrErr);
1227
21
1228
21
  // First step is collecting the import list.
1229
21
  FunctionImporter::ImportMapTy ImportList;
1230
21
  // If requested, simply import all functions in the index. This is used
1231
21
  // when testing distributed backend handling via the opt tool, when
1232
21
  // we have distributed indexes containing exactly the summaries to import.
1233
21
  if (ImportAllIndex)
1234
4
    ComputeCrossModuleImportForModuleFromIndex(M.getModuleIdentifier(), *Index,
1235
4
                                               ImportList);
1236
17
  else
1237
17
    ComputeCrossModuleImportForModule(M.getModuleIdentifier(), *Index,
1238
17
                                      ImportList);
1239
21
1240
21
  // Conservatively mark all internal values as promoted. This interface is
1241
21
  // only used when doing importing via the function importing pass. The pass
1242
21
  // is only enabled when testing importing via the 'opt' tool, which does
1243
21
  // not do the ThinLink that would normally determine what values to promote.
1244
166
  for (auto &I : *Index) {
1245
166
    for (auto &S : I.second.SummaryList) {
1246
164
      if (GlobalValue::isLocalLinkage(S->linkage()))
1247
5
        S->setLinkage(GlobalValue::ExternalLinkage);
1248
164
    }
1249
166
  }
1250
21
1251
21
  // Next we need to promote to global scope and rename any local values that
1252
21
  // are potentially exported to other modules.
1253
21
  if (renameModuleForThinLTO(M, *Index, nullptr)) {
1254
0
    errs() << "Error renaming module\n";
1255
0
    return false;
1256
0
  }
1257
21
1258
21
  // Perform the import now.
1259
21
  auto ModuleLoader = [&M](StringRef Identifier) {
1260
20
    return loadFile(Identifier, M.getContext());
1261
20
  };
1262
21
  FunctionImporter Importer(*Index, ModuleLoader);
1263
21
  Expected<bool> Result = Importer.importFunctions(M, ImportList);
1264
21
1265
21
  // FIXME: Probably need to propagate Errors through the pass manager.
1266
21
  if (!Result) {
1267
0
    logAllUnhandledErrors(Result.takeError(), errs(),
1268
0
                          "Error importing module: ");
1269
0
    return false;
1270
0
  }
1271
21
1272
21
  return *Result;
1273
21
}
1274
1275
namespace {
1276
1277
/// Pass that performs cross-module function import provided a summary file.
1278
class FunctionImportLegacyPass : public ModulePass {
1279
public:
1280
  /// Pass identification, replacement for typeid
1281
  static char ID;
1282
1283
21
  explicit FunctionImportLegacyPass() : ModulePass(ID) {}
1284
1285
  /// Specify pass name for debug output
1286
0
  StringRef getPassName() const override { return "Function Importing"; }
1287
1288
21
  bool runOnModule(Module &M) override {
1289
21
    if (skipModule(M))
1290
0
      return false;
1291
21
1292
21
    return doImportingForModule(M);
1293
21
  }
1294
};
1295
1296
} // end anonymous namespace
1297
1298
PreservedAnalyses FunctionImportPass::run(Module &M,
1299
0
                                          ModuleAnalysisManager &AM) {
1300
0
  if (!doImportingForModule(M))
1301
0
    return PreservedAnalyses::all();
1302
0
1303
0
  return PreservedAnalyses::none();
1304
0
}
1305
1306
char FunctionImportLegacyPass::ID = 0;
1307
INITIALIZE_PASS(FunctionImportLegacyPass, "function-import",
1308
                "Summary Based Function Import", false, false)
1309
1310
namespace llvm {
1311
1312
0
Pass *createFunctionImportPass() {
1313
0
  return new FunctionImportLegacyPass();
1314
0
}
1315
1316
} // end namespace llvm