Coverage Report

Created: 2020-02-25 14:32

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp
Line
Count
Source (jump to first uncovered line)
1
//===-- clang-offload-bundler/ClangOffloadBundler.cpp ---------------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
///
9
/// \file
10
/// This file implements a clang-offload-bundler that bundles different
11
/// files that relate with the same source code but different targets into a
12
/// single one. Also the implements the opposite functionality, i.e. unbundle
13
/// files previous created by this tool.
14
///
15
//===----------------------------------------------------------------------===//
16
17
#include "clang/Basic/Version.h"
18
#include "llvm/ADT/ArrayRef.h"
19
#include "llvm/ADT/SmallString.h"
20
#include "llvm/ADT/SmallVector.h"
21
#include "llvm/ADT/StringMap.h"
22
#include "llvm/ADT/StringRef.h"
23
#include "llvm/ADT/StringSwitch.h"
24
#include "llvm/ADT/Triple.h"
25
#include "llvm/Object/Binary.h"
26
#include "llvm/Object/ObjectFile.h"
27
#include "llvm/Support/Casting.h"
28
#include "llvm/Support/CommandLine.h"
29
#include "llvm/Support/Errc.h"
30
#include "llvm/Support/Error.h"
31
#include "llvm/Support/ErrorOr.h"
32
#include "llvm/Support/FileSystem.h"
33
#include "llvm/Support/MemoryBuffer.h"
34
#include "llvm/Support/Path.h"
35
#include "llvm/Support/Program.h"
36
#include "llvm/Support/Signals.h"
37
#include "llvm/Support/StringSaver.h"
38
#include "llvm/Support/WithColor.h"
39
#include "llvm/Support/raw_ostream.h"
40
#include <algorithm>
41
#include <cassert>
42
#include <cstddef>
43
#include <cstdint>
44
#include <memory>
45
#include <string>
46
#include <system_error>
47
#include <utility>
48
49
using namespace llvm;
50
using namespace llvm::object;
51
52
static cl::opt<bool> Help("h", cl::desc("Alias for -help"), cl::Hidden);
53
54
// Mark all our options with this category, everything else (except for -version
55
// and -help) will be hidden.
56
static cl::OptionCategory
57
    ClangOffloadBundlerCategory("clang-offload-bundler options");
58
59
static cl::list<std::string>
60
    InputFileNames("inputs", cl::CommaSeparated, cl::OneOrMore,
61
                   cl::desc("[<input file>,...]"),
62
                   cl::cat(ClangOffloadBundlerCategory));
63
static cl::list<std::string>
64
    OutputFileNames("outputs", cl::CommaSeparated, cl::OneOrMore,
65
                    cl::desc("[<output file>,...]"),
66
                    cl::cat(ClangOffloadBundlerCategory));
67
static cl::list<std::string>
68
    TargetNames("targets", cl::CommaSeparated, cl::OneOrMore,
69
                cl::desc("[<offload kind>-<target triple>,...]"),
70
                cl::cat(ClangOffloadBundlerCategory));
71
static cl::opt<std::string>
72
    FilesType("type", cl::Required,
73
              cl::desc("Type of the files to be bundled/unbundled.\n"
74
                       "Current supported types are:\n"
75
                       "  i   - cpp-output\n"
76
                       "  ii  - c++-cpp-output\n"
77
                       "  cui - cuda/hip-output\n"
78
                       "  d   - dependency\n"
79
                       "  ll  - llvm\n"
80
                       "  bc  - llvm-bc\n"
81
                       "  s   - assembler\n"
82
                       "  o   - object\n"
83
                       "  gch - precompiled-header\n"
84
                       "  ast - clang AST file"),
85
              cl::cat(ClangOffloadBundlerCategory));
86
static cl::opt<bool>
87
    Unbundle("unbundle",
88
             cl::desc("Unbundle bundled file into several output files.\n"),
89
             cl::init(false), cl::cat(ClangOffloadBundlerCategory));
90
91
static cl::opt<bool> PrintExternalCommands(
92
    "###",
93
    cl::desc("Print any external commands that are to be executed "
94
             "instead of actually executing them - for testing purposes.\n"),
95
    cl::init(false), cl::cat(ClangOffloadBundlerCategory));
96
97
/// Magic string that marks the existence of offloading data.
98
0
#define OFFLOAD_BUNDLER_MAGIC_STR "__CLANG_OFFLOAD_BUNDLE__"
99
100
/// The index of the host input in the list of inputs.
101
static unsigned HostInputIndex = ~0u;
102
103
/// Path to the current binary.
104
static std::string BundlerExecutable;
105
106
/// Obtain the offload kind and real machine triple out of the target
107
/// information specified by the user.
108
static void getOffloadKindAndTriple(StringRef Target, StringRef &OffloadKind,
109
4
                                    StringRef &Triple) {
110
4
  auto KindTriplePair = Target.split('-');
111
4
  OffloadKind = KindTriplePair.first;
112
4
  Triple = KindTriplePair.second;
113
4
}
114
0
static bool hasHostKind(StringRef Target) {
115
0
  StringRef OffloadKind;
116
0
  StringRef Triple;
117
0
  getOffloadKindAndTriple(Target, OffloadKind, Triple);
118
0
  return OffloadKind == "host";
119
0
}
120
121
/// Generic file handler interface.
122
class FileHandler {
123
public:
124
2
  FileHandler() {}
125
126
2
  virtual ~FileHandler() {}
127
128
  /// Update the file handler with information from the header of the bundled
129
  /// file.
130
  virtual Error ReadHeader(MemoryBuffer &Input) = 0;
131
132
  /// Read the marker of the next bundled to be read in the file. The bundle
133
  /// name is returned if there is one in the file, or `None` if there are no
134
  /// more bundles to be read.
135
  virtual Expected<Optional<StringRef>>
136
  ReadBundleStart(MemoryBuffer &Input) = 0;
137
138
  /// Read the marker that closes the current bundle.
139
  virtual Error ReadBundleEnd(MemoryBuffer &Input) = 0;
140
141
  /// Read the current bundle and write the result into the stream \a OS.
142
  virtual Error ReadBundle(raw_fd_ostream &OS, MemoryBuffer &Input) = 0;
143
144
  /// Write the header of the bundled file to \a OS based on the information
145
  /// gathered from \a Inputs.
146
  virtual Error WriteHeader(raw_fd_ostream &OS,
147
                            ArrayRef<std::unique_ptr<MemoryBuffer>> Inputs) = 0;
148
149
  /// Write the marker that initiates a bundle for the triple \a TargetTriple to
150
  /// \a OS.
151
  virtual Error WriteBundleStart(raw_fd_ostream &OS,
152
                                 StringRef TargetTriple) = 0;
153
154
  /// Write the marker that closes a bundle for the triple \a TargetTriple to \a
155
  /// OS.
156
  virtual Error WriteBundleEnd(raw_fd_ostream &OS, StringRef TargetTriple) = 0;
157
158
  /// Write the bundle from \a Input into \a OS.
159
  virtual Error WriteBundle(raw_fd_ostream &OS, MemoryBuffer &Input) = 0;
160
};
161
162
/// Handler for binary files. The bundled file will have the following format
163
/// (all integers are stored in little-endian format):
164
///
165
/// "OFFLOAD_BUNDLER_MAGIC_STR" (ASCII encoding of the string)
166
///
167
/// NumberOfOffloadBundles (8-byte integer)
168
///
169
/// OffsetOfBundle1 (8-byte integer)
170
/// SizeOfBundle1 (8-byte integer)
171
/// NumberOfBytesInTripleOfBundle1 (8-byte integer)
172
/// TripleOfBundle1 (byte length defined before)
173
///
174
/// ...
175
///
176
/// OffsetOfBundleN (8-byte integer)
177
/// SizeOfBundleN (8-byte integer)
178
/// NumberOfBytesInTripleOfBundleN (8-byte integer)
179
/// TripleOfBundleN (byte length defined before)
180
///
181
/// Bundle1
182
/// ...
183
/// BundleN
184
185
/// Read 8-byte integers from a buffer in little-endian format.
186
0
static uint64_t Read8byteIntegerFromBuffer(StringRef Buffer, size_t pos) {
187
0
  uint64_t Res = 0;
188
0
  const char *Data = Buffer.data();
189
0
190
0
  for (unsigned i = 0; i < 8; ++i) {
191
0
    Res <<= 8;
192
0
    uint64_t Char = (uint64_t)Data[pos + 7 - i];
193
0
    Res |= 0xffu & Char;
194
0
  }
195
0
  return Res;
196
0
}
197
198
/// Write 8-byte integers to a buffer in little-endian format.
199
0
static void Write8byteIntegerToBuffer(raw_fd_ostream &OS, uint64_t Val) {
200
0
  for (unsigned i = 0; i < 8; ++i) {
201
0
    char Char = (char)(Val & 0xffu);
202
0
    OS.write(&Char, 1);
203
0
    Val >>= 8;
204
0
  }
205
0
}
206
207
class BinaryFileHandler final : public FileHandler {
208
  /// Information about the bundles extracted from the header.
209
  struct BundleInfo final {
210
    /// Size of the bundle.
211
    uint64_t Size = 0u;
212
    /// Offset at which the bundle starts in the bundled file.
213
    uint64_t Offset = 0u;
214
215
0
    BundleInfo() {}
216
0
    BundleInfo(uint64_t Size, uint64_t Offset) : Size(Size), Offset(Offset) {}
217
  };
218
219
  /// Map between a triple and the corresponding bundle information.
220
  StringMap<BundleInfo> BundlesInfo;
221
222
  /// Iterator for the bundle information that is being read.
223
  StringMap<BundleInfo>::iterator CurBundleInfo;
224
  StringMap<BundleInfo>::iterator NextBundleInfo;
225
226
public:
227
0
  BinaryFileHandler() : FileHandler() {}
228
229
0
  ~BinaryFileHandler() final {}
230
231
0
  Error ReadHeader(MemoryBuffer &Input) final {
232
0
    StringRef FC = Input.getBuffer();
233
0
234
0
    // Initialize the current bundle with the end of the container.
235
0
    CurBundleInfo = BundlesInfo.end();
236
0
237
0
    // Check if buffer is smaller than magic string.
238
0
    size_t ReadChars = sizeof(OFFLOAD_BUNDLER_MAGIC_STR) - 1;
239
0
    if (ReadChars > FC.size())
240
0
      return Error::success();
241
0
242
0
    // Check if no magic was found.
243
0
    StringRef Magic(FC.data(), sizeof(OFFLOAD_BUNDLER_MAGIC_STR) - 1);
244
0
    if (!Magic.equals(OFFLOAD_BUNDLER_MAGIC_STR))
245
0
      return Error::success();
246
0
247
0
    // Read number of bundles.
248
0
    if (ReadChars + 8 > FC.size())
249
0
      return Error::success();
250
0
251
0
    uint64_t NumberOfBundles = Read8byteIntegerFromBuffer(FC, ReadChars);
252
0
    ReadChars += 8;
253
0
254
0
    // Read bundle offsets, sizes and triples.
255
0
    for (uint64_t i = 0; i < NumberOfBundles; ++i) {
256
0
257
0
      // Read offset.
258
0
      if (ReadChars + 8 > FC.size())
259
0
        return Error::success();
260
0
261
0
      uint64_t Offset = Read8byteIntegerFromBuffer(FC, ReadChars);
262
0
      ReadChars += 8;
263
0
264
0
      // Read size.
265
0
      if (ReadChars + 8 > FC.size())
266
0
        return Error::success();
267
0
268
0
      uint64_t Size = Read8byteIntegerFromBuffer(FC, ReadChars);
269
0
      ReadChars += 8;
270
0
271
0
      // Read triple size.
272
0
      if (ReadChars + 8 > FC.size())
273
0
        return Error::success();
274
0
275
0
      uint64_t TripleSize = Read8byteIntegerFromBuffer(FC, ReadChars);
276
0
      ReadChars += 8;
277
0
278
0
      // Read triple.
279
0
      if (ReadChars + TripleSize > FC.size())
280
0
        return Error::success();
281
0
282
0
      StringRef Triple(&FC.data()[ReadChars], TripleSize);
283
0
      ReadChars += TripleSize;
284
0
285
0
      // Check if the offset and size make sense.
286
0
      if (!Offset || Offset + Size > FC.size())
287
0
        return Error::success();
288
0
289
0
      assert(BundlesInfo.find(Triple) == BundlesInfo.end() &&
290
0
             "Triple is duplicated??");
291
0
      BundlesInfo[Triple] = BundleInfo(Size, Offset);
292
0
    }
293
0
    // Set the iterator to where we will start to read.
294
0
    CurBundleInfo = BundlesInfo.end();
295
0
    NextBundleInfo = BundlesInfo.begin();
296
0
    return Error::success();
297
0
  }
298
299
0
  Expected<Optional<StringRef>> ReadBundleStart(MemoryBuffer &Input) final {
300
0
    if (NextBundleInfo == BundlesInfo.end())
301
0
      return None;
302
0
    CurBundleInfo = NextBundleInfo++;
303
0
    return CurBundleInfo->first();
304
0
  }
305
306
0
  Error ReadBundleEnd(MemoryBuffer &Input) final {
307
0
    assert(CurBundleInfo != BundlesInfo.end() && "Invalid reader info!");
308
0
    return Error::success();
309
0
  }
310
311
0
  Error ReadBundle(raw_fd_ostream &OS, MemoryBuffer &Input) final {
312
0
    assert(CurBundleInfo != BundlesInfo.end() && "Invalid reader info!");
313
0
    StringRef FC = Input.getBuffer();
314
0
    OS.write(FC.data() + CurBundleInfo->second.Offset,
315
0
             CurBundleInfo->second.Size);
316
0
    return Error::success();
317
0
  }
318
319
  Error WriteHeader(raw_fd_ostream &OS,
320
0
                    ArrayRef<std::unique_ptr<MemoryBuffer>> Inputs) final {
321
0
    // Compute size of the header.
322
0
    uint64_t HeaderSize = 0;
323
0
324
0
    HeaderSize += sizeof(OFFLOAD_BUNDLER_MAGIC_STR) - 1;
325
0
    HeaderSize += 8; // Number of Bundles
326
0
327
0
    for (auto &T : TargetNames) {
328
0
      HeaderSize += 3 * 8; // Bundle offset, Size of bundle and size of triple.
329
0
      HeaderSize += T.size(); // The triple.
330
0
    }
331
0
332
0
    // Write to the buffer the header.
333
0
    OS << OFFLOAD_BUNDLER_MAGIC_STR;
334
0
335
0
    Write8byteIntegerToBuffer(OS, TargetNames.size());
336
0
337
0
    unsigned Idx = 0;
338
0
    for (auto &T : TargetNames) {
339
0
      MemoryBuffer &MB = *Inputs[Idx++];
340
0
      // Bundle offset.
341
0
      Write8byteIntegerToBuffer(OS, HeaderSize);
342
0
      // Size of the bundle (adds to the next bundle's offset)
343
0
      Write8byteIntegerToBuffer(OS, MB.getBufferSize());
344
0
      HeaderSize += MB.getBufferSize();
345
0
      // Size of the triple
346
0
      Write8byteIntegerToBuffer(OS, T.size());
347
0
      // Triple
348
0
      OS << T;
349
0
    }
350
0
    return Error::success();
351
0
  }
352
353
0
  Error WriteBundleStart(raw_fd_ostream &OS, StringRef TargetTriple) final {
354
0
    return Error::success();
355
0
  }
356
357
0
  Error WriteBundleEnd(raw_fd_ostream &OS, StringRef TargetTriple) final {
358
0
    return Error::success();
359
0
  }
360
361
0
  Error WriteBundle(raw_fd_ostream &OS, MemoryBuffer &Input) final {
362
0
    OS.write(Input.getBufferStart(), Input.getBufferSize());
363
0
    return Error::success();
364
0
  }
365
};
366
367
namespace {
368
369
// This class implements a list of temporary files that are removed upon
370
// object destruction.
371
class TempFileHandlerRAII {
372
public:
373
0
  ~TempFileHandlerRAII() {
374
0
    for (const auto &File : Files)
375
0
      sys::fs::remove(File);
376
0
  }
377
378
  // Creates temporary file with given contents.
379
0
  Expected<StringRef> Create(Optional<ArrayRef<char>> Contents) {
380
0
    SmallString<128u> File;
381
0
    if (std::error_code EC =
382
0
            sys::fs::createTemporaryFile("clang-offload-bundler", "tmp", File))
383
0
      return createFileError(File, EC);
384
0
    Files.push_back(File);
385
0
386
0
    if (Contents) {
387
0
      std::error_code EC;
388
0
      raw_fd_ostream OS(File, EC);
389
0
      if (EC)
390
0
        return createFileError(File, EC);
391
0
      OS.write(Contents->data(), Contents->size());
392
0
    }
393
0
    return Files.back();
394
0
  }
395
396
private:
397
  SmallVector<SmallString<128u>, 4u> Files;
398
};
399
400
} // end anonymous namespace
401
402
/// Handler for object files. The bundles are organized by sections with a
403
/// designated name.
404
///
405
/// To unbundle, we just copy the contents of the designated section.
406
class ObjectFileHandler final : public FileHandler {
407
408
  /// The object file we are currently dealing with.
409
  std::unique_ptr<ObjectFile> Obj;
410
411
  /// Return the input file contents.
412
0
  StringRef getInputFileContents() const { return Obj->getData(); }
413
414
  /// Return bundle name (<kind>-<triple>) if the provided section is an offload
415
  /// section.
416
0
  static Expected<Optional<StringRef>> IsOffloadSection(SectionRef CurSection) {
417
0
    Expected<StringRef> NameOrErr = CurSection.getName();
418
0
    if (!NameOrErr)
419
0
      return NameOrErr.takeError();
420
0
421
0
    // If it does not start with the reserved suffix, just skip this section.
422
0
    if (!NameOrErr->startswith(OFFLOAD_BUNDLER_MAGIC_STR))
423
0
      return None;
424
0
425
0
    // Return the triple that is right after the reserved prefix.
426
0
    return NameOrErr->substr(sizeof(OFFLOAD_BUNDLER_MAGIC_STR) - 1);
427
0
  }
428
429
  /// Total number of inputs.
430
  unsigned NumberOfInputs = 0;
431
432
  /// Total number of processed inputs, i.e, inputs that were already
433
  /// read from the buffers.
434
  unsigned NumberOfProcessedInputs = 0;
435
436
  /// Iterator of the current and next section.
437
  section_iterator CurrentSection;
438
  section_iterator NextSection;
439
440
public:
441
  ObjectFileHandler(std::unique_ptr<ObjectFile> ObjIn)
442
      : FileHandler(), Obj(std::move(ObjIn)),
443
        CurrentSection(Obj->section_begin()),
444
0
        NextSection(Obj->section_begin()) {}
445
446
0
  ~ObjectFileHandler() final {}
447
448
0
  Error ReadHeader(MemoryBuffer &Input) final { return Error::success(); }
449
450
0
  Expected<Optional<StringRef>> ReadBundleStart(MemoryBuffer &Input) final {
451
0
    while (NextSection != Obj->section_end()) {
452
0
      CurrentSection = NextSection;
453
0
      ++NextSection;
454
0
455
0
      // Check if the current section name starts with the reserved prefix. If
456
0
      // so, return the triple.
457
0
      Expected<Optional<StringRef>> TripleOrErr =
458
0
          IsOffloadSection(*CurrentSection);
459
0
      if (!TripleOrErr)
460
0
        return TripleOrErr.takeError();
461
0
      if (*TripleOrErr)
462
0
        return **TripleOrErr;
463
0
    }
464
0
    return None;
465
0
  }
466
467
0
  Error ReadBundleEnd(MemoryBuffer &Input) final { return Error::success(); }
468
469
0
  Error ReadBundle(raw_fd_ostream &OS, MemoryBuffer &Input) final {
470
0
    Expected<StringRef> ContentOrErr = CurrentSection->getContents();
471
0
    if (!ContentOrErr)
472
0
      return ContentOrErr.takeError();
473
0
    StringRef Content = *ContentOrErr;
474
0
475
0
    // Copy fat object contents to the output when extracting host bundle.
476
0
    if (Content.size() == 1u && Content.front() == 0)
477
0
      Content = StringRef(Input.getBufferStart(), Input.getBufferSize());
478
0
479
0
    OS.write(Content.data(), Content.size());
480
0
    return Error::success();
481
0
  }
482
483
  Error WriteHeader(raw_fd_ostream &OS,
484
0
                    ArrayRef<std::unique_ptr<MemoryBuffer>> Inputs) final {
485
0
    assert(HostInputIndex != ~0u && "Host input index not defined.");
486
0
487
0
    // Record number of inputs.
488
0
    NumberOfInputs = Inputs.size();
489
0
    return Error::success();
490
0
  }
491
492
0
  Error WriteBundleStart(raw_fd_ostream &OS, StringRef TargetTriple) final {
493
0
    ++NumberOfProcessedInputs;
494
0
    return Error::success();
495
0
  }
496
497
0
  Error WriteBundleEnd(raw_fd_ostream &OS, StringRef TargetTriple) final {
498
0
    assert(NumberOfProcessedInputs <= NumberOfInputs &&
499
0
           "Processing more inputs that actually exist!");
500
0
    assert(HostInputIndex != ~0u && "Host input index not defined.");
501
0
502
0
    // If this is not the last output, we don't have to do anything.
503
0
    if (NumberOfProcessedInputs != NumberOfInputs)
504
0
      return Error::success();
505
0
506
0
    // We will use llvm-objcopy to add target objects sections to the output
507
0
    // fat object. These sections should have 'exclude' flag set which tells
508
0
    // link editor to remove them from linker inputs when linking executable or
509
0
    // shared library. llvm-objcopy currently does not support adding new
510
0
    // section and changing flags for the added section in one invocation, and
511
0
    // because of that we have to run it two times. First run adds sections and
512
0
    // the second changes flags.
513
0
    // TODO: change it to one run once llvm-objcopy starts supporting that.
514
0
515
0
    // Find llvm-objcopy in order to create the bundle binary.
516
0
    ErrorOr<std::string> Objcopy = sys::findProgramByName(
517
0
        "llvm-objcopy", sys::path::parent_path(BundlerExecutable));
518
0
    if (!Objcopy)
519
0
      Objcopy = sys::findProgramByName("llvm-objcopy");
520
0
    if (!Objcopy)
521
0
      return createStringError(Objcopy.getError(),
522
0
                               "unable to find 'llvm-objcopy' in path");
523
0
524
0
    // We write to the output file directly. So, we close it and use the name
525
0
    // to pass down to llvm-objcopy.
526
0
    OS.close();
527
0
528
0
    // Temporary files that need to be removed.
529
0
    TempFileHandlerRAII TempFiles;
530
0
531
0
    // Create an intermediate temporary file to save object after the first
532
0
    // llvm-objcopy run.
533
0
    Expected<StringRef> IntermediateObjOrErr = TempFiles.Create(None);
534
0
    if (!IntermediateObjOrErr)
535
0
      return IntermediateObjOrErr.takeError();
536
0
    StringRef IntermediateObj = *IntermediateObjOrErr;
537
0
538
0
    // Compose llvm-objcopy command line for add target objects' sections.
539
0
    BumpPtrAllocator Alloc;
540
0
    StringSaver SS{Alloc};
541
0
    SmallVector<StringRef, 8u> ObjcopyArgs{"llvm-objcopy"};
542
0
    for (unsigned I = 0; I < NumberOfInputs; ++I) {
543
0
      StringRef InputFile = InputFileNames[I];
544
0
      if (I == HostInputIndex) {
545
0
        // Special handling for the host bundle. We do not need to add a
546
0
        // standard bundle for the host object since we are going to use fat
547
0
        // object as a host object. Therefore use dummy contents (one zero byte)
548
0
        // when creating section for the host bundle.
549
0
        Expected<StringRef> TempFileOrErr = TempFiles.Create(ArrayRef<char>(0));
550
0
        if (!TempFileOrErr)
551
0
          return TempFileOrErr.takeError();
552
0
        InputFile = *TempFileOrErr;
553
0
      }
554
0
555
0
      ObjcopyArgs.push_back(SS.save(Twine("--add-section=") +
556
0
                                    OFFLOAD_BUNDLER_MAGIC_STR + TargetNames[I] +
557
0
                                    "=" + InputFile));
558
0
    }
559
0
    ObjcopyArgs.push_back(InputFileNames[HostInputIndex]);
560
0
    ObjcopyArgs.push_back(IntermediateObj);
561
0
562
0
    if (Error Err = executeObjcopy(*Objcopy, ObjcopyArgs))
563
0
      return Err;
564
0
565
0
    // And run llvm-objcopy for the second time to update section flags.
566
0
    ObjcopyArgs.resize(1);
567
0
    for (unsigned I = 0; I < NumberOfInputs; ++I)
568
0
      ObjcopyArgs.push_back(SS.save(Twine("--set-section-flags=") +
569
0
                                    OFFLOAD_BUNDLER_MAGIC_STR + TargetNames[I] +
570
0
                                    "=readonly,exclude"));
571
0
    ObjcopyArgs.push_back(IntermediateObj);
572
0
    ObjcopyArgs.push_back(OutputFileNames.front());
573
0
574
0
    if (Error Err = executeObjcopy(*Objcopy, ObjcopyArgs))
575
0
      return Err;
576
0
577
0
    return Error::success();
578
0
  }
579
580
0
  Error WriteBundle(raw_fd_ostream &OS, MemoryBuffer &Input) final {
581
0
    return Error::success();
582
0
  }
583
584
private:
585
0
  static Error executeObjcopy(StringRef Objcopy, ArrayRef<StringRef> Args) {
586
0
    // If the user asked for the commands to be printed out, we do that
587
0
    // instead of executing it.
588
0
    if (PrintExternalCommands) {
589
0
      errs() << "\"" << Objcopy << "\"";
590
0
      for (StringRef Arg : drop_begin(Args, 1))
591
0
        errs() << " \"" << Arg << "\"";
592
0
      errs() << "\n";
593
0
    } else {
594
0
      if (sys::ExecuteAndWait(Objcopy, Args))
595
0
        return createStringError(inconvertibleErrorCode(),
596
0
                                 "'llvm-objcopy' tool failed");
597
0
    }
598
0
    return Error::success();
599
0
  }
600
};
601
602
/// Handler for text files. The bundled file will have the following format.
603
///
604
/// "Comment OFFLOAD_BUNDLER_MAGIC_STR__START__ triple"
605
/// Bundle 1
606
/// "Comment OFFLOAD_BUNDLER_MAGIC_STR__END__ triple"
607
/// ...
608
/// "Comment OFFLOAD_BUNDLER_MAGIC_STR__START__ triple"
609
/// Bundle N
610
/// "Comment OFFLOAD_BUNDLER_MAGIC_STR__END__ triple"
611
class TextFileHandler final : public FileHandler {
612
  /// String that begins a line comment.
613
  StringRef Comment;
614
615
  /// String that initiates a bundle.
616
  std::string BundleStartString;
617
618
  /// String that closes a bundle.
619
  std::string BundleEndString;
620
621
  /// Number of chars read from input.
622
  size_t ReadChars = 0u;
623
624
protected:
625
0
  Error ReadHeader(MemoryBuffer &Input) final { return Error::success(); }
626
627
0
  Expected<Optional<StringRef>> ReadBundleStart(MemoryBuffer &Input) final {
628
0
    StringRef FC = Input.getBuffer();
629
0
630
0
    // Find start of the bundle.
631
0
    ReadChars = FC.find(BundleStartString, ReadChars);
632
0
    if (ReadChars == FC.npos)
633
0
      return None;
634
0
635
0
    // Get position of the triple.
636
0
    size_t TripleStart = ReadChars = ReadChars + BundleStartString.size();
637
0
638
0
    // Get position that closes the triple.
639
0
    size_t TripleEnd = ReadChars = FC.find("\n", ReadChars);
640
0
    if (TripleEnd == FC.npos)
641
0
      return None;
642
0
643
0
    // Next time we read after the new line.
644
0
    ++ReadChars;
645
0
646
0
    return StringRef(&FC.data()[TripleStart], TripleEnd - TripleStart);
647
0
  }
648
649
0
  Error ReadBundleEnd(MemoryBuffer &Input) final {
650
0
    StringRef FC = Input.getBuffer();
651
0
652
0
    // Read up to the next new line.
653
0
    assert(FC[ReadChars] == '\n' && "The bundle should end with a new line.");
654
0
655
0
    size_t TripleEnd = ReadChars = FC.find("\n", ReadChars + 1);
656
0
    if (TripleEnd != FC.npos)
657
0
      // Next time we read after the new line.
658
0
      ++ReadChars;
659
0
660
0
    return Error::success();
661
0
  }
662
663
0
  Error ReadBundle(raw_fd_ostream &OS, MemoryBuffer &Input) final {
664
0
    StringRef FC = Input.getBuffer();
665
0
    size_t BundleStart = ReadChars;
666
0
667
0
    // Find end of the bundle.
668
0
    size_t BundleEnd = ReadChars = FC.find(BundleEndString, ReadChars);
669
0
670
0
    StringRef Bundle(&FC.data()[BundleStart], BundleEnd - BundleStart);
671
0
    OS << Bundle;
672
0
673
0
    return Error::success();
674
0
  }
675
676
  Error WriteHeader(raw_fd_ostream &OS,
677
2
                    ArrayRef<std::unique_ptr<MemoryBuffer>> Inputs) final {
678
2
    return Error::success();
679
2
  }
680
681
4
  Error WriteBundleStart(raw_fd_ostream &OS, StringRef TargetTriple) final {
682
4
    OS << BundleStartString << TargetTriple << "\n";
683
4
    return Error::success();
684
4
  }
685
686
4
  Error WriteBundleEnd(raw_fd_ostream &OS, StringRef TargetTriple) final {
687
4
    OS << BundleEndString << TargetTriple << "\n";
688
4
    return Error::success();
689
4
  }
690
691
4
  Error WriteBundle(raw_fd_ostream &OS, MemoryBuffer &Input) final {
692
4
    OS << Input.getBuffer();
693
4
    return Error::success();
694
4
  }
695
696
public:
697
  TextFileHandler(StringRef Comment)
698
2
      : FileHandler(), Comment(Comment), ReadChars(0) {
699
2
    BundleStartString =
700
2
        "\n" + Comment.str() + " " OFFLOAD_BUNDLER_MAGIC_STR "__START__ ";
701
2
    BundleEndString =
702
2
        "\n" + Comment.str() + " " OFFLOAD_BUNDLER_MAGIC_STR "__END__ ";
703
2
  }
704
};
705
706
/// Return an appropriate object file handler. We use the specific object
707
/// handler if we know how to deal with that format, otherwise we use a default
708
/// binary file handler.
709
static std::unique_ptr<FileHandler>
710
0
CreateObjectFileHandler(MemoryBuffer &FirstInput) {
711
0
  // Check if the input file format is one that we know how to deal with.
712
0
  Expected<std::unique_ptr<Binary>> BinaryOrErr = createBinary(FirstInput);
713
0
714
0
  // We only support regular object files. If failed to open the input as a
715
0
  // known binary or this is not an object file use the default binary handler.
716
0
  if (errorToBool(BinaryOrErr.takeError()) || !isa<ObjectFile>(*BinaryOrErr))
717
0
    return std::make_unique<BinaryFileHandler>();
718
0
719
0
  // Otherwise create an object file handler. The handler will be owned by the
720
0
  // client of this function.
721
0
  return std::make_unique<ObjectFileHandler>(
722
0
      std::unique_ptr<ObjectFile>(cast<ObjectFile>(BinaryOrErr->release())));
723
0
}
724
725
/// Return an appropriate handler given the input files and options.
726
static Expected<std::unique_ptr<FileHandler>>
727
2
CreateFileHandler(MemoryBuffer &FirstInput) {
728
2
  if (FilesType == "i")
729
1
    return std::make_unique<TextFileHandler>(/*Comment=*/"//");
730
1
  if (FilesType == "ii")
731
0
    return std::make_unique<TextFileHandler>(/*Comment=*/"//");
732
1
  if (FilesType == "cui")
733
1
    return std::make_unique<TextFileHandler>(/*Comment=*/"//");
734
0
  // TODO: `.d` should be eventually removed once `-M` and its variants are
735
0
  // handled properly in offload compilation.
736
0
  if (FilesType == "d")
737
0
    return std::make_unique<TextFileHandler>(/*Comment=*/"#");
738
0
  if (FilesType == "ll")
739
0
    return std::make_unique<TextFileHandler>(/*Comment=*/";");
740
0
  if (FilesType == "bc")
741
0
    return std::make_unique<BinaryFileHandler>();
742
0
  if (FilesType == "s")
743
0
    return std::make_unique<TextFileHandler>(/*Comment=*/"#");
744
0
  if (FilesType == "o")
745
0
    return CreateObjectFileHandler(FirstInput);
746
0
  if (FilesType == "gch")
747
0
    return std::make_unique<BinaryFileHandler>();
748
0
  if (FilesType == "ast")
749
0
    return std::make_unique<BinaryFileHandler>();
750
0
751
0
  return createStringError(errc::invalid_argument,
752
0
                           "'" + FilesType + "': invalid file type specified");
753
0
}
754
755
/// Bundle the files. Return true if an error was found.
756
2
static Error BundleFiles() {
757
2
  std::error_code EC;
758
2
759
2
  // Create output file.
760
2
  raw_fd_ostream OutputFile(OutputFileNames.front(), EC, sys::fs::OF_None);
761
2
  if (EC)
762
0
    return createFileError(OutputFileNames.front(), EC);
763
2
764
2
  // Open input files.
765
2
  SmallVector<std::unique_ptr<MemoryBuffer>, 8u> InputBuffers;
766
2
  InputBuffers.reserve(InputFileNames.size());
767
4
  for (auto &I : InputFileNames) {
768
4
    ErrorOr<std::unique_ptr<MemoryBuffer>> CodeOrErr =
769
4
        MemoryBuffer::getFileOrSTDIN(I);
770
4
    if (std::error_code EC = CodeOrErr.getError())
771
0
      return createFileError(I, EC);
772
4
    InputBuffers.emplace_back(std::move(*CodeOrErr));
773
4
  }
774
2
775
2
  // Get the file handler. We use the host buffer as reference.
776
2
  assert(HostInputIndex != ~0u && "Host input index undefined??");
777
2
  Expected<std::unique_ptr<FileHandler>> FileHandlerOrErr =
778
2
      CreateFileHandler(*InputBuffers[HostInputIndex]);
779
2
  if (!FileHandlerOrErr)
780
0
    return FileHandlerOrErr.takeError();
781
2
782
2
  std::unique_ptr<FileHandler> &FH = *FileHandlerOrErr;
783
2
  assert(FH);
784
2
785
2
  // Write header.
786
2
  if (Error Err = FH->WriteHeader(OutputFile, InputBuffers))
787
0
    return Err;
788
2
789
2
  // Write all bundles along with the start/end markers. If an error was found
790
2
  // writing the end of the bundle component, abort the bundle writing.
791
2
  auto Input = InputBuffers.begin();
792
4
  for (auto &Triple : TargetNames) {
793
4
    if (Error Err = FH->WriteBundleStart(OutputFile, Triple))
794
0
      return Err;
795
4
    if (Error Err = FH->WriteBundle(OutputFile, **Input))
796
0
      return Err;
797
4
    if (Error Err = FH->WriteBundleEnd(OutputFile, Triple))
798
0
      return Err;
799
4
    ++Input;
800
4
  }
801
2
  return Error::success();
802
2
}
803
804
// Unbundle the files. Return true if an error was found.
805
0
static Error UnbundleFiles() {
806
0
  // Open Input file.
807
0
  ErrorOr<std::unique_ptr<MemoryBuffer>> CodeOrErr =
808
0
      MemoryBuffer::getFileOrSTDIN(InputFileNames.front());
809
0
  if (std::error_code EC = CodeOrErr.getError())
810
0
    return createFileError(InputFileNames.front(), EC);
811
0
812
0
  MemoryBuffer &Input = **CodeOrErr;
813
0
814
0
  // Select the right files handler.
815
0
  Expected<std::unique_ptr<FileHandler>> FileHandlerOrErr =
816
0
      CreateFileHandler(Input);
817
0
  if (!FileHandlerOrErr)
818
0
    return FileHandlerOrErr.takeError();
819
0
820
0
  std::unique_ptr<FileHandler> &FH = *FileHandlerOrErr;
821
0
  assert(FH);
822
0
823
0
  // Read the header of the bundled file.
824
0
  if (Error Err = FH->ReadHeader(Input))
825
0
    return Err;
826
0
827
0
  // Create a work list that consist of the map triple/output file.
828
0
  StringMap<StringRef> Worklist;
829
0
  auto Output = OutputFileNames.begin();
830
0
  for (auto &Triple : TargetNames) {
831
0
    Worklist[Triple] = *Output;
832
0
    ++Output;
833
0
  }
834
0
835
0
  // Read all the bundles that are in the work list. If we find no bundles we
836
0
  // assume the file is meant for the host target.
837
0
  bool FoundHostBundle = false;
838
0
  while (!Worklist.empty()) {
839
0
    Expected<Optional<StringRef>> CurTripleOrErr = FH->ReadBundleStart(Input);
840
0
    if (!CurTripleOrErr)
841
0
      return CurTripleOrErr.takeError();
842
0
843
0
    // We don't have more bundles.
844
0
    if (!*CurTripleOrErr)
845
0
      break;
846
0
847
0
    StringRef CurTriple = **CurTripleOrErr;
848
0
    assert(!CurTriple.empty());
849
0
850
0
    auto Output = Worklist.find(CurTriple);
851
0
    // The file may have more bundles for other targets, that we don't care
852
0
    // about. Therefore, move on to the next triple
853
0
    if (Output == Worklist.end())
854
0
      continue;
855
0
856
0
    // Check if the output file can be opened and copy the bundle to it.
857
0
    std::error_code EC;
858
0
    raw_fd_ostream OutputFile(Output->second, EC, sys::fs::OF_None);
859
0
    if (EC)
860
0
      return createFileError(Output->second, EC);
861
0
    if (Error Err = FH->ReadBundle(OutputFile, Input))
862
0
      return Err;
863
0
    if (Error Err = FH->ReadBundleEnd(Input))
864
0
      return Err;
865
0
    Worklist.erase(Output);
866
0
867
0
    // Record if we found the host bundle.
868
0
    if (hasHostKind(CurTriple))
869
0
      FoundHostBundle = true;
870
0
  }
871
0
872
0
  // If no bundles were found, assume the input file is the host bundle and
873
0
  // create empty files for the remaining targets.
874
0
  if (Worklist.size() == TargetNames.size()) {
875
0
    for (auto &E : Worklist) {
876
0
      std::error_code EC;
877
0
      raw_fd_ostream OutputFile(E.second, EC, sys::fs::OF_None);
878
0
      if (EC)
879
0
        return createFileError(E.second, EC);
880
0
881
0
      // If this entry has a host kind, copy the input file to the output file.
882
0
      if (hasHostKind(E.first()))
883
0
        OutputFile.write(Input.getBufferStart(), Input.getBufferSize());
884
0
    }
885
0
    return Error::success();
886
0
  }
887
0
888
0
  // If we found elements, we emit an error if none of those were for the host
889
0
  // in case host bundle name was provided in command line.
890
0
  if (!FoundHostBundle && HostInputIndex != ~0u)
891
0
    return createStringError(inconvertibleErrorCode(),
892
0
                             "Can't find bundle for the host target");
893
0
894
0
  // If we still have any elements in the worklist, create empty files for them.
895
0
  for (auto &E : Worklist) {
896
0
    std::error_code EC;
897
0
    raw_fd_ostream OutputFile(E.second, EC, sys::fs::OF_None);
898
0
    if (EC)
899
0
      return createFileError(E.second, EC);
900
0
  }
901
0
902
0
  return Error::success();
903
0
}
904
905
0
static void PrintVersion(raw_ostream &OS) {
906
0
  OS << clang::getClangToolFullVersion("clang-offload-bundler") << '\n';
907
0
}
908
909
2
int main(int argc, const char **argv) {
910
2
  sys::PrintStackTraceOnErrorSignal(argv[0]);
911
2
912
2
  cl::HideUnrelatedOptions(ClangOffloadBundlerCategory);
913
2
  cl::SetVersionPrinter(PrintVersion);
914
2
  cl::ParseCommandLineOptions(
915
2
      argc, argv,
916
2
      "A tool to bundle several input files of the specified type <type> \n"
917
2
      "referring to the same source file but different targets into a single \n"
918
2
      "one. The resulting file can also be unbundled into different files by \n"
919
2
      "this tool if -unbundle is provided.\n");
920
2
921
2
  if (Help) {
922
0
    cl::PrintHelpMessage();
923
0
    return 0;
924
0
  }
925
2
926
2
  auto reportError = [argv](Error E) {
927
0
    logAllUnhandledErrors(std::move(E), WithColor::error(errs(), argv[0]));
928
0
  };
929
2
930
2
  bool Error = false;
931
2
  if (Unbundle) {
932
0
    if (InputFileNames.size() != 1) {
933
0
      Error = true;
934
0
      reportError(createStringError(
935
0
          errc::invalid_argument,
936
0
          "only one input file supported in unbundling mode"));
937
0
    }
938
0
    if (OutputFileNames.size() != TargetNames.size()) {
939
0
      Error = true;
940
0
      reportError(createStringError(errc::invalid_argument,
941
0
                                    "number of output files and targets should "
942
0
                                    "match in unbundling mode"));
943
0
    }
944
2
  } else {
945
2
    if (OutputFileNames.size() != 1) {
946
0
      Error = true;
947
0
      reportError(createStringError(
948
0
          errc::invalid_argument,
949
0
          "only one output file supported in bundling mode"));
950
0
    }
951
2
    if (InputFileNames.size() != TargetNames.size()) {
952
0
      Error = true;
953
0
      reportError(createStringError(
954
0
          errc::invalid_argument,
955
0
          "number of input files and targets should match in bundling mode"));
956
0
    }
957
2
  }
958
2
959
2
  // Verify that the offload kinds and triples are known. We also check that we
960
2
  // have exactly one host target.
961
2
  unsigned Index = 0u;
962
2
  unsigned HostTargetNum = 0u;
963
4
  for (StringRef Target : TargetNames) {
964
4
    StringRef Kind;
965
4
    StringRef Triple;
966
4
    getOffloadKindAndTriple(Target, Kind, Triple);
967
4
968
4
    bool KindIsValid = !Kind.empty();
969
4
    KindIsValid = KindIsValid && StringSwitch<bool>(Kind)
970
4
                                     .Case("host", true)
971
4
                                     .Case("openmp", true)
972
4
                                     .Case("hip", true)
973
4
                                     .Default(false);
974
4
975
4
    bool TripleIsValid = !Triple.empty();
976
4
    llvm::Triple T(Triple);
977
4
    TripleIsValid &= T.getArch() != Triple::UnknownArch;
978
4
979
4
    if (!KindIsValid || !TripleIsValid) {
980
0
      Error = true;
981
0
982
0
      SmallVector<char, 128u> Buf;
983
0
      raw_svector_ostream Msg(Buf);
984
0
      Msg << "invalid target '" << Target << "'";
985
0
      if (!KindIsValid)
986
0
        Msg << ", unknown offloading kind '" << Kind << "'";
987
0
      if (!TripleIsValid)
988
0
        Msg << ", unknown target triple '" << Triple << "'";
989
0
      reportError(createStringError(errc::invalid_argument, Msg.str()));
990
0
    }
991
4
992
4
    if (KindIsValid && Kind == "host") {
993
2
      ++HostTargetNum;
994
2
      // Save the index of the input that refers to the host.
995
2
      HostInputIndex = Index;
996
2
    }
997
4
998
4
    ++Index;
999
4
  }
1000
2
1001
2
  // Host triple is not really needed for unbundling operation, so do not
1002
2
  // treat missing host triple as error if we do unbundling.
1003
2
  if ((Unbundle && 
HostTargetNum > 10
) || (!Unbundle && HostTargetNum != 1)) {
1004
0
    Error = true;
1005
0
    reportError(createStringError(errc::invalid_argument,
1006
0
                                  "expecting exactly one host target but got " +
1007
0
                                      Twine(HostTargetNum)));
1008
0
  }
1009
2
1010
2
  if (Error)
1011
0
    return 1;
1012
2
1013
2
  // Save the current executable directory as it will be useful to find other
1014
2
  // tools.
1015
2
  BundlerExecutable = sys::fs::getMainExecutable(argv[0], &BundlerExecutable);
1016
2
1017
2
  if (llvm::Error Err = Unbundle ? UnbundleFiles() : BundleFiles()) {
1018
0
    reportError(std::move(Err));
1019
0
    return 1;
1020
0
  }
1021
2
  return 0;
1022
2
}