Coverage Report

Created: 2021-01-26 06:56

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/Lex/LiteralSupport.cpp
Line
Count
Source (jump to first uncovered line)
1
//===--- LiteralSupport.cpp - Code to parse and process literals ----------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This file implements the NumericLiteralParser, CharLiteralParser, and
10
// StringLiteralParser interfaces.
11
//
12
//===----------------------------------------------------------------------===//
13
14
#include "clang/Lex/LiteralSupport.h"
15
#include "clang/Basic/CharInfo.h"
16
#include "clang/Basic/LangOptions.h"
17
#include "clang/Basic/SourceLocation.h"
18
#include "clang/Basic/TargetInfo.h"
19
#include "clang/Lex/LexDiagnostic.h"
20
#include "clang/Lex/Lexer.h"
21
#include "clang/Lex/Preprocessor.h"
22
#include "clang/Lex/Token.h"
23
#include "llvm/ADT/APInt.h"
24
#include "llvm/ADT/SmallVector.h"
25
#include "llvm/ADT/StringExtras.h"
26
#include "llvm/ADT/StringSwitch.h"
27
#include "llvm/Support/ConvertUTF.h"
28
#include "llvm/Support/Error.h"
29
#include "llvm/Support/ErrorHandling.h"
30
#include <algorithm>
31
#include <cassert>
32
#include <cstddef>
33
#include <cstdint>
34
#include <cstring>
35
#include <string>
36
37
using namespace clang;
38
39
5.48M
static unsigned getCharWidth(tok::TokenKind kind, const TargetInfo &Target) {
40
5.48M
  switch (kind) {
41
0
  default: llvm_unreachable("Unknown token type!");
42
4.76k
  case tok::char_constant:
43
5.48M
  case tok::string_literal:
44
5.48M
  case tok::utf8_char_constant:
45
5.48M
  case tok::utf8_string_literal:
46
5.48M
    return Target.getCharWidth();
47
129
  case tok::wide_char_constant:
48
1.22k
  case tok::wide_string_literal:
49
1.22k
    return Target.getWCharWidth();
50
34
  case tok::utf16_char_constant:
51
177
  case tok::utf16_string_literal:
52
177
    return Target.getChar16Width();
53
27
  case tok::utf32_char_constant:
54
169
  case tok::utf32_string_literal:
55
169
    return Target.getChar32Width();
56
5.48M
  }
57
5.48M
}
58
59
static CharSourceRange MakeCharSourceRange(const LangOptions &Features,
60
                                           FullSourceLoc TokLoc,
61
                                           const char *TokBegin,
62
                                           const char *TokRangeBegin,
63
367
                                           const char *TokRangeEnd) {
64
367
  SourceLocation Begin =
65
367
    Lexer::AdvanceToTokenCharacter(TokLoc, TokRangeBegin - TokBegin,
66
367
                                   TokLoc.getManager(), Features);
67
367
  SourceLocation End =
68
367
    Lexer::AdvanceToTokenCharacter(Begin, TokRangeEnd - TokRangeBegin,
69
367
                                   TokLoc.getManager(), Features);
70
367
  return CharSourceRange::getCharRange(Begin, End);
71
367
}
72
73
/// Produce a diagnostic highlighting some portion of a literal.
74
///
75
/// Emits the diagnostic \p DiagID, highlighting the range of characters from
76
/// \p TokRangeBegin (inclusive) to \p TokRangeEnd (exclusive), which must be
77
/// a substring of a spelling buffer for the token beginning at \p TokBegin.
78
static DiagnosticBuilder Diag(DiagnosticsEngine *Diags,
79
                              const LangOptions &Features, FullSourceLoc TokLoc,
80
                              const char *TokBegin, const char *TokRangeBegin,
81
223
                              const char *TokRangeEnd, unsigned DiagID) {
82
223
  SourceLocation Begin =
83
223
    Lexer::AdvanceToTokenCharacter(TokLoc, TokRangeBegin - TokBegin,
84
223
                                   TokLoc.getManager(), Features);
85
223
  return Diags->Report(Begin, DiagID) <<
86
223
    MakeCharSourceRange(Features, TokLoc, TokBegin, TokRangeBegin, TokRangeEnd);
87
223
}
88
89
/// ProcessCharEscape - Parse a standard C escape sequence, which can occur in
90
/// either a character or a string literal.
91
static unsigned ProcessCharEscape(const char *ThisTokBegin,
92
                                  const char *&ThisTokBuf,
93
                                  const char *ThisTokEnd, bool &HadError,
94
                                  FullSourceLoc Loc, unsigned CharWidth,
95
                                  DiagnosticsEngine *Diags,
96
72.2k
                                  const LangOptions &Features) {
97
72.2k
  const char *EscapeBegin = ThisTokBuf;
98
99
  // Skip the '\' char.
100
72.2k
  ++ThisTokBuf;
101
102
  // We know that this character can't be off the end of the buffer, because
103
  // that would have been \", which would not have been the end of string.
104
72.2k
  unsigned ResultChar = *ThisTokBuf++;
105
72.2k
  switch (ResultChar) {
106
  // These map to themselves.
107
975
  
case '\\': 399
case '\'': 433
case '"': 974
case '?': break;
108
109
    // These have fixed mappings.
110
26
  case 'a':
111
    // TODO: K&R: the meaning of '\\a' is different in traditional C
112
26
    ResultChar = 7;
113
26
    break;
114
25
  case 'b':
115
25
    ResultChar = 8;
116
25
    break;
117
15
  case 'e':
118
15
    if (Diags)
119
15
      Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
120
15
           diag::ext_nonstandard_escape) << "e";
121
15
    ResultChar = 27;
122
15
    break;
123
1
  case 'E':
124
1
    if (Diags)
125
1
      Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
126
1
           diag::ext_nonstandard_escape) << "E";
127
1
    ResultChar = 27;
128
1
    break;
129
17
  case 'f':
130
17
    ResultChar = 12;
131
17
    break;
132
61.0k
  case 'n':
133
61.0k
    ResultChar = 10;
134
61.0k
    break;
135
64
  case 'r':
136
64
    ResultChar = 13;
137
64
    break;
138
5.96k
  case 't':
139
5.96k
    ResultChar = 9;
140
5.96k
    break;
141
29
  case 'v':
142
29
    ResultChar = 11;
143
29
    break;
144
1.15k
  case 'x': { // Hex escape.
145
1.15k
    ResultChar = 0;
146
1.15k
    if (ThisTokBuf == ThisTokEnd || 
!isHexDigit(*ThisTokBuf)1.15k
) {
147
6
      if (Diags)
148
6
        Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
149
6
             diag::err_hex_escape_no_digits) << "x";
150
6
      HadError = true;
151
6
      break;
152
6
    }
153
154
    // Hex escapes are a maximal series of hex digits.
155
1.15k
    bool Overflow = false;
156
3.69k
    for (; ThisTokBuf != ThisTokEnd; 
++ThisTokBuf2.53k
) {
157
2.83k
      int CharVal = llvm::hexDigitValue(ThisTokBuf[0]);
158
2.83k
      if (CharVal == -1) 
break300
;
159
      // About to shift out a digit?
160
2.53k
      if (ResultChar & 0xF0000000)
161
0
        Overflow = true;
162
2.53k
      ResultChar <<= 4;
163
2.53k
      ResultChar |= CharVal;
164
2.53k
    }
165
166
    // See if any bits will be truncated when evaluated as a character.
167
1.15k
    if (CharWidth != 32 && 
(ResultChar >> CharWidth) != 01.06k
) {
168
0
      Overflow = true;
169
0
      ResultChar &= ~0U >> (32-CharWidth);
170
0
    }
171
172
    // Check for overflow.
173
1.15k
    if (Overflow && 
Diags0
) // Too many digits to fit in
174
0
      Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
175
0
           diag::err_escape_too_large) << 0;
176
1.15k
    break;
177
1.15k
  }
178
2.90k
  
case '0': 2.51k
case '1': 2.71k
case '2': 2.89k
case '3':
179
2.90k
  
case '4': 2.90k
case '5': 2.90k
case '6': 2.90k
case '7': {
180
    // Octal escapes.
181
2.90k
    --ThisTokBuf;
182
2.90k
    ResultChar = 0;
183
184
    // Octal escapes are a series of octal digits with maximum length 3.
185
    // "\0123" is a two digit sequence equal to "\012" "3".
186
2.90k
    unsigned NumDigits = 0;
187
3.17k
    do {
188
3.17k
      ResultChar <<= 3;
189
3.17k
      ResultChar |= *ThisTokBuf++ - '0';
190
3.17k
      ++NumDigits;
191
3.17k
    } while (ThisTokBuf != ThisTokEnd && 
NumDigits < 32.08k
&&
192
2.06k
             ThisTokBuf[0] >= '0' && 
ThisTokBuf[0] <= '7'2.01k
);
193
194
    // Check for overflow.  Reject '\777', but not L'\777'.
195
2.90k
    if (CharWidth != 32 && 
(ResultChar >> CharWidth) != 02.71k
) {
196
1
      if (Diags)
197
1
        Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
198
1
             diag::err_escape_too_large) << 1;
199
1
      ResultChar &= ~0U >> (32-CharWidth);
200
1
    }
201
2.90k
    break;
202
2.90k
  }
203
204
    // Otherwise, these are not valid escapes.
205
28
  
case '(': 5
case '{': 10
case '[': 15
case '%':
206
    // GCC accepts these as extensions.  We warn about them as such though.
207
28
    if (Diags)
208
22
      Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
209
22
           diag::ext_nonstandard_escape)
210
22
        << std::string(1, ResultChar);
211
28
    break;
212
10
  default:
213
10
    if (!Diags)
214
0
      break;
215
216
10
    if (isPrintable(ResultChar))
217
8
      Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
218
8
           diag::ext_unknown_escape)
219
8
        << std::string(1, ResultChar);
220
2
    else
221
2
      Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
222
2
           diag::ext_unknown_escape)
223
2
        << "x" + llvm::utohexstr(ResultChar);
224
10
    break;
225
72.2k
  }
226
227
72.2k
  return ResultChar;
228
72.2k
}
229
230
static void appendCodePoint(unsigned Codepoint,
231
165
                            llvm::SmallVectorImpl<char> &Str) {
232
165
  char ResultBuf[4];
233
165
  char *ResultPtr = ResultBuf;
234
165
  bool Res = llvm::ConvertCodePointToUTF8(Codepoint, ResultPtr);
235
165
  (void)Res;
236
165
  assert(Res && "Unexpected conversion failure");
237
165
  Str.append(ResultBuf, ResultPtr);
238
165
}
239
240
933
void clang::expandUCNs(SmallVectorImpl<char> &Buf, StringRef Input) {
241
3.45k
  for (StringRef::iterator I = Input.begin(), E = Input.end(); I != E; 
++I2.52k
) {
242
2.52k
    if (*I != '\\') {
243
2.35k
      Buf.push_back(*I);
244
2.35k
      continue;
245
2.35k
    }
246
247
165
    ++I;
248
165
    assert(*I == 'u' || *I == 'U');
249
250
165
    unsigned NumHexDigits;
251
165
    if (*I == 'u')
252
144
      NumHexDigits = 4;
253
21
    else
254
21
      NumHexDigits = 8;
255
256
165
    assert(I + NumHexDigits <= E);
257
258
165
    uint32_t CodePoint = 0;
259
909
    for (++I; NumHexDigits != 0; 
++I, --NumHexDigits744
) {
260
744
      unsigned Value = llvm::hexDigitValue(*I);
261
744
      assert(Value != -1U);
262
263
744
      CodePoint <<= 4;
264
744
      CodePoint += Value;
265
744
    }
266
267
165
    appendCodePoint(CodePoint, Buf);
268
165
    --I;
269
165
  }
270
933
}
271
272
/// ProcessUCNEscape - Read the Universal Character Name, check constraints and
273
/// return the UTF32.
274
static bool ProcessUCNEscape(const char *ThisTokBegin, const char *&ThisTokBuf,
275
                             const char *ThisTokEnd,
276
                             uint32_t &UcnVal, unsigned short &UcnLen,
277
                             FullSourceLoc Loc, DiagnosticsEngine *Diags,
278
                             const LangOptions &Features,
279
400
                             bool in_char_string_literal = false) {
280
400
  const char *UcnBegin = ThisTokBuf;
281
282
  // Skip the '\u' char's.
283
400
  ThisTokBuf += 2;
284
285
400
  if (ThisTokBuf == ThisTokEnd || 
!isHexDigit(*ThisTokBuf)397
) {
286
4
    if (Diags)
287
4
      Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf,
288
4
           diag::err_hex_escape_no_digits) << StringRef(&ThisTokBuf[-1], 1);
289
4
    return false;
290
4
  }
291
396
  UcnLen = (ThisTokBuf[-1] == 'u' ? 
4307
:
889
);
292
396
  unsigned short UcnLenSave = UcnLen;
293
2.31k
  for (; ThisTokBuf != ThisTokEnd && 
UcnLenSave2.04k
;
++ThisTokBuf, UcnLenSave--1.92k
) {
294
1.92k
    int CharVal = llvm::hexDigitValue(ThisTokBuf[0]);
295
1.92k
    if (CharVal == -1) 
break4
;
296
1.92k
    UcnVal <<= 4;
297
1.92k
    UcnVal |= CharVal;
298
1.92k
  }
299
  // If we didn't consume the proper number of digits, there is a problem.
300
396
  if (UcnLenSave) {
301
6
    if (Diags)
302
6
      Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf,
303
6
           diag::err_ucn_escape_incomplete);
304
6
    return false;
305
6
  }
306
307
  // Check UCN constraints (C99 6.4.3p2) [C++11 lex.charset p2]
308
390
  if ((0xD800 <= UcnVal && 
UcnVal <= 0xDFFF119
) || // surrogate codepoints
309
374
      UcnVal > 0x10FFFF) {                      // maximum legal UTF32 value
310
19
    if (Diags)
311
19
      Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf,
312
19
           diag::err_ucn_escape_invalid);
313
19
    return false;
314
19
  }
315
316
  // C++11 allows UCNs that refer to control characters and basic source
317
  // characters inside character and string literals
318
371
  if (UcnVal < 0xa0 &&
319
115
      (UcnVal != 0x24 && 
UcnVal != 0x40111
&&
UcnVal != 0x60107
)) { // $, @, `
320
103
    bool IsError = (!Features.CPlusPlus11 || 
!in_char_string_literal70
);
321
103
    if (Diags) {
322
103
      char BasicSCSChar = UcnVal;
323
103
      if (UcnVal >= 0x20 && 
UcnVal < 0x7f79
)
324
48
        Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf,
325
18
             IsError ? diag::err_ucn_escape_basic_scs :
326
30
                       diag::warn_cxx98_compat_literal_ucn_escape_basic_scs)
327
48
            << StringRef(&BasicSCSChar, 1);
328
55
      else
329
55
        Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf,
330
15
             IsError ? diag::err_ucn_control_character :
331
40
                       diag::warn_cxx98_compat_literal_ucn_control_character);
332
103
    }
333
103
    if (IsError)
334
33
      return false;
335
338
  }
336
337
338
  if (!Features.CPlusPlus && 
!Features.C99108
&&
Diags2
)
338
2
    Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf,
339
2
         diag::warn_ucn_not_valid_in_c89_literal);
340
341
338
  return true;
342
338
}
343
344
/// MeasureUCNEscape - Determine the number of bytes within the resulting string
345
/// which this UCN will occupy.
346
static int MeasureUCNEscape(const char *ThisTokBegin, const char *&ThisTokBuf,
347
                            const char *ThisTokEnd, unsigned CharByteWidth,
348
6
                            const LangOptions &Features, bool &HadError) {
349
  // UTF-32: 4 bytes per escape.
350
6
  if (CharByteWidth == 4)
351
0
    return 4;
352
353
6
  uint32_t UcnVal = 0;
354
6
  unsigned short UcnLen = 0;
355
6
  FullSourceLoc Loc;
356
357
6
  if (!ProcessUCNEscape(ThisTokBegin, ThisTokBuf, ThisTokEnd, UcnVal,
358
0
                        UcnLen, Loc, nullptr, Features, true)) {
359
0
    HadError = true;
360
0
    return 0;
361
0
  }
362
363
  // UTF-16: 2 bytes for BMP, 4 bytes otherwise.
364
6
  if (CharByteWidth == 2)
365
0
    return UcnVal <= 0xFFFF ? 2 : 4;
366
367
  // UTF-8.
368
6
  if (UcnVal < 0x80)
369
0
    return 1;
370
6
  if (UcnVal < 0x800)
371
0
    return 2;
372
6
  if (UcnVal < 0x10000)
373
3
    return 3;
374
3
  return 4;
375
3
}
376
377
/// EncodeUCNEscape - Read the Universal Character Name, check constraints and
378
/// convert the UTF32 to UTF8 or UTF16. This is a subroutine of
379
/// StringLiteralParser. When we decide to implement UCN's for identifiers,
380
/// we will likely rework our support for UCN's.
381
static void EncodeUCNEscape(const char *ThisTokBegin, const char *&ThisTokBuf,
382
                            const char *ThisTokEnd,
383
                            char *&ResultBuf, bool &HadError,
384
                            FullSourceLoc Loc, unsigned CharByteWidth,
385
                            DiagnosticsEngine *Diags,
386
287
                            const LangOptions &Features) {
387
287
  typedef uint32_t UTF32;
388
287
  UTF32 UcnVal = 0;
389
287
  unsigned short UcnLen = 0;
390
287
  if (!ProcessUCNEscape(ThisTokBegin, ThisTokBuf, ThisTokEnd, UcnVal, UcnLen,
391
33
                        Loc, Diags, Features, true)) {
392
33
    HadError = true;
393
33
    return;
394
33
  }
395
396
254
  assert((CharByteWidth == 1 || CharByteWidth == 2 || CharByteWidth == 4) &&
397
254
         "only character widths of 1, 2, or 4 bytes supported");
398
399
254
  (void)UcnLen;
400
254
  assert((UcnLen== 4 || UcnLen== 8) && "only ucn length of 4 or 8 supported");
401
402
254
  if (CharByteWidth == 4) {
403
    // FIXME: Make the type of the result buffer correct instead of
404
    // using reinterpret_cast.
405
69
    llvm::UTF32 *ResultPtr = reinterpret_cast<llvm::UTF32*>(ResultBuf);
406
69
    *ResultPtr = UcnVal;
407
69
    ResultBuf += 4;
408
69
    return;
409
69
  }
410
411
185
  if (CharByteWidth == 2) {
412
    // FIXME: Make the type of the result buffer correct instead of
413
    // using reinterpret_cast.
414
54
    llvm::UTF16 *ResultPtr = reinterpret_cast<llvm::UTF16*>(ResultBuf);
415
416
54
    if (UcnVal <= (UTF32)0xFFFF) {
417
40
      *ResultPtr = UcnVal;
418
40
      ResultBuf += 2;
419
40
      return;
420
40
    }
421
422
    // Convert to UTF16.
423
14
    UcnVal -= 0x10000;
424
14
    *ResultPtr     = 0xD800 + (UcnVal >> 10);
425
14
    *(ResultPtr+1) = 0xDC00 + (UcnVal & 0x3FF);
426
14
    ResultBuf += 4;
427
14
    return;
428
14
  }
429
430
131
  assert(CharByteWidth == 1 && "UTF-8 encoding is only for 1 byte characters");
431
432
  // Now that we've parsed/checked the UCN, we convert from UTF32->UTF8.
433
  // The conversion below was inspired by:
434
  //   http://www.unicode.org/Public/PROGRAMS/CVTUTF/ConvertUTF.c
435
  // First, we determine how many bytes the result will require.
436
131
  typedef uint8_t UTF8;
437
438
131
  unsigned short bytesToWrite = 0;
439
131
  if (UcnVal < (UTF32)0x80)
440
28
    bytesToWrite = 1;
441
103
  else if (UcnVal < (UTF32)0x800)
442
15
    bytesToWrite = 2;
443
88
  else if (UcnVal < (UTF32)0x10000)
444
64
    bytesToWrite = 3;
445
24
  else
446
24
    bytesToWrite = 4;
447
448
131
  const unsigned byteMask = 0xBF;
449
131
  const unsigned byteMark = 0x80;
450
451
  // Once the bits are split out into bytes of UTF8, this is a mask OR-ed
452
  // into the first byte, depending on how many bytes follow.
453
131
  static const UTF8 firstByteMark[5] = {
454
131
    0x00, 0x00, 0xC0, 0xE0, 0xF0
455
131
  };
456
  // Finally, we write the bytes into ResultBuf.
457
131
  ResultBuf += bytesToWrite;
458
131
  switch (bytesToWrite) { // note: everything falls through.
459
24
  case 4:
460
24
    *--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6;
461
24
    LLVM_FALLTHROUGH;
462
88
  case 3:
463
88
    *--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6;
464
88
    LLVM_FALLTHROUGH;
465
103
  case 2:
466
103
    *--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6;
467
103
    LLVM_FALLTHROUGH;
468
131
  case 1:
469
131
    *--ResultBuf = (UTF8) (UcnVal | firstByteMark[bytesToWrite]);
470
131
  }
471
  // Update the buffer.
472
131
  ResultBuf += bytesToWrite;
473
131
}
474
475
///       integer-constant: [C99 6.4.4.1]
476
///         decimal-constant integer-suffix
477
///         octal-constant integer-suffix
478
///         hexadecimal-constant integer-suffix
479
///         binary-literal integer-suffix [GNU, C++1y]
480
///       user-defined-integer-literal: [C++11 lex.ext]
481
///         decimal-literal ud-suffix
482
///         octal-literal ud-suffix
483
///         hexadecimal-literal ud-suffix
484
///         binary-literal ud-suffix [GNU, C++1y]
485
///       decimal-constant:
486
///         nonzero-digit
487
///         decimal-constant digit
488
///       octal-constant:
489
///         0
490
///         octal-constant octal-digit
491
///       hexadecimal-constant:
492
///         hexadecimal-prefix hexadecimal-digit
493
///         hexadecimal-constant hexadecimal-digit
494
///       hexadecimal-prefix: one of
495
///         0x 0X
496
///       binary-literal:
497
///         0b binary-digit
498
///         0B binary-digit
499
///         binary-literal binary-digit
500
///       integer-suffix:
501
///         unsigned-suffix [long-suffix]
502
///         unsigned-suffix [long-long-suffix]
503
///         long-suffix [unsigned-suffix]
504
///         long-long-suffix [unsigned-sufix]
505
///       nonzero-digit:
506
///         1 2 3 4 5 6 7 8 9
507
///       octal-digit:
508
///         0 1 2 3 4 5 6 7
509
///       hexadecimal-digit:
510
///         0 1 2 3 4 5 6 7 8 9
511
///         a b c d e f
512
///         A B C D E F
513
///       binary-digit:
514
///         0
515
///         1
516
///       unsigned-suffix: one of
517
///         u U
518
///       long-suffix: one of
519
///         l L
520
///       long-long-suffix: one of
521
///         ll LL
522
///
523
///       floating-constant: [C99 6.4.4.2]
524
///         TODO: add rules...
525
///
526
NumericLiteralParser::NumericLiteralParser(StringRef TokSpelling,
527
                                           SourceLocation TokLoc,
528
                                           const SourceManager &SM,
529
                                           const LangOptions &LangOpts,
530
                                           const TargetInfo &Target,
531
                                           DiagnosticsEngine &Diags)
532
    : SM(SM), LangOpts(LangOpts), Diags(Diags),
533
6.86M
      ThisTokBegin(TokSpelling.begin()), ThisTokEnd(TokSpelling.end()) {
534
535
  // This routine assumes that the range begin/end matches the regex for integer
536
  // and FP constants (specifically, the 'pp-number' regex), and assumes that
537
  // the byte at "*end" is both valid and not part of the regex.  Because of
538
  // this, it doesn't have to check for 'overscan' in various places.
539
6.86M
  assert(!isPreprocessingNumberBody(*ThisTokEnd) && "didn't maximally munch?");
540
541
6.86M
  s = DigitsBegin = ThisTokBegin;
542
6.86M
  saw_exponent = false;
543
6.86M
  saw_period = false;
544
6.86M
  saw_ud_suffix = false;
545
6.86M
  saw_fixed_point_suffix = false;
546
6.86M
  isLong = false;
547
6.86M
  isUnsigned = false;
548
6.86M
  isLongLong = false;
549
6.86M
  isHalf = false;
550
6.86M
  isFloat = false;
551
6.86M
  isImaginary = false;
552
6.86M
  isFloat16 = false;
553
6.86M
  isFloat128 = false;
554
6.86M
  MicrosoftInteger = 0;
555
6.86M
  isFract = false;
556
6.86M
  isAccum = false;
557
6.86M
  hadError = false;
558
559
6.86M
  if (*s == '0') { // parse radix
560
1.40M
    ParseNumberStartingWithZero(TokLoc);
561
1.40M
    if (hadError)
562
29
      return;
563
5.45M
  } else { // the first digit is non-zero
564
5.45M
    radix = 10;
565
5.45M
    s = SkipDigits(s);
566
5.45M
    if (s == ThisTokEnd) {
567
      // Done.
568
424k
    } else {
569
424k
      ParseDecimalOrOctalCommon(TokLoc);
570
424k
      if (hadError)
571
7
        return;
572
6.86M
    }
573
5.45M
  }
574
575
6.86M
  SuffixBegin = s;
576
6.86M
  checkSeparator(TokLoc, s, CSK_AfterDigits);
577
578
  // Initial scan to lookahead for fixed point suffix.
579
6.86M
  if (LangOpts.FixedPoint) {
580
1.81k
    for (const char *c = s; c != ThisTokEnd; 
++c886
) {
581
1.74k
      if (*c == 'r' || 
*c == 'k'1.52k
||
*c == 'R'886
||
*c == 'K'886
) {
582
858
        saw_fixed_point_suffix = true;
583
858
        break;
584
858
      }
585
1.74k
    }
586
928
  }
587
588
  // Parse the suffix.  At this point we can classify whether we have an FP or
589
  // integer constant.
590
6.86M
  bool isFixedPointConstant = isFixedPointLiteral();
591
6.86M
  bool isFPConstant = isFloatingLiteral();
592
593
  // Loop over all of the characters of the suffix.  If we see something bad,
594
  // we break out of the loop.
595
7.44M
  for (; s != ThisTokEnd; 
++s579k
) {
596
579k
    switch (*s) {
597
0
    case 'R':
598
224
    case 'r':
599
224
      if (!LangOpts.FixedPoint)
600
6
        break;
601
218
      if (isFract || 
isAccum217
)
break1
;
602
217
      if (!(saw_period || 
saw_exponent20
))
break14
;
603
203
      isFract = true;
604
203
      continue;
605
0
    case 'K':
606
632
    case 'k':
607
632
      if (!LangOpts.FixedPoint)
608
6
        break;
609
626
      if (isFract || 
isAccum625
)
break2
;
610
624
      if (!(saw_period || 
saw_exponent43
))
break16
;
611
608
      isAccum = true;
612
608
      continue;
613
514
    case 'h':      // FP Suffix for "half".
614
516
    case 'H':
615
      // OpenCL Extension v1.2 s9.5 - h or H suffix for half type.
616
516
      if (!(LangOpts.Half || 
LangOpts.FixedPoint495
))
617
3
        break;
618
513
      if (isIntegerLiteral()) 
break13
; // Error for integer constant.
619
500
      if (isHalf || 
isFloat499
||
isLong498
)
break3
; // HH, FH, LH invalid.
620
497
      isHalf = true;
621
497
      continue;  // Success.
622
4.08k
    case 'f':      // FP Suffix for "float"
623
6.97k
    case 'F':
624
6.97k
      if (!isFPConstant) 
break4
; // Error for integer constant.
625
6.97k
      if (isHalf || 
isFloat6.97k
||
isLong6.97k
||
isFloat1286.97k
)
626
1
        break; // HF, FF, LF, QF invalid.
627
628
      // CUDA host and device may have different _Float16 support, therefore
629
      // allows f16 literals to avoid false alarm.
630
      // ToDo: more precise check for CUDA.
631
6.97k
      if ((Target.hasFloat16Type() || 
LangOpts.CUDA6.58k
) &&
s + 2 < ThisTokEnd452
&&
632
142
          s[1] == '1' && s[2] == '6') {
633
142
        s += 2; // success, eat up 2 characters.
634
142
        isFloat16 = true;
635
142
        continue;
636
142
      }
637
638
6.82k
      isFloat = true;
639
6.82k
      continue;  // Success.
640
171
    case 'q':    // FP Suffix for "__float128"
641
173
    case 'Q':
642
173
      if (!isFPConstant) 
break1
; // Error for integer constant.
643
172
      if (isHalf || isFloat || isLong || isFloat128)
644
0
        break; // HQ, FQ, LQ, QQ invalid.
645
172
      isFloat128 = true;
646
172
      continue;  // Success.
647
20.2k
    case 'u':
648
132k
    case 'U':
649
132k
      if (isFPConstant) 
break0
; // Error for floating constant.
650
132k
      if (isUnsigned) 
break0
; // Cannot be repeated.
651
132k
      isUnsigned = true;
652
132k
      continue;  // Success.
653
6.17k
    case 'l':
654
437k
    case 'L':
655
437k
      if (isLong || isLongLong) 
break0
; // Cannot be repeated.
656
437k
      if (isHalf || isFloat || isFloat128) 
break0
; // LH, LF, LQ invalid.
657
658
      // Check for long long.  The L's need to be adjacent and the same case.
659
437k
      if (s[1] == s[0]) {
660
47.9k
        assert(s + 1 < ThisTokEnd && "didn't maximally munch?");
661
47.9k
        if (isFPConstant) 
break0
; // long long invalid for floats.
662
47.9k
        isLongLong = true;
663
47.9k
        ++s;  // Eat both of them.
664
389k
      } else {
665
389k
        isLong = true;
666
389k
      }
667
437k
      continue; // Success.
668
334
    case 'i':
669
339
    case 'I':
670
339
      if (LangOpts.MicrosoftExt) {
671
68
        if (isLong || 
isLongLong65
||
MicrosoftInteger62
)
672
6
          break;
673
674
62
        if (!isFPConstant) {
675
          // Allow i8, i16, i32, and i64.
676
54
          switch (s[1]) {
677
10
          case '8':
678
10
            s += 2; // i8 suffix
679
10
            MicrosoftInteger = 8;
680
10
            break;
681
9
          case '1':
682
9
            if (s[2] == '6') {
683
9
              s += 3; // i16 suffix
684
9
              MicrosoftInteger = 16;
685
9
            }
686
9
            break;
687
9
          case '3':
688
9
            if (s[2] == '2') {
689
9
              s += 3; // i32 suffix
690
9
              MicrosoftInteger = 32;
691
9
            }
692
9
            break;
693
23
          case '6':
694
23
            if (s[2] == '4') {
695
23
              s += 3; // i64 suffix
696
23
              MicrosoftInteger = 64;
697
23
            }
698
23
            break;
699
3
          default:
700
3
            break;
701
62
          }
702
62
        }
703
62
        if (MicrosoftInteger) {
704
51
          assert(s <= ThisTokEnd && "didn't maximally munch?");
705
51
          break;
706
51
        }
707
282
      }
708
282
      LLVM_FALLTHROUGH;
709
408
    case 'j':
710
408
    case 'J':
711
408
      if (isImaginary) 
break0
; // Cannot be repeated.
712
408
      isImaginary = true;
713
408
      continue;  // Success.
714
360
    }
715
    // If we reached here, there was an error or a ud-suffix.
716
360
    break;
717
360
  }
718
719
  // "i", "if", and "il" are user-defined suffixes in C++1y.
720
6.86M
  if (s != ThisTokEnd || 
isImaginary6.86M
) {
721
    // FIXME: Don't bother expanding UCNs if !tok.hasUCN().
722
717
    expandUCNs(UDSuffixBuf, StringRef(SuffixBegin, ThisTokEnd - SuffixBegin));
723
717
    if (isValidUDSuffix(LangOpts, UDSuffixBuf)) {
724
247
      if (!isImaginary) {
725
        // Any suffix pieces we might have parsed are actually part of the
726
        // ud-suffix.
727
201
        isLong = false;
728
201
        isUnsigned = false;
729
201
        isLongLong = false;
730
201
        isFloat = false;
731
201
        isFloat16 = false;
732
201
        isHalf = false;
733
201
        isImaginary = false;
734
201
        MicrosoftInteger = 0;
735
201
        saw_fixed_point_suffix = false;
736
201
        isFract = false;
737
201
        isAccum = false;
738
201
      }
739
740
247
      saw_ud_suffix = true;
741
247
      return;
742
247
    }
743
744
470
    if (s != ThisTokEnd) {
745
      // Report an error if there are any.
746
108
      Diags.Report(Lexer::AdvanceToTokenCharacter(
747
108
                       TokLoc, SuffixBegin - ThisTokBegin, SM, LangOpts),
748
108
                   diag::err_invalid_suffix_constant)
749
108
          << StringRef(SuffixBegin, ThisTokEnd - SuffixBegin)
750
101
          << (isFixedPointConstant ? 
27
: isFPConstant);
751
108
      hadError = true;
752
108
    }
753
470
  }
754
755
6.86M
  if (!hadError && 
saw_fixed_point_suffix6.86M
) {
756
808
    assert(isFract || isAccum);
757
808
  }
758
6.86M
}
759
760
/// ParseDecimalOrOctalCommon - This method is called for decimal or octal
761
/// numbers. It issues an error for illegal digits, and handles floating point
762
/// parsing. If it detects a floating point number, the radix is set to 10.
763
449k
void NumericLiteralParser::ParseDecimalOrOctalCommon(SourceLocation TokLoc){
764
449k
  assert((radix == 8 || radix == 10) && "Unexpected radix");
765
766
  // If we have a hex digit other than 'e' (which denotes a FP exponent) then
767
  // the code is using an incorrect base.
768
449k
  if (isHexDigit(*s) && 
*s != 'e'342
&&
*s != 'E'43
&&
769
10
      !isValidUDSuffix(LangOpts, StringRef(s, ThisTokEnd - s))) {
770
8
    Diags.Report(
771
8
        Lexer::AdvanceToTokenCharacter(TokLoc, s - ThisTokBegin, SM, LangOpts),
772
8
        diag::err_invalid_digit)
773
7
        << StringRef(s, 1) << (radix == 8 ? 1 : 
01
);
774
8
    hadError = true;
775
8
    return;
776
8
  }
777
778
449k
  if (*s == '.') {
779
36.7k
    checkSeparator(TokLoc, s, CSK_AfterDigits);
780
36.7k
    s++;
781
36.7k
    radix = 10;
782
36.7k
    saw_period = true;
783
36.7k
    checkSeparator(TokLoc, s, CSK_BeforeDigits);
784
36.7k
    s = SkipDigits(s); // Skip suffix.
785
36.7k
  }
786
449k
  if (*s == 'e' || 
*s == 'E'442k
) { // exponent
787
6.74k
    checkSeparator(TokLoc, s, CSK_AfterDigits);
788
6.74k
    const char *Exponent = s;
789
6.74k
    s++;
790
6.74k
    radix = 10;
791
6.74k
    saw_exponent = true;
792
6.74k
    if (s != ThisTokEnd && 
(6.74k
*s == '+'6.74k
||
*s == '-'5.38k
))
s++6.39k
; // sign
793
6.74k
    const char *first_non_digit = SkipDigits(s);
794
6.74k
    if (containsDigits(s, first_non_digit)) {
795
6.74k
      checkSeparator(TokLoc, s, CSK_BeforeDigits);
796
6.74k
      s = first_non_digit;
797
4
    } else {
798
4
      if (!hadError) {
799
3
        Diags.Report(Lexer::AdvanceToTokenCharacter(
800
3
                         TokLoc, Exponent - ThisTokBegin, SM, LangOpts),
801
3
                     diag::err_exponent_has_no_digits);
802
3
        hadError = true;
803
3
      }
804
4
      return;
805
4
    }
806
6.74k
  }
807
449k
}
808
809
/// Determine whether a suffix is a valid ud-suffix. We avoid treating reserved
810
/// suffixes as ud-suffixes, because the diagnostic experience is better if we
811
/// treat it as an invalid suffix.
812
bool NumericLiteralParser::isValidUDSuffix(const LangOptions &LangOpts,
813
1.23k
                                           StringRef Suffix) {
814
1.23k
  if (!LangOpts.CPlusPlus11 || 
Suffix.empty()944
)
815
290
    return false;
816
817
  // By C++11 [lex.ext]p10, ud-suffixes starting with an '_' are always valid.
818
944
  if (Suffix[0] == '_')
819
187
    return true;
820
821
  // In C++11, there are no library suffixes.
822
757
  if (!LangOpts.CPlusPlus14)
823
35
    return false;
824
825
  // In C++14, "s", "h", "min", "ms", "us", and "ns" are used in the library.
826
  // Per tweaked N3660, "il", "i", and "if" are also used in the library.
827
  // In C++2a "d" and "y" are used in the library.
828
722
  return llvm::StringSwitch<bool>(Suffix)
829
722
      .Cases("h", "min", "s", true)
830
722
      .Cases("ms", "us", "ns", true)
831
722
      .Cases("il", "i", "if", true)
832
722
      .Cases("d", "y", LangOpts.CPlusPlus20)
833
722
      .Default(false);
834
722
}
835
836
void NumericLiteralParser::checkSeparator(SourceLocation TokLoc,
837
                                          const char *Pos,
838
6.95M
                                          CheckSeparatorKind IsAfterDigits) {
839
6.95M
  if (IsAfterDigits == CSK_AfterDigits) {
840
6.90M
    if (Pos == ThisTokBegin)
841
542
      return;
842
6.90M
    --Pos;
843
43.7k
  } else if (Pos == ThisTokEnd)
844
576
    return;
845
846
6.94M
  if (isDigitSeparator(*Pos)) {
847
20
    Diags.Report(Lexer::AdvanceToTokenCharacter(TokLoc, Pos - ThisTokBegin, SM,
848
20
                                                LangOpts),
849
20
                 diag::err_digit_separator_not_between_digits)
850
20
        << IsAfterDigits;
851
20
    hadError = true;
852
20
  }
853
6.94M
}
854
855
/// ParseNumberStartingWithZero - This method is called when the first character
856
/// of the number is found to be a zero.  This means it is either an octal
857
/// number (like '04') or a hex number ('0x123a') a binary number ('0b1010') or
858
/// a floating point number (01239.123e4).  Eat the prefix, determining the
859
/// radix etc.
860
1.40M
void NumericLiteralParser::ParseNumberStartingWithZero(SourceLocation TokLoc) {
861
1.40M
  assert(s[0] == '0' && "Invalid method call");
862
1.40M
  s++;
863
864
1.40M
  int c1 = s[0];
865
866
  // Handle a hex number like 0x1234.
867
1.40M
  if ((c1 == 'x' || 
c1 == 'X'280k
) &&
(1.12M
isHexDigit(s[1])1.12M
||
s[1] == '.'18
)) {
868
1.12M
    s++;
869
1.12M
    assert(s < ThisTokEnd && "didn't maximally munch?");
870
1.12M
    radix = 16;
871
1.12M
    DigitsBegin = s;
872
1.12M
    s = SkipHexDigits(s);
873
1.12M
    bool HasSignificandDigits = containsDigits(DigitsBegin, s);
874
1.12M
    if (s == ThisTokEnd) {
875
      // Done.
876
52.9k
    } else if (*s == '.') {
877
128
      s++;
878
128
      saw_period = true;
879
128
      const char *floatDigitsBegin = s;
880
128
      s = SkipHexDigits(s);
881
128
      if (containsDigits(floatDigitsBegin, s))
882
115
        HasSignificandDigits = true;
883
128
      if (HasSignificandDigits)
884
123
        checkSeparator(TokLoc, floatDigitsBegin, CSK_BeforeDigits);
885
128
    }
886
887
1.12M
    if (!HasSignificandDigits) {
888
5
      Diags.Report(Lexer::AdvanceToTokenCharacter(TokLoc, s - ThisTokBegin, SM,
889
5
                                                  LangOpts),
890
5
                   diag::err_hex_constant_requires)
891
5
          << LangOpts.CPlusPlus << 1;
892
5
      hadError = true;
893
5
      return;
894
5
    }
895
896
    // A binary exponent can appear with or with a '.'. If dotted, the
897
    // binary exponent is required.
898
1.12M
    if (*s == 'p' || 
*s == 'P'1.12M
) {
899
159
      checkSeparator(TokLoc, s, CSK_AfterDigits);
900
159
      const char *Exponent = s;
901
159
      s++;
902
159
      saw_exponent = true;
903
159
      if (s != ThisTokEnd && 
(158
*s == '+'158
||
*s == '-'138
))
s++44
; // sign
904
159
      const char *first_non_digit = SkipDigits(s);
905
159
      if (!containsDigits(s, first_non_digit)) {
906
2
        if (!hadError) {
907
1
          Diags.Report(Lexer::AdvanceToTokenCharacter(
908
1
                           TokLoc, Exponent - ThisTokBegin, SM, LangOpts),
909
1
                       diag::err_exponent_has_no_digits);
910
1
          hadError = true;
911
1
        }
912
2
        return;
913
2
      }
914
157
      checkSeparator(TokLoc, s, CSK_BeforeDigits);
915
157
      s = first_non_digit;
916
917
157
      if (!LangOpts.HexFloats)
918
45
        Diags.Report(TokLoc, LangOpts.CPlusPlus
919
42
                                 ? diag::ext_hex_literal_invalid
920
3
                                 : diag::ext_hex_constant_invalid);
921
112
      else if (LangOpts.CPlusPlus17)
922
31
        Diags.Report(TokLoc, diag::warn_cxx17_hex_literal);
923
1.12M
    } else if (saw_period) {
924
2
      Diags.Report(Lexer::AdvanceToTokenCharacter(TokLoc, s - ThisTokBegin, SM,
925
2
                                                  LangOpts),
926
2
                   diag::err_hex_constant_requires)
927
2
          << LangOpts.CPlusPlus << 0;
928
2
      hadError = true;
929
2
    }
930
1.12M
    return;
931
279k
  }
932
933
  // Handle simple binary numbers 0b01010
934
279k
  if ((c1 == 'b' || 
c1 == 'B'279k
) &&
(59
s[1] == '0'59
||
s[1] == '1'47
)) {
935
    // 0b101010 is a C++1y / GCC extension.
936
55
    Diags.Report(TokLoc, LangOpts.CPlusPlus14
937
33
                             ? diag::warn_cxx11_compat_binary_literal
938
22
                         : LangOpts.CPlusPlus ? 
diag::ext_binary_literal_cxx146
939
16
                                              : diag::ext_binary_literal);
940
55
    ++s;
941
55
    assert(s < ThisTokEnd && "didn't maximally munch?");
942
55
    radix = 2;
943
55
    DigitsBegin = s;
944
55
    s = SkipBinaryDigits(s);
945
55
    if (s == ThisTokEnd) {
946
      // Done.
947
7
    } else if (isHexDigit(*s) &&
948
3
               !isValidUDSuffix(LangOpts, StringRef(s, ThisTokEnd - s))) {
949
2
      Diags.Report(Lexer::AdvanceToTokenCharacter(TokLoc, s - ThisTokBegin, SM,
950
2
                                                  LangOpts),
951
2
                   diag::err_invalid_digit)
952
2
          << StringRef(s, 1) << 2;
953
2
      hadError = true;
954
2
    }
955
    // Other suffixes will be diagnosed by the caller.
956
55
    return;
957
55
  }
958
959
  // For now, the radix is set to 8. If we discover that we have a
960
  // floating point constant, the radix will change to 10. Octal floating
961
  // point constants are not permitted (only decimal and hexadecimal).
962
279k
  radix = 8;
963
279k
  DigitsBegin = s;
964
279k
  s = SkipOctalDigits(s);
965
279k
  if (s == ThisTokEnd)
966
255k
    return; // Done, simple octal number like 01234
967
968
  // If we have some other non-octal digit that *is* a decimal digit, see if
969
  // this is part of a floating point number like 094.123 or 09e1.
970
24.4k
  if (isDigit(*s)) {
971
3
    const char *EndDecimal = SkipDigits(s);
972
3
    if (EndDecimal[0] == '.' || 
EndDecimal[0] == 'e'2
||
EndDecimal[0] == 'E'2
) {
973
1
      s = EndDecimal;
974
1
      radix = 10;
975
1
    }
976
3
  }
977
978
24.4k
  ParseDecimalOrOctalCommon(TokLoc);
979
24.4k
}
980
981
6.82M
static bool alwaysFitsInto64Bits(unsigned Radix, unsigned NumDigits) {
982
6.82M
  switch (Radix) {
983
52
  case 2:
984
52
    return NumDigits <= 64;
985
265k
  case 8:
986
265k
    return NumDigits <= 64 / 3; // Digits are groups of 3 bits.
987
5.43M
  case 10:
988
5.43M
    return NumDigits <= 19; // floor(log10(2^64))
989
1.12M
  case 16:
990
1.12M
    return NumDigits <= 64 / 4; // Digits are groups of 4 bits.
991
0
  default:
992
0
    llvm_unreachable("impossible Radix");
993
6.82M
  }
994
6.82M
}
995
996
/// GetIntegerValue - Convert this numeric literal value to an APInt that
997
/// matches Val's input width.  If there is an overflow, set Val to the low bits
998
/// of the result and return true.  Otherwise, return false.
999
6.82M
bool NumericLiteralParser::GetIntegerValue(llvm::APInt &Val) {
1000
  // Fast path: Compute a conservative bound on the maximum number of
1001
  // bits per digit in this radix. If we can't possibly overflow a
1002
  // uint64 based on that bound then do the simple conversion to
1003
  // integer. This avoids the expensive overflow checking below, and
1004
  // handles the common cases that matter (small decimal integers and
1005
  // hex/octal values which don't overflow).
1006
6.82M
  const unsigned NumDigits = SuffixBegin - DigitsBegin;
1007
6.82M
  if (alwaysFitsInto64Bits(radix, NumDigits)) {
1008
6.82M
    uint64_t N = 0;
1009
28.3M
    for (const char *Ptr = DigitsBegin; Ptr != SuffixBegin; 
++Ptr21.4M
)
1010
21.4M
      if (!isDigitSeparator(*Ptr))
1011
21.4M
        N = N * radix + llvm::hexDigitValue(*Ptr);
1012
1013
    // This will truncate the value to Val's input width. Simply check
1014
    // for overflow by comparing.
1015
6.82M
    Val = N;
1016
6.82M
    return Val.getZExtValue() != N;
1017
6.82M
  }
1018
1019
175
  Val = 0;
1020
175
  const char *Ptr = DigitsBegin;
1021
1022
175
  llvm::APInt RadixVal(Val.getBitWidth(), radix);
1023
175
  llvm::APInt CharVal(Val.getBitWidth(), 0);
1024
175
  llvm::APInt OldVal = Val;
1025
1026
175
  bool OverflowOccurred = false;
1027
3.70k
  while (Ptr < SuffixBegin) {
1028
3.53k
    if (isDigitSeparator(*Ptr)) {
1029
30
      ++Ptr;
1030
30
      continue;
1031
30
    }
1032
1033
3.50k
    unsigned C = llvm::hexDigitValue(*Ptr++);
1034
1035
    // If this letter is out of bound for this radix, reject it.
1036
3.50k
    assert(C < radix && "NumericLiteralParser ctor should have rejected this");
1037
1038
3.50k
    CharVal = C;
1039
1040
    // Add the digit to the value in the appropriate radix.  If adding in digits
1041
    // made the value smaller, then this overflowed.
1042
3.50k
    OldVal = Val;
1043
1044
    // Multiply by radix, did overflow occur on the multiply?
1045
3.50k
    Val *= RadixVal;
1046
3.50k
    OverflowOccurred |= Val.udiv(RadixVal) != OldVal;
1047
1048
    // Add value, did overflow occur on the value?
1049
    //   (a + b) ult b  <=> overflow
1050
3.50k
    Val += CharVal;
1051
3.50k
    OverflowOccurred |= Val.ult(CharVal);
1052
3.50k
  }
1053
175
  return OverflowOccurred;
1054
175
}
1055
1056
llvm::APFloat::opStatus
1057
36.3k
NumericLiteralParser::GetFloatValue(llvm::APFloat &Result) {
1058
36.3k
  using llvm::APFloat;
1059
1060
36.3k
  unsigned n = std::min(SuffixBegin - ThisTokBegin, ThisTokEnd - ThisTokBegin);
1061
1062
36.3k
  llvm::SmallString<16> Buffer;
1063
36.3k
  StringRef Str(ThisTokBegin, n);
1064
36.3k
  if (Str.find('\'') != StringRef::npos) {
1065
4
    Buffer.reserve(n);
1066
4
    std::remove_copy_if(Str.begin(), Str.end(), std::back_inserter(Buffer),
1067
4
                        &isDigitSeparator);
1068
4
    Str = Buffer;
1069
4
  }
1070
1071
36.3k
  auto StatusOrErr =
1072
36.3k
      Result.convertFromString(Str, APFloat::rmNearestTiesToEven);
1073
36.3k
  assert(StatusOrErr && "Invalid floating point representation");
1074
36.3k
  return !errorToBool(StatusOrErr.takeError()) ? *StatusOrErr
1075
0
                                               : APFloat::opInvalidOp;
1076
36.3k
}
1077
1078
270
static inline bool IsExponentPart(char c) {
1079
270
  return c == 'p' || 
c == 'P'232
||
c == 'e'228
||
c == 'E'207
;
1080
270
}
1081
1082
808
bool NumericLiteralParser::GetFixedPointValue(llvm::APInt &StoreVal, unsigned Scale) {
1083
808
  assert(radix == 16 || radix == 10);
1084
1085
  // Find how many digits are needed to store the whole literal.
1086
808
  unsigned NumDigits = SuffixBegin - DigitsBegin;
1087
808
  if (saw_period) 
--NumDigits775
;
1088
1089
  // Initial scan of the exponent if it exists
1090
808
  bool ExpOverflowOccurred = false;
1091
808
  bool NegativeExponent = false;
1092
808
  const char *ExponentBegin;
1093
808
  uint64_t Exponent = 0;
1094
808
  int64_t BaseShift = 0;
1095
808
  if (saw_exponent) {
1096
67
    const char *Ptr = DigitsBegin;
1097
1098
270
    while (!IsExponentPart(*Ptr)) 
++Ptr203
;
1099
67
    ExponentBegin = Ptr;
1100
67
    ++Ptr;
1101
67
    NegativeExponent = *Ptr == '-';
1102
67
    if (NegativeExponent) 
++Ptr26
;
1103
1104
67
    unsigned NumExpDigits = SuffixBegin - Ptr;
1105
67
    if (alwaysFitsInto64Bits(radix, NumExpDigits)) {
1106
66
      llvm::StringRef ExpStr(Ptr, NumExpDigits);
1107
66
      llvm::APInt ExpInt(/*numBits=*/64, ExpStr, /*radix=*/10);
1108
66
      Exponent = ExpInt.getZExtValue();
1109
1
    } else {
1110
1
      ExpOverflowOccurred = true;
1111
1
    }
1112
1113
67
    if (NegativeExponent) 
BaseShift -= Exponent26
;
1114
41
    else BaseShift += Exponent;
1115
67
  }
1116
1117
  // Number of bits needed for decimal literal is
1118
  //   ceil(NumDigits * log2(10))       Integral part
1119
  // + Scale                            Fractional part
1120
  // + ceil(Exponent * log2(10))        Exponent
1121
  // --------------------------------------------------
1122
  //   ceil((NumDigits + Exponent) * log2(10)) + Scale
1123
  //
1124
  // But for simplicity in handling integers, we can round up log2(10) to 4,
1125
  // making:
1126
  // 4 * (NumDigits + Exponent) + Scale
1127
  //
1128
  // Number of digits needed for hexadecimal literal is
1129
  //   4 * NumDigits                    Integral part
1130
  // + Scale                            Fractional part
1131
  // + Exponent                         Exponent
1132
  // --------------------------------------------------
1133
  //   (4 * NumDigits) + Scale + Exponent
1134
808
  uint64_t NumBitsNeeded;
1135
808
  if (radix == 10)
1136
766
    NumBitsNeeded = 4 * (NumDigits + Exponent) + Scale;
1137
42
  else
1138
42
    NumBitsNeeded = 4 * NumDigits + Exponent + Scale;
1139
1140
808
  if (NumBitsNeeded > std::numeric_limits<unsigned>::max())
1141
0
    ExpOverflowOccurred = true;
1142
808
  llvm::APInt Val(static_cast<unsigned>(NumBitsNeeded), 0, /*isSigned=*/false);
1143
1144
808
  bool FoundDecimal = false;
1145
1146
808
  int64_t FractBaseShift = 0;
1147
741
  const char *End = saw_exponent ? 
ExponentBegin67
: SuffixBegin;
1148
4.08k
  for (const char *Ptr = DigitsBegin; Ptr < End; 
++Ptr3.27k
) {
1149
3.27k
    if (*Ptr == '.') {
1150
775
      FoundDecimal = true;
1151
775
      continue;
1152
775
    }
1153
1154
    // Normal reading of an integer
1155
2.50k
    unsigned C = llvm::hexDigitValue(*Ptr);
1156
2.50k
    assert(C < radix && "NumericLiteralParser ctor should have rejected this");
1157
1158
2.50k
    Val *= radix;
1159
2.50k
    Val += C;
1160
1161
2.50k
    if (FoundDecimal)
1162
      // Keep track of how much we will need to adjust this value by from the
1163
      // number of digits past the radix point.
1164
1.63k
      --FractBaseShift;
1165
2.50k
  }
1166
1167
  // For a radix of 16, we will be multiplying by 2 instead of 16.
1168
808
  if (radix == 16) 
FractBaseShift *= 442
;
1169
808
  BaseShift += FractBaseShift;
1170
1171
808
  Val <<= Scale;
1172
1173
766
  uint64_t Base = (radix == 16) ? 
242
: 10;
1174
808
  if (BaseShift > 0) {
1175
146
    for (int64_t i = 0; i < BaseShift; 
++i136
) {
1176
136
      Val *= Base;
1177
136
    }
1178
798
  } else if (BaseShift < 0) {
1179
2.80k
    for (int64_t i = BaseShift; i < 0 && 
!Val.isNullValue()2.05k
;
++i2.02k
)
1180
2.02k
      Val = Val.udiv(Base);
1181
781
  }
1182
1183
808
  bool IntOverflowOccurred = false;
1184
808
  auto MaxVal = llvm::APInt::getMaxValue(StoreVal.getBitWidth());
1185
808
  if (Val.getBitWidth() > StoreVal.getBitWidth()) {
1186
407
    IntOverflowOccurred |= Val.ugt(MaxVal.zext(Val.getBitWidth()));
1187
407
    StoreVal = Val.trunc(StoreVal.getBitWidth());
1188
401
  } else if (Val.getBitWidth() < StoreVal.getBitWidth()) {
1189
374
    IntOverflowOccurred |= Val.zext(MaxVal.getBitWidth()).ugt(MaxVal);
1190
374
    StoreVal = Val.zext(StoreVal.getBitWidth());
1191
27
  } else {
1192
27
    StoreVal = Val;
1193
27
  }
1194
1195
808
  return IntOverflowOccurred || 
ExpOverflowOccurred778
;
1196
808
}
1197
1198
/// \verbatim
1199
///       user-defined-character-literal: [C++11 lex.ext]
1200
///         character-literal ud-suffix
1201
///       ud-suffix:
1202
///         identifier
1203
///       character-literal: [C++11 lex.ccon]
1204
///         ' c-char-sequence '
1205
///         u' c-char-sequence '
1206
///         U' c-char-sequence '
1207
///         L' c-char-sequence '
1208
///         u8' c-char-sequence ' [C++1z lex.ccon]
1209
///       c-char-sequence:
1210
///         c-char
1211
///         c-char-sequence c-char
1212
///       c-char:
1213
///         any member of the source character set except the single-quote ',
1214
///           backslash \, or new-line character
1215
///         escape-sequence
1216
///         universal-character-name
1217
///       escape-sequence:
1218
///         simple-escape-sequence
1219
///         octal-escape-sequence
1220
///         hexadecimal-escape-sequence
1221
///       simple-escape-sequence:
1222
///         one of \' \" \? \\ \a \b \f \n \r \t \v
1223
///       octal-escape-sequence:
1224
///         \ octal-digit
1225
///         \ octal-digit octal-digit
1226
///         \ octal-digit octal-digit octal-digit
1227
///       hexadecimal-escape-sequence:
1228
///         \x hexadecimal-digit
1229
///         hexadecimal-escape-sequence hexadecimal-digit
1230
///       universal-character-name: [C++11 lex.charset]
1231
///         \u hex-quad
1232
///         \U hex-quad hex-quad
1233
///       hex-quad:
1234
///         hex-digit hex-digit hex-digit hex-digit
1235
/// \endverbatim
1236
///
1237
CharLiteralParser::CharLiteralParser(const char *begin, const char *end,
1238
                                     SourceLocation Loc, Preprocessor &PP,
1239
709k
                                     tok::TokenKind kind) {
1240
  // At this point we know that the character matches the regex "(L|u|U)?'.*'".
1241
709k
  HadError = false;
1242
1243
709k
  Kind = kind;
1244
1245
709k
  const char *TokBegin = begin;
1246
1247
  // Skip over wide character determinant.
1248
709k
  if (Kind != tok::char_constant)
1249
1.43k
    ++begin;
1250
709k
  if (Kind == tok::utf8_char_constant)
1251
147
    ++begin;
1252
1253
  // Skip over the entry quote.
1254
709k
  assert(begin[0] == '\'' && "Invalid token lexed");
1255
709k
  ++begin;
1256
1257
  // Remove an optional ud-suffix.
1258
709k
  if (end[-1] != '\'') {
1259
59
    const char *UDSuffixEnd = end;
1260
216
    do {
1261
216
      --end;
1262
216
    } while (end[-1] != '\'');
1263
    // FIXME: Don't bother with this if !tok.hasUCN().
1264
59
    expandUCNs(UDSuffixBuf, StringRef(end, UDSuffixEnd - end));
1265
59
    UDSuffixOffset = end - TokBegin;
1266
59
  }
1267
1268
  // Trim the ending quote.
1269
709k
  assert(end != begin && "Invalid token lexed");
1270
709k
  --end;
1271
1272
  // FIXME: The "Value" is an uint64_t so we can handle char literals of
1273
  // up to 64-bits.
1274
  // FIXME: This extensively assumes that 'char' is 8-bits.
1275
709k
  assert(PP.getTargetInfo().getCharWidth() == 8 &&
1276
709k
         "Assumes char is 8 bits");
1277
709k
  assert(PP.getTargetInfo().getIntWidth() <= 64 &&
1278
709k
         (PP.getTargetInfo().getIntWidth() & 7) == 0 &&
1279
709k
         "Assumes sizeof(int) on target is <= 64 and a multiple of char");
1280
709k
  assert(PP.getTargetInfo().getWCharWidth() <= 64 &&
1281
709k
         "Assumes sizeof(wchar) on target is <= 64");
1282
1283
709k
  SmallVector<uint32_t, 4> codepoint_buffer;
1284
709k
  codepoint_buffer.resize(end - begin);
1285
709k
  uint32_t *buffer_begin = &codepoint_buffer.front();
1286
709k
  uint32_t *buffer_end = buffer_begin + codepoint_buffer.size();
1287
1288
  // Unicode escapes representing characters that cannot be correctly
1289
  // represented in a single code unit are disallowed in character literals
1290
  // by this implementation.
1291
709k
  uint32_t largest_character_for_kind;
1292
709k
  if (tok::wide_char_constant == Kind) {
1293
1.10k
    largest_character_for_kind =
1294
1.10k
        0xFFFFFFFFu >> (32-PP.getTargetInfo().getWCharWidth());
1295
708k
  } else if (tok::utf8_char_constant == Kind) {
1296
147
    largest_character_for_kind = 0x7F;
1297
708k
  } else if (tok::utf16_char_constant == Kind) {
1298
96
    largest_character_for_kind = 0xFFFF;
1299
708k
  } else if (tok::utf32_char_constant == Kind) {
1300
87
    largest_character_for_kind = 0x10FFFF;
1301
708k
  } else {
1302
708k
    largest_character_for_kind = 0x7Fu;
1303
708k
  }
1304
1305
1.42M
  while (begin != end) {
1306
    // Is this a span of non-escape characters?
1307
710k
    if (begin[0] != '\\') {
1308
705k
      char const *start = begin;
1309
2.74M
      do {
1310
2.74M
        ++begin;
1311
2.74M
      } while (begin != end && 
*begin != '\\'2.03M
);
1312
1313
705k
      char const *tmp_in_start = start;
1314
705k
      uint32_t *tmp_out_start = buffer_begin;
1315
705k
      llvm::ConversionResult res =
1316
705k
          llvm::ConvertUTF8toUTF32(reinterpret_cast<llvm::UTF8 const **>(&start),
1317
705k
                             reinterpret_cast<llvm::UTF8 const *>(begin),
1318
705k
                             &buffer_begin, buffer_end, llvm::strictConversion);
1319
705k
      if (res != llvm::conversionOK) {
1320
        // If we see bad encoding for unprefixed character literals, warn and
1321
        // simply copy the byte values, for compatibility with gcc and
1322
        // older versions of clang.
1323
6
        bool NoErrorOnBadEncoding = isAscii();
1324
6
        unsigned Msg = diag::err_bad_character_encoding;
1325
6
        if (NoErrorOnBadEncoding)
1326
3
          Msg = diag::warn_bad_character_encoding;
1327
6
        PP.Diag(Loc, Msg);
1328
6
        if (NoErrorOnBadEncoding) {
1329
3
          start = tmp_in_start;
1330
3
          buffer_begin = tmp_out_start;
1331
7
          for (; start != begin; 
++start, ++buffer_begin4
)
1332
4
            *buffer_begin = static_cast<uint8_t>(*start);
1333
3
        } else {
1334
3
          HadError = true;
1335
3
        }
1336
705k
      } else {
1337
3.44M
        for (; tmp_out_start < buffer_begin; 
++tmp_out_start2.74M
) {
1338
2.74M
          if (*tmp_out_start > largest_character_for_kind) {
1339
13
            HadError = true;
1340
13
            PP.Diag(Loc, diag::err_character_too_large);
1341
13
          }
1342
2.74M
        }
1343
705k
      }
1344
1345
705k
      continue;
1346
705k
    }
1347
    // Is this a Universal Character Name escape?
1348
5.10k
    if (begin[1] == 'u' || 
begin[1] == 'U'5.01k
) {
1349
107
      unsigned short UcnLen = 0;
1350
107
      if (!ProcessUCNEscape(TokBegin, begin, end, *buffer_begin, UcnLen,
1351
107
                            FullSourceLoc(Loc, PP.getSourceManager()),
1352
29
                            &PP.getDiagnostics(), PP.getLangOpts(), true)) {
1353
29
        HadError = true;
1354
78
      } else if (*buffer_begin > largest_character_for_kind) {
1355
8
        HadError = true;
1356
8
        PP.Diag(Loc, diag::err_character_too_large);
1357
8
      }
1358
1359
107
      ++buffer_begin;
1360
107
      continue;
1361
107
    }
1362
4.99k
    unsigned CharWidth = getCharWidth(Kind, PP.getTargetInfo());
1363
4.99k
    uint64_t result =
1364
4.99k
      ProcessCharEscape(TokBegin, begin, end, HadError,
1365
4.99k
                        FullSourceLoc(Loc,PP.getSourceManager()),
1366
4.99k
                        CharWidth, &PP.getDiagnostics(), PP.getLangOpts());
1367
4.99k
    *buffer_begin++ = result;
1368
4.99k
  }
1369
1370
709k
  unsigned NumCharsSoFar = buffer_begin - &codepoint_buffer.front();
1371
1372
709k
  if (NumCharsSoFar > 1) {
1373
679k
    if (isWide())
1374
17
      PP.Diag(Loc, diag::warn_extraneous_char_constant);
1375
679k
    else if (isAscii() && 
NumCharsSoFar == 4679k
)
1376
679k
      PP.Diag(Loc, diag::warn_four_char_character_literal);
1377
31
    else if (isAscii())
1378
26
      PP.Diag(Loc, diag::warn_multichar_character_literal);
1379
5
    else
1380
5
      PP.Diag(Loc, diag::err_multichar_utf_character_literal);
1381
679k
    IsMultiChar = true;
1382
30.3k
  } else {
1383
30.3k
    IsMultiChar = false;
1384
30.3k
  }
1385
1386
709k
  llvm::APInt LitVal(PP.getTargetInfo().getIntWidth(), 0);
1387
1388
  // Narrow character literals act as though their value is concatenated
1389
  // in this implementation, but warn on overflow.
1390
709k
  bool multi_char_too_long = false;
1391
709k
  if (isAscii() && 
isMultiChar()708k
) {
1392
679k
    LitVal = 0;
1393
3.39M
    for (size_t i = 0; i < NumCharsSoFar; 
++i2.71M
) {
1394
      // check for enough leading zeros to shift into
1395
2.71M
      multi_char_too_long |= (LitVal.countLeadingZeros() < 8);
1396
2.71M
      LitVal <<= 8;
1397
2.71M
      LitVal = LitVal + (codepoint_buffer[i] & 0xFF);
1398
2.71M
    }
1399
30.3k
  } else if (NumCharsSoFar > 0) {
1400
    // otherwise just take the last character
1401
30.3k
    LitVal = buffer_begin[-1];
1402
30.3k
  }
1403
1404
709k
  if (!HadError && 
multi_char_too_long709k
) {
1405
4
    PP.Diag(Loc, diag::warn_char_constant_too_large);
1406
4
  }
1407
1408
  // Transfer the value from APInt to uint64_t
1409
709k
  Value = LitVal.getZExtValue();
1410
1411
  // If this is a single narrow character, sign extend it (e.g. '\xFF' is "-1")
1412
  // if 'char' is signed for this target (C99 6.4.4.4p10).  Note that multiple
1413
  // character constants are not sign extended in the this implementation:
1414
  // '\xFF\xFF' = 65536 and '\x0\xFF' = 255, which matches GCC.
1415
709k
  if (isAscii() && 
NumCharsSoFar == 1708k
&&
(Value & 128)28.9k
&&
1416
101
      PP.getLangOpts().CharIsSigned)
1417
79
    Value = (signed char)Value;
1418
709k
}
1419
1420
/// \verbatim
1421
///       string-literal: [C++0x lex.string]
1422
///         encoding-prefix " [s-char-sequence] "
1423
///         encoding-prefix R raw-string
1424
///       encoding-prefix:
1425
///         u8
1426
///         u
1427
///         U
1428
///         L
1429
///       s-char-sequence:
1430
///         s-char
1431
///         s-char-sequence s-char
1432
///       s-char:
1433
///         any member of the source character set except the double-quote ",
1434
///           backslash \, or new-line character
1435
///         escape-sequence
1436
///         universal-character-name
1437
///       raw-string:
1438
///         " d-char-sequence ( r-char-sequence ) d-char-sequence "
1439
///       r-char-sequence:
1440
///         r-char
1441
///         r-char-sequence r-char
1442
///       r-char:
1443
///         any member of the source character set, except a right parenthesis )
1444
///           followed by the initial d-char-sequence (which may be empty)
1445
///           followed by a double quote ".
1446
///       d-char-sequence:
1447
///         d-char
1448
///         d-char-sequence d-char
1449
///       d-char:
1450
///         any member of the basic source character set except:
1451
///           space, the left parenthesis (, the right parenthesis ),
1452
///           the backslash \, and the control characters representing horizontal
1453
///           tab, vertical tab, form feed, and newline.
1454
///       escape-sequence: [C++0x lex.ccon]
1455
///         simple-escape-sequence
1456
///         octal-escape-sequence
1457
///         hexadecimal-escape-sequence
1458
///       simple-escape-sequence:
1459
///         one of \' \" \? \\ \a \b \f \n \r \t \v
1460
///       octal-escape-sequence:
1461
///         \ octal-digit
1462
///         \ octal-digit octal-digit
1463
///         \ octal-digit octal-digit octal-digit
1464
///       hexadecimal-escape-sequence:
1465
///         \x hexadecimal-digit
1466
///         hexadecimal-escape-sequence hexadecimal-digit
1467
///       universal-character-name:
1468
///         \u hex-quad
1469
///         \U hex-quad hex-quad
1470
///       hex-quad:
1471
///         hex-digit hex-digit hex-digit hex-digit
1472
/// \endverbatim
1473
///
1474
StringLiteralParser::
1475
StringLiteralParser(ArrayRef<Token> StringToks,
1476
                    Preprocessor &PP, bool Complain)
1477
  : SM(PP.getSourceManager()), Features(PP.getLangOpts()),
1478
    Target(PP.getTargetInfo()), Diags(Complain ? &PP.getDiagnostics() :nullptr),
1479
    MaxTokenLength(0), SizeBound(0), CharByteWidth(0), Kind(tok::unknown),
1480
4.35M
    ResultPtr(ResultBuf.data()), hadError(false), Pascal(false) {
1481
4.35M
  init(StringToks);
1482
4.35M
}
1483
1484
5.47M
void StringLiteralParser::init(ArrayRef<Token> StringToks){
1485
  // The literal token may have come from an invalid source location (e.g. due
1486
  // to a PCH error), in which case the token length will be 0.
1487
5.47M
  if (
StringToks.empty()5.47M
|| StringToks[0].getLength() < 2)
1488
0
    return DiagnoseLexingError(SourceLocation());
1489
1490
  // Scan all of the string portions, remember the max individual token length,
1491
  // computing a bound on the concatenated string length, and see whether any
1492
  // piece is a wide-string.  If any of the string portions is a wide-string
1493
  // literal, the result is a wide-string literal [C99 6.4.5p4].
1494
5.47M
  assert(!StringToks.empty() && "expected at least one token");
1495
5.47M
  MaxTokenLength = StringToks[0].getLength();
1496
5.47M
  assert(StringToks[0].getLength() >= 2 && "literal token is invalid!");
1497
5.47M
  SizeBound = StringToks[0].getLength()-2;  // -2 for "".
1498
5.47M
  Kind = StringToks[0].getKind();
1499
1500
5.47M
  hadError = false;
1501
1502
  // Implement Translation Phase #6: concatenation of string literals
1503
  /// (C99 5.1.1.2p1).  The common case is only one string fragment.
1504
5.68M
  for (unsigned i = 1; i != StringToks.size(); 
++i204k
) {
1505
204k
    if (StringToks[i].getLength() < 2)
1506
0
      return DiagnoseLexingError(StringToks[i].getLocation());
1507
1508
    // The string could be shorter than this if it needs cleaning, but this is a
1509
    // reasonable bound, which is all we need.
1510
204k
    assert(StringToks[i].getLength() >= 2 && "literal token is invalid!");
1511
204k
    SizeBound += StringToks[i].getLength()-2;  // -2 for "".
1512
1513
    // Remember maximum string piece length.
1514
204k
    if (StringToks[i].getLength() > MaxTokenLength)
1515
173k
      MaxTokenLength = StringToks[i].getLength();
1516
1517
    // Remember if we see any wide or utf-8/16/32 strings.
1518
    // Also check for illegal concatenations.
1519
204k
    if (StringToks[i].isNot(Kind) && 
StringToks[i].isNot(tok::string_literal)83
) {
1520
56
      if (isAscii()) {
1521
20
        Kind = StringToks[i].getKind();
1522
36
      } else {
1523
36
        if (Diags)
1524
36
          Diags->Report(StringToks[i].getLocation(),
1525
36
                        diag::err_unsupported_string_concat);
1526
36
        hadError = true;
1527
36
      }
1528
56
    }
1529
204k
  }
1530
1531
  // Include space for the null terminator.
1532
5.47M
  ++SizeBound;
1533
1534
  // TODO: K&R warning: "traditional C rejects string constant concatenation"
1535
1536
  // Get the width in bytes of char/wchar_t/char16_t/char32_t
1537
5.47M
  CharByteWidth = getCharWidth(Kind, Target);
1538
5.47M
  assert((CharByteWidth & 7) == 0 && "Assumes character size is byte multiple");
1539
5.47M
  CharByteWidth /= 8;
1540
1541
  // The output buffer size needs to be large enough to hold wide characters.
1542
  // This is a worst-case assumption which basically corresponds to L"" "long".
1543
5.47M
  SizeBound *= CharByteWidth;
1544
1545
  // Size the temporary buffer to hold the result string data.
1546
5.47M
  ResultBuf.resize(SizeBound);
1547
1548
  // Likewise, but for each string piece.
1549
5.47M
  SmallString<512> TokenBuf;
1550
5.47M
  TokenBuf.resize(MaxTokenLength);
1551
1552
  // Loop over all the strings, getting their spelling, and expanding them to
1553
  // wide strings as appropriate.
1554
5.47M
  ResultPtr = &ResultBuf[0];   // Next byte to fill in.
1555
1556
5.47M
  Pascal = false;
1557
1558
5.47M
  SourceLocation UDSuffixTokLoc;
1559
1560
11.1M
  for (unsigned i = 0, e = StringToks.size(); i != e; 
++i5.68M
) {
1561
5.68M
    const char *ThisTokBuf = &TokenBuf[0];
1562
    // Get the spelling of the token, which eliminates trigraphs, etc.  We know
1563
    // that ThisTokBuf points to a buffer that is big enough for the whole token
1564
    // and 'spelled' tokens can only shrink.
1565
5.68M
    bool StringInvalid = false;
1566
5.68M
    unsigned ThisTokLen =
1567
5.68M
      Lexer::getSpelling(StringToks[i], ThisTokBuf, SM, Features,
1568
5.68M
                         &StringInvalid);
1569
5.68M
    if (StringInvalid)
1570
0
      return DiagnoseLexingError(StringToks[i].getLocation());
1571
1572
5.68M
    const char *ThisTokBegin = ThisTokBuf;
1573
5.68M
    const char *ThisTokEnd = ThisTokBuf+ThisTokLen;
1574
1575
    // Remove an optional ud-suffix.
1576
5.68M
    if (ThisTokEnd[-1] != '"') {
1577
429
      const char *UDSuffixEnd = ThisTokEnd;
1578
1.24k
      do {
1579
1.24k
        --ThisTokEnd;
1580
1.24k
      } while (ThisTokEnd[-1] != '"');
1581
1582
429
      StringRef UDSuffix(ThisTokEnd, UDSuffixEnd - ThisTokEnd);
1583
1584
429
      if (UDSuffixBuf.empty()) {
1585
411
        if (StringToks[i].hasUCN())
1586
8
          expandUCNs(UDSuffixBuf, UDSuffix);
1587
403
        else
1588
403
          UDSuffixBuf.assign(UDSuffix);
1589
411
        UDSuffixToken = i;
1590
411
        UDSuffixOffset = ThisTokEnd - ThisTokBuf;
1591
411
        UDSuffixTokLoc = StringToks[i].getLocation();
1592
18
      } else {
1593
18
        SmallString<32> ExpandedUDSuffix;
1594
18
        if (StringToks[i].hasUCN()) {
1595
9
          expandUCNs(ExpandedUDSuffix, UDSuffix);
1596
9
          UDSuffix = ExpandedUDSuffix;
1597
9
        }
1598
1599
        // C++11 [lex.ext]p8: At the end of phase 6, if a string literal is the
1600
        // result of a concatenation involving at least one user-defined-string-
1601
        // literal, all the participating user-defined-string-literals shall
1602
        // have the same ud-suffix.
1603
18
        if (UDSuffixBuf != UDSuffix) {
1604
6
          if (Diags) {
1605
6
            SourceLocation TokLoc = StringToks[i].getLocation();
1606
6
            Diags->Report(TokLoc, diag::err_string_concat_mixed_suffix)
1607
6
              << UDSuffixBuf << UDSuffix
1608
6
              << SourceRange(UDSuffixTokLoc, UDSuffixTokLoc)
1609
6
              << SourceRange(TokLoc, TokLoc);
1610
6
          }
1611
6
          hadError = true;
1612
6
        }
1613
18
      }
1614
429
    }
1615
1616
    // Strip the end quote.
1617
5.68M
    --ThisTokEnd;
1618
1619
    // TODO: Input character set mapping support.
1620
1621
    // Skip marker for wide or unicode strings.
1622
5.68M
    if (ThisTokBuf[0] == 'L' || 
ThisTokBuf[0] == 'u'5.68M
||
ThisTokBuf[0] == 'U'5.68M
) {
1623
1.72k
      ++ThisTokBuf;
1624
      // Skip 8 of u8 marker for utf8 strings.
1625
1.72k
      if (ThisTokBuf[0] == '8')
1626
307
        ++ThisTokBuf;
1627
1.72k
    }
1628
1629
    // Check for raw string
1630
5.68M
    if (ThisTokBuf[0] == 'R') {
1631
126
      ThisTokBuf += 2; // skip R"
1632
1633
126
      const char *Prefix = ThisTokBuf;
1634
341
      while (ThisTokBuf[0] != '(')
1635
215
        ++ThisTokBuf;
1636
126
      ++ThisTokBuf; // skip '('
1637
1638
      // Remove same number of characters from the end
1639
126
      ThisTokEnd -= ThisTokBuf - Prefix;
1640
126
      assert(ThisTokEnd >= ThisTokBuf && "malformed raw string literal");
1641
1642
      // C++14 [lex.string]p4: A source-file new-line in a raw string literal
1643
      // results in a new-line in the resulting execution string-literal.
1644
126
      StringRef RemainingTokenSpan(ThisTokBuf, ThisTokEnd - ThisTokBuf);
1645
249
      while (!RemainingTokenSpan.empty()) {
1646
        // Split the string literal on \r\n boundaries.
1647
123
        size_t CRLFPos = RemainingTokenSpan.find("\r\n");
1648
123
        StringRef BeforeCRLF = RemainingTokenSpan.substr(0, CRLFPos);
1649
123
        StringRef AfterCRLF = RemainingTokenSpan.substr(CRLFPos);
1650
1651
        // Copy everything before the \r\n sequence into the string literal.
1652
123
        if (CopyStringFragment(StringToks[i], ThisTokBegin, BeforeCRLF))
1653
6
          hadError = true;
1654
1655
        // Point into the \n inside the \r\n sequence and operate on the
1656
        // remaining portion of the literal.
1657
123
        RemainingTokenSpan = AfterCRLF.substr(1);
1658
123
      }
1659
5.68M
    } else {
1660
5.68M
      if (ThisTokBuf[0] != '"') {
1661
        // The file may have come from PCH and then changed after loading the
1662
        // PCH; Fail gracefully.
1663
0
        return DiagnoseLexingError(StringToks[i].getLocation());
1664
0
      }
1665
5.68M
      ++ThisTokBuf; // skip "
1666
1667
      // Check if this is a pascal string
1668
5.68M
      if (Features.PascalStrings && 
ThisTokBuf + 1 != ThisTokEnd96
&&
1669
96
          ThisTokBuf[0] == '\\' && 
ThisTokBuf[1] == 'p'15
) {
1670
1671
        // If the \p sequence is found in the first token, we have a pascal string
1672
        // Otherwise, if we already have a pascal string, ignore the first \p
1673
15
        if (i == 0) {
1674
15
          ++ThisTokBuf;
1675
15
          Pascal = true;
1676
0
        } else if (Pascal)
1677
0
          ThisTokBuf += 2;
1678
15
      }
1679
1680
11.3M
      while (ThisTokBuf != ThisTokEnd) {
1681
        // Is this a span of non-escape characters?
1682
5.66M
        if (ThisTokBuf[0] != '\\') {
1683
5.59M
          const char *InStart = ThisTokBuf;
1684
77.8M
          do {
1685
77.8M
            ++ThisTokBuf;
1686
77.8M
          } while (ThisTokBuf != ThisTokEnd && 
ThisTokBuf[0] != '\\'72.3M
);
1687
1688
          // Copy the character span over.
1689
5.59M
          if (CopyStringFragment(StringToks[i], ThisTokBegin,
1690
5.59M
                                 StringRef(InStart, ThisTokBuf - InStart)))
1691
8
            hadError = true;
1692
5.59M
          continue;
1693
5.59M
        }
1694
        // Is this a Universal Character Name escape?
1695
66.6k
        if (ThisTokBuf[1] == 'u' || 
ThisTokBuf[1] == 'U'66.4k
) {
1696
287
          EncodeUCNEscape(ThisTokBegin, ThisTokBuf, ThisTokEnd,
1697
287
                          ResultPtr, hadError,
1698
287
                          FullSourceLoc(StringToks[i].getLocation(), SM),
1699
287
                          CharByteWidth, Diags, Features);
1700
287
          continue;
1701
287
        }
1702
        // Otherwise, this is a non-UCN escape character.  Process it.
1703
66.4k
        unsigned ResultChar =
1704
66.4k
          ProcessCharEscape(ThisTokBegin, ThisTokBuf, ThisTokEnd, hadError,
1705
66.4k
                            FullSourceLoc(StringToks[i].getLocation(), SM),
1706
66.4k
                            CharByteWidth*8, Diags, Features);
1707
1708
66.4k
        if (CharByteWidth == 4) {
1709
          // FIXME: Make the type of the result buffer correct instead of
1710
          // using reinterpret_cast.
1711
240
          llvm::UTF32 *ResultWidePtr = reinterpret_cast<llvm::UTF32*>(ResultPtr);
1712
240
          *ResultWidePtr = ResultChar;
1713
240
          ResultPtr += 4;
1714
66.1k
        } else if (CharByteWidth == 2) {
1715
          // FIXME: Make the type of the result buffer correct instead of
1716
          // using reinterpret_cast.
1717
31
          llvm::UTF16 *ResultWidePtr = reinterpret_cast<llvm::UTF16*>(ResultPtr);
1718
31
          *ResultWidePtr = ResultChar & 0xFFFF;
1719
31
          ResultPtr += 2;
1720
66.1k
        } else {
1721
66.1k
          assert(CharByteWidth == 1 && "Unexpected char width");
1722
66.1k
          *ResultPtr++ = ResultChar & 0xFF;
1723
66.1k
        }
1724
66.4k
      }
1725
5.68M
    }
1726
5.68M
  }
1727
1728
5.47M
  if (Pascal) {
1729
15
    if (CharByteWidth == 4) {
1730
      // FIXME: Make the type of the result buffer correct instead of
1731
      // using reinterpret_cast.
1732
1
      llvm::UTF32 *ResultWidePtr = reinterpret_cast<llvm::UTF32*>(ResultBuf.data());
1733
1
      ResultWidePtr[0] = GetNumStringChars() - 1;
1734
14
    } else if (CharByteWidth == 2) {
1735
      // FIXME: Make the type of the result buffer correct instead of
1736
      // using reinterpret_cast.
1737
3
      llvm::UTF16 *ResultWidePtr = reinterpret_cast<llvm::UTF16*>(ResultBuf.data());
1738
3
      ResultWidePtr[0] = GetNumStringChars() - 1;
1739
11
    } else {
1740
11
      assert(CharByteWidth == 1 && "Unexpected char width");
1741
11
      ResultBuf[0] = GetNumStringChars() - 1;
1742
11
    }
1743
1744
    // Verify that pascal strings aren't too large.
1745
15
    if (GetStringLength() > 256) {
1746
0
      if (Diags)
1747
0
        Diags->Report(StringToks.front().getLocation(),
1748
0
                      diag::err_pascal_string_too_long)
1749
0
          << SourceRange(StringToks.front().getLocation(),
1750
0
                         StringToks.back().getLocation());
1751
0
      hadError = true;
1752
0
      return;
1753
0
    }
1754
5.47M
  } else if (Diags) {
1755
    // Complain if this string literal has too many characters.
1756
2.34M
    unsigned MaxChars = Features.CPlusPlus? 
655362.00M
: Features.C99 ?
40952.34M
:
5093.20k
;
1757
1758
4.35M
    if (GetNumStringChars() > MaxChars)
1759
1
      Diags->Report(StringToks.front().getLocation(),
1760
1
                    diag::ext_string_too_long)
1761
1
        << GetNumStringChars() << MaxChars
1762
1
        << (Features.CPlusPlus ? 
20
: Features.C99 ?
10
: 0)
1763
1
        << SourceRange(StringToks.front().getLocation(),
1764
1
                       StringToks.back().getLocation());
1765
4.35M
  }
1766
5.47M
}
1767
1768
212
static const char *resyncUTF8(const char *Err, const char *End) {
1769
212
  if (Err == End)
1770
0
    return End;
1771
212
  End = Err + std::min<unsigned>(llvm::getNumBytesForUTF8(*Err), End-Err);
1772
239
  while (++Err != End && 
(*Err & 0xC0) == 0x80186
)
1773
27
    ;
1774
212
  return Err;
1775
212
}
1776
1777
/// This function copies from Fragment, which is a sequence of bytes
1778
/// within Tok's contents (which begin at TokBegin) into ResultPtr.
1779
/// Performs widening for multi-byte characters.
1780
bool StringLiteralParser::CopyStringFragment(const Token &Tok,
1781
                                             const char *TokBegin,
1782
5.59M
                                             StringRef Fragment) {
1783
5.59M
  const llvm::UTF8 *ErrorPtrTmp;
1784
5.59M
  if (ConvertUTF8toWide(CharByteWidth, Fragment, ResultPtr, ErrorPtrTmp))
1785
5.59M
    return false;
1786
1787
  // If we see bad encoding for unprefixed string literals, warn and
1788
  // simply copy the byte values, for compatibility with gcc and older
1789
  // versions of clang.
1790
37
  bool NoErrorOnBadEncoding = isAscii();
1791
37
  if (NoErrorOnBadEncoding) {
1792
20
    memcpy(ResultPtr, Fragment.data(), Fragment.size());
1793
20
    ResultPtr += Fragment.size();
1794
20
  }
1795
1796
37
  if (Diags) {
1797
34
    const char *ErrorPtr = reinterpret_cast<const char *>(ErrorPtrTmp);
1798
1799
34
    FullSourceLoc SourceLoc(Tok.getLocation(), SM);
1800
34
    const DiagnosticBuilder &Builder =
1801
34
      Diag(Diags, Features, SourceLoc, TokBegin,
1802
34
           ErrorPtr, resyncUTF8(ErrorPtr, Fragment.end()),
1803
20
           NoErrorOnBadEncoding ? diag::warn_bad_string_encoding
1804
14
                                : diag::err_bad_string_encoding);
1805
1806
34
    const char *NextStart = resyncUTF8(ErrorPtr, Fragment.end());
1807
34
    StringRef NextFragment(NextStart, Fragment.end()-NextStart);
1808
1809
    // Decode into a dummy buffer.
1810
34
    SmallString<512> Dummy;
1811
34
    Dummy.reserve(Fragment.size() * CharByteWidth);
1812
34
    char *Ptr = Dummy.data();
1813
1814
178
    while (!ConvertUTF8toWide(CharByteWidth, NextFragment, Ptr, ErrorPtrTmp)) {
1815
144
      const char *ErrorPtr = reinterpret_cast<const char *>(ErrorPtrTmp);
1816
144
      NextStart = resyncUTF8(ErrorPtr, Fragment.end());
1817
144
      Builder << MakeCharSourceRange(Features, SourceLoc, TokBegin,
1818
144
                                     ErrorPtr, NextStart);
1819
144
      NextFragment = StringRef(NextStart, Fragment.end()-NextStart);
1820
144
    }
1821
34
  }
1822
37
  return !NoErrorOnBadEncoding;
1823
37
}
1824
1825
0
void StringLiteralParser::DiagnoseLexingError(SourceLocation Loc) {
1826
0
  hadError = true;
1827
0
  if (Diags)
1828
0
    Diags->Report(Loc, diag::err_lexing_string);
1829
0
}
1830
1831
/// getOffsetOfStringByte - This function returns the offset of the
1832
/// specified byte of the string data represented by Token.  This handles
1833
/// advancing over escape sequences in the string.
1834
unsigned StringLiteralParser::getOffsetOfStringByte(const Token &Tok,
1835
39.3k
                                                    unsigned ByteNo) const {
1836
  // Get the spelling of the token.
1837
39.3k
  SmallString<32> SpellingBuffer;
1838
39.3k
  SpellingBuffer.resize(Tok.getLength());
1839
1840
39.3k
  bool StringInvalid = false;
1841
39.3k
  const char *SpellingPtr = &SpellingBuffer[0];
1842
39.3k
  unsigned TokLen = Lexer::getSpelling(Tok, SpellingPtr, SM, Features,
1843
39.3k
                                       &StringInvalid);
1844
39.3k
  if (StringInvalid)
1845
0
    return 0;
1846
1847
39.3k
  const char *SpellingStart = SpellingPtr;
1848
39.3k
  const char *SpellingEnd = SpellingPtr+TokLen;
1849
1850
  // Handle UTF-8 strings just like narrow strings.
1851
39.3k
  if (SpellingPtr[0] == 'u' && 
SpellingPtr[1] == '8'3
)
1852
3
    SpellingPtr += 2;
1853
1854
39.3k
  assert(SpellingPtr[0] != 'L' && SpellingPtr[0] != 'u' &&
1855
39.3k
         SpellingPtr[0] != 'U' && "Doesn't handle wide or utf strings yet");
1856
1857
  // For raw string literals, this is easy.
1858
39.3k
  if (SpellingPtr[0] == 'R') {
1859
6
    assert(SpellingPtr[1] == '"' && "Should be a raw string literal!");
1860
    // Skip 'R"'.
1861
6
    SpellingPtr += 2;
1862
35
    while (*SpellingPtr != '(') {
1863
29
      ++SpellingPtr;
1864
29
      assert(SpellingPtr < SpellingEnd && "Missing ( for raw string literal");
1865
29
    }
1866
    // Skip '('.
1867
6
    ++SpellingPtr;
1868
6
    return SpellingPtr - SpellingStart + ByteNo;
1869
6
  }
1870
1871
  // Skip over the leading quote
1872
39.3k
  assert(SpellingPtr[0] == '"' && "Should be a string literal!");
1873
39.3k
  ++SpellingPtr;
1874
1875
  // Skip over bytes until we find the offset we're looking for.
1876
696k
  while (ByteNo) {
1877
657k
    assert(SpellingPtr < SpellingEnd && "Didn't find byte offset!");
1878
1879
    // Step over non-escapes simply.
1880
657k
    if (*SpellingPtr != '\\') {
1881
656k
      ++SpellingPtr;
1882
656k
      --ByteNo;
1883
656k
      continue;
1884
656k
    }
1885
1886
    // Otherwise, this is an escape character.  Advance over it.
1887
834
    bool HadError = false;
1888
834
    if (SpellingPtr[1] == 'u' || 
SpellingPtr[1] == 'U'831
) {
1889
6
      const char *EscapePtr = SpellingPtr;
1890
6
      unsigned Len = MeasureUCNEscape(SpellingStart, SpellingPtr, SpellingEnd,
1891
6
                                      1, Features, HadError);
1892
6
      if (Len > ByteNo) {
1893
        // ByteNo is somewhere within the escape sequence.
1894
6
        SpellingPtr = EscapePtr;
1895
6
        break;
1896
6
      }
1897
0
      ByteNo -= Len;
1898
828
    } else {
1899
828
      ProcessCharEscape(SpellingStart, SpellingPtr, SpellingEnd, HadError,
1900
828
                        FullSourceLoc(Tok.getLocation(), SM),
1901
828
                        CharByteWidth*8, Diags, Features);
1902
828
      --ByteNo;
1903
828
    }
1904
828
    assert(!HadError && "This method isn't valid on erroneous strings");
1905
828
  }
1906
1907
39.3k
  return SpellingPtr-SpellingStart;
1908
39.3k
}
1909
1910
/// Determine whether a suffix is a valid ud-suffix. We avoid treating reserved
1911
/// suffixes as ud-suffixes, because the diagnostic experience is better if we
1912
/// treat it as an invalid suffix.
1913
bool StringLiteralParser::isValidUDSuffix(const LangOptions &LangOpts,
1914
504
                                          StringRef Suffix) {
1915
504
  return NumericLiteralParser::isValidUDSuffix(LangOpts, Suffix) ||
1916
76
         Suffix == "sv";
1917
504
}