Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/Support/ConvertUTF.cpp
Line
Count
Source (jump to first uncovered line)
1
/*===--- ConvertUTF.c - Universal Character Names conversions ---------------===
2
 *
3
 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
 * See https://llvm.org/LICENSE.txt for license information.
5
 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
 *
7
 *===------------------------------------------------------------------------=*/
8
/*
9
 * Copyright 2001-2004 Unicode, Inc.
10
 *
11
 * Disclaimer
12
 *
13
 * This source code is provided as is by Unicode, Inc. No claims are
14
 * made as to fitness for any particular purpose. No warranties of any
15
 * kind are expressed or implied. The recipient agrees to determine
16
 * applicability of information provided. If this file has been
17
 * purchased on magnetic or optical media from Unicode, Inc., the
18
 * sole remedy for any claim will be exchange of defective media
19
 * within 90 days of receipt.
20
 *
21
 * Limitations on Rights to Redistribute This Code
22
 *
23
 * Unicode, Inc. hereby grants the right to freely use the information
24
 * supplied in this file in the creation of products supporting the
25
 * Unicode Standard, and to make copies of this file in any form
26
 * for internal or external distribution as long as this notice
27
 * remains attached.
28
 */
29
30
/* ---------------------------------------------------------------------
31
32
    Conversions between UTF32, UTF-16, and UTF-8. Source code file.
33
    Author: Mark E. Davis, 1994.
34
    Rev History: Rick McGowan, fixes & updates May 2001.
35
    Sept 2001: fixed const & error conditions per
36
        mods suggested by S. Parent & A. Lillich.
37
    June 2002: Tim Dodd added detection and handling of incomplete
38
        source sequences, enhanced error detection, added casts
39
        to eliminate compiler warnings.
40
    July 2003: slight mods to back out aggressive FFFE detection.
41
    Jan 2004: updated switches in from-UTF8 conversions.
42
    Oct 2004: updated to use UNI_MAX_LEGAL_UTF32 in UTF-32 conversions.
43
44
    See the header file "ConvertUTF.h" for complete documentation.
45
46
------------------------------------------------------------------------ */
47
48
#include "llvm/Support/ConvertUTF.h"
49
#ifdef CVTUTF_DEBUG
50
#include <stdio.h>
51
#endif
52
#include <assert.h>
53
54
/*
55
 * This code extensively uses fall-through switches.
56
 * Keep the compiler from warning about that.
57
 */
58
#if defined(__clang__) && defined(__has_warning)
59
# if __has_warning("-Wimplicit-fallthrough")
60
#  define ConvertUTF_DISABLE_WARNINGS \
61
    _Pragma("clang diagnostic push")  \
62
    _Pragma("clang diagnostic ignored \"-Wimplicit-fallthrough\"")
63
#  define ConvertUTF_RESTORE_WARNINGS \
64
    _Pragma("clang diagnostic pop")
65
# endif
66
#elif defined(__GNUC__) && __GNUC__ > 6
67
# define ConvertUTF_DISABLE_WARNINGS \
68
   _Pragma("GCC diagnostic push")    \
69
   _Pragma("GCC diagnostic ignored \"-Wimplicit-fallthrough\"")
70
# define ConvertUTF_RESTORE_WARNINGS \
71
   _Pragma("GCC diagnostic pop")
72
#endif
73
#ifndef ConvertUTF_DISABLE_WARNINGS
74
# define ConvertUTF_DISABLE_WARNINGS
75
#endif
76
#ifndef ConvertUTF_RESTORE_WARNINGS
77
# define ConvertUTF_RESTORE_WARNINGS
78
#endif
79
80
ConvertUTF_DISABLE_WARNINGS
81
82
namespace llvm {
83
84
static const int halfShift  = 10; /* used for shifting by 10 bits */
85
86
static const UTF32 halfBase = 0x0010000UL;
87
static const UTF32 halfMask = 0x3FFUL;
88
89
28.3M
#define UNI_SUR_HIGH_START  (UTF32)0xD800
90
1
#define UNI_SUR_HIGH_END    (UTF32)0xDBFF
91
2.86k
#define UNI_SUR_LOW_START   (UTF32)0xDC00
92
1.28k
#define UNI_SUR_LOW_END     (UTF32)0xDFFF
93
94
/* --------------------------------------------------------------------- */
95
96
/*
97
 * Index into the table below with the first byte of a UTF-8 sequence to
98
 * get the number of trailing bytes that are supposed to follow it.
99
 * Note that *legal* UTF-8 values can't have 4 or 5-bytes. The table is
100
 * left as-is for anyone who may want to do such conversion, which was
101
 * allowed in earlier algorithms.
102
 */
103
static const char trailingBytesForUTF8[256] = {
104
    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
105
    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
106
    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
107
    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
108
    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
109
    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
110
    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
111
    2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5
112
};
113
114
/*
115
 * Magic values subtracted from a buffer value during UTF8 conversion.
116
 * This table contains as many values as there might be trailing bytes
117
 * in a UTF-8 sequence.
118
 */
119
static const UTF32 offsetsFromUTF8[6] = { 0x00000000UL, 0x00003080UL, 0x000E2080UL,
120
                     0x03C82080UL, 0xFA082080UL, 0x82082080UL };
121
122
/*
123
 * Once the bits are split out into bytes of UTF-8, this is a mask OR-ed
124
 * into the first byte, depending on how many bytes follow.  There are
125
 * as many entries in this table as there are UTF-8 sequence types.
126
 * (I.e., one byte sequence, two byte... etc.). Remember that sequencs
127
 * for *legal* UTF-8 will be 4 or fewer bytes total.
128
 */
129
static const UTF8 firstByteMark[7] = { 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC };
130
131
/* --------------------------------------------------------------------- */
132
133
/* The interface converts a whole buffer to avoid function-call overhead.
134
 * Constants have been gathered. Loops & conditionals have been removed as
135
 * much as possible for efficiency, in favor of drop-through switches.
136
 * (See "Note A" at the bottom of the file for equivalent code.)
137
 * If your compiler supports it, the "isLegalUTF8" call can be turned
138
 * into an inline function.
139
 */
140
141
142
/* --------------------------------------------------------------------- */
143
144
ConversionResult ConvertUTF32toUTF16 (
145
        const UTF32** sourceStart, const UTF32* sourceEnd,
146
0
        UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags) {
147
0
    ConversionResult result = conversionOK;
148
0
    const UTF32* source = *sourceStart;
149
0
    UTF16* target = *targetStart;
150
0
    while (source < sourceEnd) {
151
0
        UTF32 ch;
152
0
        if (target >= targetEnd) {
153
0
            result = targetExhausted; break;
154
0
        }
155
0
        ch = *source++;
156
0
        if (ch <= UNI_MAX_BMP) { /* Target is a character <= 0xFFFF */
157
0
            /* UTF-16 surrogate values are illegal in UTF-32; 0xffff or 0xfffe are both reserved values */
158
0
            if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {
159
0
                if (flags == strictConversion) {
160
0
                    --source; /* return to the illegal value itself */
161
0
                    result = sourceIllegal;
162
0
                    break;
163
0
                } else {
164
0
                    *target++ = UNI_REPLACEMENT_CHAR;
165
0
                }
166
0
            } else {
167
0
                *target++ = (UTF16)ch; /* normal case */
168
0
            }
169
0
        } else if (ch > UNI_MAX_LEGAL_UTF32) {
170
0
            if (flags == strictConversion) {
171
0
                result = sourceIllegal;
172
0
            } else {
173
0
                *target++ = UNI_REPLACEMENT_CHAR;
174
0
            }
175
0
        } else {
176
0
            /* target is a character in range 0xFFFF - 0x10FFFF. */
177
0
            if (target + 1 >= targetEnd) {
178
0
                --source; /* Back up source pointer! */
179
0
                result = targetExhausted; break;
180
0
            }
181
0
            ch -= halfBase;
182
0
            *target++ = (UTF16)((ch >> halfShift) + UNI_SUR_HIGH_START);
183
0
            *target++ = (UTF16)((ch & halfMask) + UNI_SUR_LOW_START);
184
0
        }
185
0
    }
186
0
    *sourceStart = source;
187
0
    *targetStart = target;
188
0
    return result;
189
0
}
190
191
/* --------------------------------------------------------------------- */
192
193
ConversionResult ConvertUTF16toUTF32 (
194
        const UTF16** sourceStart, const UTF16* sourceEnd,
195
0
        UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags) {
196
0
    ConversionResult result = conversionOK;
197
0
    const UTF16* source = *sourceStart;
198
0
    UTF32* target = *targetStart;
199
0
    UTF32 ch, ch2;
200
0
    while (source < sourceEnd) {
201
0
        const UTF16* oldSource = source; /*  In case we have to back up because of target overflow. */
202
0
        ch = *source++;
203
0
        /* If we have a surrogate pair, convert to UTF32 first. */
204
0
        if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) {
205
0
            /* If the 16 bits following the high surrogate are in the source buffer... */
206
0
            if (source < sourceEnd) {
207
0
                ch2 = *source;
208
0
                /* If it's a low surrogate, convert to UTF32. */
209
0
                if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) {
210
0
                    ch = ((ch - UNI_SUR_HIGH_START) << halfShift)
211
0
                        + (ch2 - UNI_SUR_LOW_START) + halfBase;
212
0
                    ++source;
213
0
                } else if (flags == strictConversion) { /* it's an unpaired high surrogate */
214
0
                    --source; /* return to the illegal value itself */
215
0
                    result = sourceIllegal;
216
0
                    break;
217
0
                }
218
0
            } else { /* We don't have the 16 bits following the high surrogate. */
219
0
                --source; /* return to the high surrogate */
220
0
                result = sourceExhausted;
221
0
                break;
222
0
            }
223
0
        } else if (flags == strictConversion) {
224
0
            /* UTF-16 surrogate values are illegal in UTF-32 */
225
0
            if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END) {
226
0
                --source; /* return to the illegal value itself */
227
0
                result = sourceIllegal;
228
0
                break;
229
0
            }
230
0
        }
231
0
        if (target >= targetEnd) {
232
0
            source = oldSource; /* Back up source pointer! */
233
0
            result = targetExhausted; break;
234
0
        }
235
0
        *target++ = ch;
236
0
    }
237
0
    *sourceStart = source;
238
0
    *targetStart = target;
239
#ifdef CVTUTF_DEBUG
240
if (result == sourceIllegal) {
241
    fprintf(stderr, "ConvertUTF16toUTF32 illegal seq 0x%04x,%04x\n", ch, ch2);
242
    fflush(stderr);
243
}
244
#endif
245
    return result;
246
0
}
247
ConversionResult ConvertUTF16toUTF8 (
248
        const UTF16** sourceStart, const UTF16* sourceEnd,
249
145
        UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags) {
250
145
    ConversionResult result = conversionOK;
251
145
    const UTF16* source = *sourceStart;
252
145
    UTF8* target = *targetStart;
253
1.57k
    while (source < sourceEnd) {
254
1.43k
        UTF32 ch;
255
1.43k
        unsigned short bytesToWrite = 0;
256
1.43k
        const UTF32 byteMask = 0xBF;
257
1.43k
        const UTF32 byteMark = 0x80;
258
1.43k
        const UTF16* oldSource = source; /* In case we have to back up because of target overflow. */
259
1.43k
        ch = *source++;
260
1.43k
        /* If we have a surrogate pair, convert to UTF32 first. */
261
1.43k
        if (ch >= UNI_SUR_HIGH_START && 
ch <= 1
UNI_SUR_HIGH_END1
) {
262
1
            /* If the 16 bits following the high surrogate are in the source buffer... */
263
1
            if (source < sourceEnd) {
264
0
                UTF32 ch2 = *source;
265
0
                /* If it's a low surrogate, convert to UTF32. */
266
0
                if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) {
267
0
                    ch = ((ch - UNI_SUR_HIGH_START) << halfShift)
268
0
                        + (ch2 - UNI_SUR_LOW_START) + halfBase;
269
0
                    ++source;
270
0
                } else if (flags == strictConversion) { /* it's an unpaired high surrogate */
271
0
                    --source; /* return to the illegal value itself */
272
0
                    result = sourceIllegal;
273
0
                    break;
274
0
                }
275
1
            } else { /* We don't have the 16 bits following the high surrogate. */
276
1
                --source; /* return to the high surrogate */
277
1
                result = sourceExhausted;
278
1
                break;
279
1
            }
280
1.43k
        } else if (flags == strictConversion) {
281
1.43k
            /* UTF-16 surrogate values are illegal in UTF-32 */
282
1.43k
            if (ch >= UNI_SUR_LOW_START && 
ch <= 0
UNI_SUR_LOW_END0
) {
283
0
                --source; /* return to the illegal value itself */
284
0
                result = sourceIllegal;
285
0
                break;
286
0
            }
287
1.43k
        }
288
1.43k
        /* Figure out how many bytes the result will require */
289
1.43k
        if (ch < (UTF32)0x80) {      bytesToWrite = 1;
290
1.42k
        } else 
if (10
ch < (UTF32)0x80010
) { bytesToWrite = 2;
291
10
        } else if (ch < (UTF32)0x10000) {   bytesToWrite = 3;
292
10
        } else 
if (0
ch < (UTF32)0x1100000
) { bytesToWrite = 4;
293
0
        } else {                            bytesToWrite = 3;
294
0
                                            ch = UNI_REPLACEMENT_CHAR;
295
0
        }
296
1.43k
297
1.43k
        target += bytesToWrite;
298
1.43k
        if (target > targetEnd) {
299
0
            source = oldSource; /* Back up source pointer! */
300
0
            target -= bytesToWrite; result = targetExhausted; break;
301
0
        }
302
1.43k
        switch (bytesToWrite) { /* note: everything falls through. */
303
1.43k
            case 4: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
304
10
            case 3: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
305
10
            case 2: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
306
1.43k
            case 1: *--target =  (UTF8)(ch | firstByteMark[bytesToWrite]);
307
1.43k
        }
308
1.43k
        target += bytesToWrite;
309
1.43k
    }
310
145
    *sourceStart = source;
311
145
    *targetStart = target;
312
145
    return result;
313
145
}
314
315
/* --------------------------------------------------------------------- */
316
317
ConversionResult ConvertUTF32toUTF8 (
318
        const UTF32** sourceStart, const UTF32* sourceEnd,
319
384
        UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags) {
320
384
    ConversionResult result = conversionOK;
321
384
    const UTF32* source = *sourceStart;
322
384
    UTF8* target = *targetStart;
323
964
    while (source < sourceEnd) {
324
580
        UTF32 ch;
325
580
        unsigned short bytesToWrite = 0;
326
580
        const UTF32 byteMask = 0xBF;
327
580
        const UTF32 byteMark = 0x80;
328
580
        ch = *source++;
329
580
        if (flags == strictConversion ) {
330
580
            /* UTF-16 surrogate values are illegal in UTF-32 */
331
580
            if (ch >= UNI_SUR_HIGH_START && 
ch <= 47
UNI_SUR_LOW_END47
) {
332
0
                --source; /* return to the illegal value itself */
333
0
                result = sourceIllegal;
334
0
                break;
335
0
            }
336
580
        }
337
580
        /*
338
580
         * Figure out how many bytes the result will require. Turn any
339
580
         * illegally large UTF32 things (> Plane 17) into replacement chars.
340
580
         */
341
580
        if (ch < (UTF32)0x80) {      bytesToWrite = 1;
342
362
        } else if (ch < (UTF32)0x800) {     bytesToWrite = 2;
343
229
        } else 
if (133
ch < (UTF32)0x10000133
) { bytesToWrite = 3;
344
119
        } else 
if (14
ch <= 14
UNI_MAX_LEGAL_UTF3214
) { bytesToWrite = 4;
345
14
        } else {                            bytesToWrite = 3;
346
0
                                            ch = UNI_REPLACEMENT_CHAR;
347
0
                                            result = sourceIllegal;
348
0
        }
349
580
350
580
        target += bytesToWrite;
351
580
        if (target > targetEnd) {
352
0
            --source; /* Back up source pointer! */
353
0
            target -= bytesToWrite; result = targetExhausted; break;
354
0
        }
355
580
        switch (bytesToWrite) { /* note: everything falls through. */
356
580
            case 4: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
357
133
            case 3: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
358
362
            case 2: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
359
580
            case 1: *--target = (UTF8) (ch | firstByteMark[bytesToWrite]);
360
580
        }
361
580
        target += bytesToWrite;
362
580
    }
363
384
    *sourceStart = source;
364
384
    *targetStart = target;
365
384
    return result;
366
384
}
367
368
/* --------------------------------------------------------------------- */
369
370
/*
371
 * Utility routine to tell whether a sequence of bytes is legal UTF-8.
372
 * This must be called with the length pre-determined by the first byte.
373
 * If not calling this from ConvertUTF8to*, then the length can be set by:
374
 *  length = trailingBytesForUTF8[*source]+1;
375
 * and the sequence is illegal right away if there aren't that many bytes
376
 * available.
377
 * If presented with a length > 4, this returns false.  The Unicode
378
 * definition of UTF-8 goes up to 4-byte sequences.
379
 */
380
381
74.9M
static Boolean isLegalUTF8(const UTF8 *source, int length) {
382
74.9M
    UTF8 a;
383
74.9M
    const UTF8 *srcptr = source+length;
384
74.9M
    switch (length) {
385
74.9M
    
default: return false92
;
386
74.9M
        /* Everything else falls through when "true"... */
387
74.9M
    
case 4: if (404
(a = (*--srcptr)) < 0x80404
||
a > 0xBF388
)
return false23
;
388
4.85k
    case 3: if (
(a = (*--srcptr)) < 0x804.47k
||
a > 0xBF4.46k
)
return false96
;
389
13.7k
    case 2: if (
(a = (*--srcptr)) < 0x809.33k
||
a > 0xBF9.29k
)
return false108
;
390
9.22k
391
9.22k
        switch (*source) {
392
9.22k
            /* no fall-through in this inner switch */
393
9.22k
            
case 0xE0: if (291
a < 0xA0291
)
return false4
; break;
394
287
            
case 0xED: if (30
a > 0x9F30
)
return false28
; break;
395
350
            case 0xF0: if (a < 0x90) 
return false5
; break;
396
345
            
case 0xF4: if (5
a > 0x8F5
)
return false2
; break;
397
8.55k
            default:   if (a < 0x80) 
return false0
;
398
9.18k
        }
399
9.18k
400
74.9M
    case 1: if (
*source >= 0x8074.9M
&&
*source < 0xC29.60k
)
return false426
;
401
74.9M
    }
402
74.9M
    if (*source > 0xF4) 
return false1
;
403
74.9M
    return true;
404
74.9M
}
405
406
/* --------------------------------------------------------------------- */
407
408
/*
409
 * Exported function to return whether a UTF-8 sequence is legal or not.
410
 * This is not used here; it's just exported.
411
 */
412
2.85M
Boolean isLegalUTF8Sequence(const UTF8 *source, const UTF8 *sourceEnd) {
413
2.85M
    int length = trailingBytesForUTF8[*source]+1;
414
2.85M
    if (length > sourceEnd - source) {
415
12
        return false;
416
12
    }
417
2.85M
    return isLegalUTF8(source, length);
418
2.85M
}
419
420
/* --------------------------------------------------------------------- */
421
422
static unsigned
423
findMaximalSubpartOfIllFormedUTF8Sequence(const UTF8 *source,
424
715
                                          const UTF8 *sourceEnd) {
425
715
  UTF8 b1, b2, b3;
426
715
427
715
  assert(!isLegalUTF8Sequence(source, sourceEnd));
428
715
429
715
  /*
430
715
   * Unicode 6.3.0, D93b:
431
715
   *
432
715
   *   Maximal subpart of an ill-formed subsequence: The longest code unit
433
715
   *   subsequence starting at an unconvertible offset that is either:
434
715
   *   a. the initial subsequence of a well-formed code unit sequence, or
435
715
   *   b. a subsequence of length one.
436
715
   */
437
715
438
715
  if (source == sourceEnd)
439
0
    return 0;
440
715
441
715
  /*
442
715
   * Perform case analysis.  See Unicode 6.3.0, Table 3-7. Well-Formed UTF-8
443
715
   * Byte Sequences.
444
715
   */
445
715
446
715
  b1 = *source;
447
715
  ++source;
448
715
  if (b1 >= 0xC2 && 
b1 <= 0xDF299
) {
449
65
    /*
450
65
     * First byte is valid, but we know that this code unit sequence is
451
65
     * invalid, so the maximal subpart has to end after the first byte.
452
65
     */
453
65
    return 1;
454
65
  }
455
650
456
650
  if (source == sourceEnd)
457
154
    return 1;
458
496
459
496
  b2 = *source;
460
496
  ++source;
461
496
462
496
  if (b1 == 0xE0) {
463
11
    return (b2 >= 0xA0 && 
b2 <= 0xBF3
) ?
22
:
19
;
464
11
  }
465
485
  if (b1 >= 0xE1 && 
b1 <= 0xEC196
) {
466
26
    return (b2 >= 0x80 && 
b2 <= 0xBF14
) ?
22
:
124
;
467
26
  }
468
459
  if (b1 == 0xED) {
469
37
    return (b2 >= 0x80 && 
b2 <= 0x9F36
) ?
22
:
135
;
470
37
  }
471
422
  if (b1 >= 0xEE && 
b1 <= 0xEF133
) {
472
6
    return (b2 >= 0x80 && 
b2 <= 0xBF4
) ?
23
:
13
;
473
6
  }
474
416
  if (b1 == 0xF0) {
475
16
    if (b2 >= 0x90 && 
b2 <= 0xBF5
) {
476
4
      if (source == sourceEnd)
477
2
        return 2;
478
2
479
2
      b3 = *source;
480
2
      return (b3 >= 0x80 && b3 <= 0xBF) ? 3 : 
20
;
481
2
    }
482
12
    return 1;
483
12
  }
484
400
  if (b1 >= 0xF1 && 
b1 <= 0xF3111
) {
485
10
    if (b2 >= 0x80 && 
b2 <= 0xBF7
) {
486
4
      if (source == sourceEnd)
487
2
        return 2;
488
2
489
2
      b3 = *source;
490
2
      return (b3 >= 0x80 && b3 <= 0xBF) ? 3 : 
20
;
491
2
    }
492
6
    return 1;
493
6
  }
494
390
  if (b1 == 0xF4) {
495
12
    if (b2 >= 0x80 && 
b2 <= 0x8F11
) {
496
4
      if (source == sourceEnd)
497
2
        return 2;
498
2
499
2
      b3 = *source;
500
2
      return (b3 >= 0x80 && b3 <= 0xBF) ? 3 : 
20
;
501
2
    }
502
8
    return 1;
503
8
  }
504
378
505
378
  assert((b1 >= 0x80 && b1 <= 0xC1) || b1 >= 0xF5);
506
378
  /*
507
378
   * There are no valid sequences that start with these bytes.  Maximal subpart
508
378
   * is defined to have length 1 in these cases.
509
378
   */
510
378
  return 1;
511
378
}
512
513
/* --------------------------------------------------------------------- */
514
515
/*
516
 * Exported function to return the total number of bytes in a codepoint
517
 * represented in UTF-8, given the value of the first byte.
518
 */
519
14.2M
unsigned getNumBytesForUTF8(UTF8 first) {
520
14.2M
  return trailingBytesForUTF8[first] + 1;
521
14.2M
}
522
523
/* --------------------------------------------------------------------- */
524
525
/*
526
 * Exported function to return whether a UTF-8 string is legal or not.
527
 * This is not used here; it's just exported.
528
 */
529
4.10M
Boolean isLegalUTF8String(const UTF8 **source, const UTF8 *sourceEnd) {
530
62.0M
    while (*source != sourceEnd) {
531
57.8M
        int length = trailingBytesForUTF8[**source] + 1;
532
57.8M
        if (length > sourceEnd - *source || 
!isLegalUTF8(*source, length)57.8M
)
533
68
            return false;
534
57.8M
        *source += length;
535
57.8M
    }
536
4.10M
    
return true4.10M
;
537
4.10M
}
538
539
/* --------------------------------------------------------------------- */
540
541
ConversionResult ConvertUTF8toUTF16 (
542
        const UTF8** sourceStart, const UTF8* sourceEnd,
543
432
        UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags) {
544
432
    ConversionResult result = conversionOK;
545
432
    const UTF8* source = *sourceStart;
546
432
    UTF16* target = *targetStart;
547
1.89k
    while (source < sourceEnd) {
548
1.49k
        UTF32 ch = 0;
549
1.49k
        unsigned short extraBytesToRead = trailingBytesForUTF8[*source];
550
1.49k
        if (extraBytesToRead >= sourceEnd - source) {
551
9
            result = sourceExhausted; break;
552
9
        }
553
1.48k
        /* Do this check whether lenient or strict */
554
1.48k
        if (!isLegalUTF8(source, extraBytesToRead+1)) {
555
15
            result = sourceIllegal;
556
15
            break;
557
15
        }
558
1.46k
        /*
559
1.46k
         * The cases all fall through. See "Note A" below.
560
1.46k
         */
561
1.46k
        switch (extraBytesToRead) {
562
1.46k
            case 5: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */
563
0
            case 4: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */
564
5
            case 3: ch += *source++; ch <<= 6;
565
105
            case 2: ch += *source++; ch <<= 6;
566
153
            case 1: ch += *source++; ch <<= 6;
567
1.46k
            case 0: ch += *source++;
568
1.46k
        }
569
1.46k
        ch -= offsetsFromUTF8[extraBytesToRead];
570
1.46k
571
1.46k
        if (target >= targetEnd) {
572
0
            source -= (extraBytesToRead+1); /* Back up source pointer! */
573
0
            result = targetExhausted; break;
574
0
        }
575
1.46k
        if (ch <= UNI_MAX_BMP) { /* Target is a character <= 0xFFFF */
576
1.46k
            /* UTF-16 surrogate values are illegal in UTF-32 */
577
1.46k
            if (ch >= UNI_SUR_HIGH_START && 
ch <= 78
UNI_SUR_LOW_END78
) {
578
0
                if (flags == strictConversion) {
579
0
                    source -= (extraBytesToRead+1); /* return to the illegal value itself */
580
0
                    result = sourceIllegal;
581
0
                    break;
582
0
                } else {
583
0
                    *target++ = UNI_REPLACEMENT_CHAR;
584
0
                }
585
1.46k
            } else {
586
1.46k
                *target++ = (UTF16)ch; /* normal case */
587
1.46k
            }
588
1.46k
        } else 
if (5
ch > 5
UNI_MAX_UTF165
) {
589
0
            if (flags == strictConversion) {
590
0
                result = sourceIllegal;
591
0
                source -= (extraBytesToRead+1); /* return to the start */
592
0
                break; /* Bail out; shouldn't continue */
593
0
            } else {
594
0
                *target++ = UNI_REPLACEMENT_CHAR;
595
0
            }
596
5
        } else {
597
5
            /* target is a character in range 0xFFFF - 0x10FFFF. */
598
5
            if (target + 1 >= targetEnd) {
599
0
                source -= (extraBytesToRead+1); /* Back up source pointer! */
600
0
                result = targetExhausted; break;
601
0
            }
602
5
            ch -= halfBase;
603
5
            *target++ = (UTF16)((ch >> halfShift) + UNI_SUR_HIGH_START);
604
5
            *target++ = (UTF16)((ch & halfMask) + UNI_SUR_LOW_START);
605
5
        }
606
1.46k
    }
607
432
    *sourceStart = source;
608
432
    *targetStart = target;
609
432
    return result;
610
432
}
611
612
/* --------------------------------------------------------------------- */
613
614
static ConversionResult ConvertUTF8toUTF32Impl(
615
        const UTF8** sourceStart, const UTF8* sourceEnd,
616
        UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags,
617
14.1M
        Boolean InputIsPartial) {
618
14.1M
    ConversionResult result = conversionOK;
619
14.1M
    const UTF8* source = *sourceStart;
620
14.1M
    UTF32* target = *targetStart;
621
28.3M
    while (source < sourceEnd) {
622
14.1M
        UTF32 ch = 0;
623
14.1M
        unsigned short extraBytesToRead = trailingBytesForUTF8[*source];
624
14.1M
        if (extraBytesToRead >= sourceEnd - source) {
625
180
            if (flags == strictConversion || 
InputIsPartial157
) {
626
40
                result = sourceExhausted;
627
40
                break;
628
140
            } else {
629
140
                result = sourceIllegal;
630
140
631
140
                /*
632
140
                 * Replace the maximal subpart of ill-formed sequence with
633
140
                 * replacement character.
634
140
                 */
635
140
                source += findMaximalSubpartOfIllFormedUTF8Sequence(source,
636
140
                                                                    sourceEnd);
637
140
                *target++ = UNI_REPLACEMENT_CHAR;
638
140
                continue;
639
140
            }
640
14.1M
        }
641
14.1M
        if (target >= targetEnd) {
642
19
            result = targetExhausted; break;
643
19
        }
644
14.1M
645
14.1M
        /* Do this check whether lenient or strict */
646
14.1M
        if (!isLegalUTF8(source, extraBytesToRead+1)) {
647
677
            result = sourceIllegal;
648
677
            if (flags == strictConversion) {
649
102
                /* Abort conversion. */
650
102
                break;
651
575
            } else {
652
575
                /*
653
575
                 * Replace the maximal subpart of ill-formed sequence with
654
575
                 * replacement character.
655
575
                 */
656
575
                source += findMaximalSubpartOfIllFormedUTF8Sequence(source,
657
575
                                                                    sourceEnd);
658
575
                *target++ = UNI_REPLACEMENT_CHAR;
659
575
                continue;
660
575
            }
661
14.1M
        }
662
14.1M
        /*
663
14.1M
         * The cases all fall through. See "Note A" below.
664
14.1M
         */
665
14.1M
        switch (extraBytesToRead) {
666
14.1M
            case 5: ch += *source++; ch <<= 6;
667
0
            case 4: ch += *source++; ch <<= 6;
668
303
            case 3: ch += *source++; ch <<= 6;
669
2.93k
            case 2: ch += *source++; ch <<= 6;
670
6.40k
            case 1: ch += *source++; ch <<= 6;
671
14.1M
            case 0: ch += *source++;
672
14.1M
        }
673
14.1M
        ch -= offsetsFromUTF8[extraBytesToRead];
674
14.1M
675
14.1M
        if (ch <= UNI_MAX_LEGAL_UTF32) {
676
14.1M
            /*
677
14.1M
             * UTF-16 surrogate values are illegal in UTF-32, and anything
678
14.1M
             * over Plane 17 (> 0x10FFFF) is illegal.
679
14.1M
             */
680
14.1M
            if (ch >= UNI_SUR_HIGH_START && 
ch <= 1.15k
UNI_SUR_LOW_END1.15k
) {
681
0
                if (flags == strictConversion) {
682
0
                    source -= (extraBytesToRead+1); /* return to the illegal value itself */
683
0
                    result = sourceIllegal;
684
0
                    break;
685
0
                } else {
686
0
                    *target++ = UNI_REPLACEMENT_CHAR;
687
0
                }
688
14.1M
            } else {
689
14.1M
                *target++ = ch;
690
14.1M
            }
691
14.1M
        } else { /* i.e., ch > UNI_MAX_LEGAL_UTF32 */
692
0
            result = sourceIllegal;
693
0
            *target++ = UNI_REPLACEMENT_CHAR;
694
0
        }
695
14.1M
    }
696
14.1M
    *sourceStart = source;
697
14.1M
    *targetStart = target;
698
14.1M
    return result;
699
14.1M
}
700
701
ConversionResult ConvertUTF8toUTF32Partial(const UTF8 **sourceStart,
702
                                           const UTF8 *sourceEnd,
703
                                           UTF32 **targetStart,
704
                                           UTF32 *targetEnd,
705
18
                                           ConversionFlags flags) {
706
18
  return ConvertUTF8toUTF32Impl(sourceStart, sourceEnd, targetStart, targetEnd,
707
18
                                flags, /*InputIsPartial=*/true);
708
18
}
709
710
ConversionResult ConvertUTF8toUTF32(const UTF8 **sourceStart,
711
                                    const UTF8 *sourceEnd, UTF32 **targetStart,
712
14.1M
                                    UTF32 *targetEnd, ConversionFlags flags) {
713
14.1M
  return ConvertUTF8toUTF32Impl(sourceStart, sourceEnd, targetStart, targetEnd,
714
14.1M
                                flags, /*InputIsPartial=*/false);
715
14.1M
}
716
717
/* ---------------------------------------------------------------------
718
719
    Note A.
720
    The fall-through switches in UTF-8 reading code save a
721
    temp variable, some decrements & conditionals.  The switches
722
    are equivalent to the following loop:
723
        {
724
            int tmpBytesToRead = extraBytesToRead+1;
725
            do {
726
                ch += *source++;
727
                --tmpBytesToRead;
728
                if (tmpBytesToRead) ch <<= 6;
729
            } while (tmpBytesToRead > 0);
730
        }
731
    In UTF-8 writing code, the switches on "bytesToWrite" are
732
    similarly unrolled loops.
733
734
   --------------------------------------------------------------------- */
735
736
} // namespace llvm
737
738
ConvertUTF_RESTORE_WARNINGS