|
1 |
| -/* auto-generated on 2023-12-01 13:59:01 -0500. Do not edit! */ |
| 1 | +/* auto-generated on 2024-01-29 10:40:15 -0500. Do not edit! */ |
2 | 2 | /* begin file src/simdutf.cpp */
|
3 | 3 | #include "simdutf.h"
|
4 | 4 | /* begin file src/implementation.cpp */
|
@@ -1151,7 +1151,7 @@ namespace icelake {
|
1151 | 1151 | // We should not get warnings while including <x86intrin.h> yet we do
|
1152 | 1152 | // under some versions of GCC.
|
1153 | 1153 | // If the x86intrin.h header has uninitialized values that are problematic,
|
1154 |
| -// it is a GCC issue, we want to ignore these warnigns. |
| 1154 | +// it is a GCC issue, we want to ignore these warnings. |
1155 | 1155 | SIMDUTF_DISABLE_GCC_WARNING(-Wuninitialized)
|
1156 | 1156 | #endif
|
1157 | 1157 |
|
@@ -1568,7 +1568,7 @@ class implementation final : public simdutf::implementation {
|
1568 | 1568 | // We should not get warnings while including <x86intrin.h> yet we do
|
1569 | 1569 | // under some versions of GCC.
|
1570 | 1570 | // If the x86intrin.h header has uninitialized values that are problematic,
|
1571 |
| -// it is a GCC issue, we want to ignore these warnigns. |
| 1571 | +// it is a GCC issue, we want to ignore these warnings. |
1572 | 1572 | SIMDUTF_DISABLE_GCC_WARNING(-Wuninitialized)
|
1573 | 1573 | #endif
|
1574 | 1574 |
|
@@ -2498,7 +2498,7 @@ class implementation final : public simdutf::implementation {
|
2498 | 2498 | // We should not get warnings while including <x86intrin.h> yet we do
|
2499 | 2499 | // under some versions of GCC.
|
2500 | 2500 | // If the x86intrin.h header has uninitialized values that are problematic,
|
2501 |
| -// it is a GCC issue, we want to ignore these warnigns. |
| 2501 | +// it is a GCC issue, we want to ignore these warnings. |
2502 | 2502 | SIMDUTF_DISABLE_GCC_WARNING(-Wuninitialized)
|
2503 | 2503 | #endif
|
2504 | 2504 |
|
@@ -11655,7 +11655,7 @@ inline result convert_with_errors(const char* buf, size_t len, char16_t* utf16_o
|
11655 | 11655 | *
|
11656 | 11656 | * The caller is responsible to ensure that len > 0.
|
11657 | 11657 | *
|
11658 |
| - * If the error is believed to have occured prior to 'buf', the count value contain in the result |
| 11658 | + * If the error is believed to have occurred prior to 'buf', the count value contain in the result |
11659 | 11659 | * will be SIZE_T - 1, SIZE_T - 2, or SIZE_T - 3.
|
11660 | 11660 | */
|
11661 | 11661 | template <endianness endian>
|
@@ -11934,7 +11934,7 @@ inline result convert_with_errors(const char* buf, size_t len, char32_t* utf32_o
|
11934 | 11934 | *
|
11935 | 11935 | * The caller is responsible to ensure that len > 0.
|
11936 | 11936 | *
|
11937 |
| - * If the error is believed to have occured prior to 'buf', the count value contain in the result |
| 11937 | + * If the error is believed to have occurred prior to 'buf', the count value contain in the result |
11938 | 11938 | * will be SIZE_T - 1, SIZE_T - 2, or SIZE_T - 3.
|
11939 | 11939 | */
|
11940 | 11940 | inline result rewind_and_convert_with_errors(size_t prior_bytes, const char* buf, size_t len, char32_t* utf32_output) {
|
@@ -16084,11 +16084,11 @@ using namespace simd;
|
16084 | 16084 | // 8 bytes when calling convert_masked_utf8_to_utf32. If you skip the last 16 bytes,
|
16085 | 16085 | // and if the data is valid, then it is entirely safe because 16 UTF-8 bytes generate
|
16086 | 16086 | // much more than 8 bytes. However, you cannot generally assume that you have valid
|
16087 |
| - // UTF-8 input, so we are going to go back from the end counting 4 leading bytes, |
| 16087 | + // UTF-8 input, so we are going to go back from the end counting 8 leading bytes, |
16088 | 16088 | // to give us a good margin.
|
16089 | 16089 | size_t leading_byte = 0;
|
16090 | 16090 | size_t margin = size;
|
16091 |
| - for(; margin > 0 && leading_byte < 4; margin--) { |
| 16091 | + for(; margin > 0 && leading_byte < 8; margin--) { |
16092 | 16092 | leading_byte += (int8_t(in[margin-1]) > -65);
|
16093 | 16093 | }
|
16094 | 16094 | // If the input is long enough, then we have that margin-1 is the fourth last leading byte.
|
@@ -16158,11 +16158,11 @@ using namespace simd;
|
16158 | 16158 | // 8 bytes when calling convert_masked_utf8_to_utf32. If you skip the last 16 bytes,
|
16159 | 16159 | // and if the data is valid, then it is entirely safe because 16 UTF-8 bytes generate
|
16160 | 16160 | // much more than 8 bytes. However, you cannot generally assume that you have valid
|
16161 |
| - // UTF-8 input, so we are going to go back from the end counting 4 leading bytes, |
| 16161 | + // UTF-8 input, so we are going to go back from the end counting 8 leading bytes, |
16162 | 16162 | // to give us a good margin.
|
16163 | 16163 | size_t leading_byte = 0;
|
16164 | 16164 | size_t margin = size;
|
16165 |
| - for(; margin > 0 && leading_byte < 4; margin--) { |
| 16165 | + for(; margin > 0 && leading_byte < 8; margin--) { |
16166 | 16166 | leading_byte += (int8_t(in[margin-1]) > -65);
|
16167 | 16167 | }
|
16168 | 16168 | // If the input is long enough, then we have that margin-1 is the fourth last leading byte.
|
@@ -17930,7 +17930,7 @@ simdutf_really_inline bool process_block_utf8_to_utf16(const char *&in, char16_t
|
17930 | 17930 | __m512i indexofthirdlastbytes = _mm512_add_epi16(mask_ffffffff,
|
17931 | 17931 | indexofsecondlastbytes); // indices of the second last bytes
|
17932 | 17932 | __m512i thirdlastbyte = _mm512_maskz_mov_epi8(m34,
|
17933 |
| - clearedbytes); // only those that are the third last byte of a sequece |
| 17933 | + clearedbytes); // only those that are the third last byte of a sequence |
17934 | 17934 | __m512i thirdlastbytes = _mm512_maskz_permutexvar_epi8(0x5555555555555555, indexofthirdlastbytes,
|
17935 | 17935 | thirdlastbyte); // the third last bytes (of three byte sequences, hi
|
17936 | 17936 | // surrogate)
|
@@ -17992,7 +17992,7 @@ simdutf_really_inline bool process_block_utf8_to_utf16(const char *&in, char16_t
|
17992 | 17992 | __m512i indexofthirdlastbytes = _mm512_add_epi16(mask_ffffffff,
|
17993 | 17993 | indexofsecondlastbytes); // indices of the second last bytes
|
17994 | 17994 | __m512i thirdlastbyte = _mm512_maskz_mov_epi8(m34,
|
17995 |
| - clearedbytes); // only those that are the third last byte of a sequece |
| 17995 | + clearedbytes); // only those that are the third last byte of a sequence |
17996 | 17996 | __m512i thirdlastbytes = _mm512_maskz_permutexvar_epi8(0x5555555555555555, indexofthirdlastbytes,
|
17997 | 17997 | thirdlastbyte); // the third last bytes (of three byte sequences, hi
|
17998 | 17998 | // surrogate)
|
@@ -18048,7 +18048,7 @@ simdutf_really_inline bool process_block_utf8_to_utf16(const char *&in, char16_t
|
18048 | 18048 | }
|
18049 | 18049 | // Fast path 2: all ASCII or 2 byte
|
18050 | 18050 | __mmask64 continuation_or_ascii = (tail == SIMDUTF_FULL) ? _knot_mask64(m234) : _kand_mask64(_knot_mask64(m234), b);
|
18051 |
| - // on top of -0xc0 we substract -2 which we get back later of the |
| 18051 | + // on top of -0xc0 we subtract -2 which we get back later of the |
18052 | 18052 | // continuation byte tags
|
18053 | 18053 | __m512i leading2byte = _mm512_maskz_sub_epi8(m234, input, mask_c2c2c2c2);
|
18054 | 18054 | __mmask64 leading = tail == (tail == SIMDUTF_FULL) ? _kor_mask64(m1, m234) : _kand_mask64(_kor_mask64(m1, m234), b); // first bytes of each sequence
|
@@ -18296,7 +18296,7 @@ __m512i rotate_by_N_epi8(const __m512i input) {
|
18296 | 18296 | stored at separate 32-bit lanes.
|
18297 | 18297 |
|
18298 | 18298 | For each lane we have also a character class (`char_class), given in form
|
18299 |
| - 0x8080800N, where N is 4 higest bits from the leading byte; 0x80 resets |
| 18299 | + 0x8080800N, where N is 4 highest bits from the leading byte; 0x80 resets |
18300 | 18300 | corresponding bytes during pshufb.
|
18301 | 18301 | */
|
18302 | 18302 | simdutf_really_inline __m512i expanded_utf8_to_utf32(__m512i char_class, __m512i utf8) {
|
@@ -19214,7 +19214,7 @@ simdutf_really_inline size_t process_block_from_utf8_to_latin1(const char *buf,
|
19214 | 19214 | // _mm512_storeu_si512((__m512i *)latin_output, output); I tried using
|
19215 | 19215 | // _mm512_storeu_si512 and have the next process_block start from the
|
19216 | 19216 | // "written_out" point but the compiler shuffles memory in such a way that it
|
19217 |
| - // is signifcantly slower... |
| 19217 | + // is significantly slower... |
19218 | 19218 | // ****************************
|
19219 | 19219 | _mm512_mask_storeu_epi8((__m512i *)latin_output, store_mask, output);
|
19220 | 19220 |
|
@@ -22028,10 +22028,9 @@ simdutf_unused simdutf_really_inline simd8<bool> must_be_continuation(const simd
|
22028 | 22028 | }
|
22029 | 22029 |
|
22030 | 22030 | simdutf_really_inline simd8<bool> must_be_2_3_continuation(const simd8<uint8_t> prev2, const simd8<uint8_t> prev3) {
|
22031 |
| - simd8<uint8_t> is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0 |
22032 |
| - simd8<uint8_t> is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0 |
22033 |
| - // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine. |
22034 |
| - return simd8<int8_t>(is_third_byte | is_fourth_byte) > int8_t(0); |
| 22031 | + simd8<uint8_t> is_third_byte = prev2.saturating_sub(0xe0u-0x80); // Only 111_____ will be > 0x80 |
| 22032 | + simd8<uint8_t> is_fourth_byte = prev3.saturating_sub(0xf0u-0x80); // Only 1111____ will be > 0x80 |
| 22033 | + return simd8<bool>(is_third_byte | is_fourth_byte); |
22035 | 22034 | }
|
22036 | 22035 |
|
22037 | 22036 | /* begin file src/haswell/avx2_detect_encodings.cpp */
|
@@ -25495,11 +25494,11 @@ using namespace simd;
|
25495 | 25494 | // 8 bytes when calling convert_masked_utf8_to_utf32. If you skip the last 16 bytes,
|
25496 | 25495 | // and if the data is valid, then it is entirely safe because 16 UTF-8 bytes generate
|
25497 | 25496 | // much more than 8 bytes. However, you cannot generally assume that you have valid
|
25498 |
| - // UTF-8 input, so we are going to go back from the end counting 4 leading bytes, |
| 25497 | + // UTF-8 input, so we are going to go back from the end counting 8 leading bytes, |
25499 | 25498 | // to give us a good margin.
|
25500 | 25499 | size_t leading_byte = 0;
|
25501 | 25500 | size_t margin = size;
|
25502 |
| - for(; margin > 0 && leading_byte < 4; margin--) { |
| 25501 | + for(; margin > 0 && leading_byte < 8; margin--) { |
25503 | 25502 | leading_byte += (int8_t(in[margin-1]) > -65);
|
25504 | 25503 | }
|
25505 | 25504 | // If the input is long enough, then we have that margin-1 is the fourth last leading byte.
|
@@ -25569,11 +25568,11 @@ using namespace simd;
|
25569 | 25568 | // 8 bytes when calling convert_masked_utf8_to_utf32. If you skip the last 16 bytes,
|
25570 | 25569 | // and if the data is valid, then it is entirely safe because 16 UTF-8 bytes generate
|
25571 | 25570 | // much more than 8 bytes. However, you cannot generally assume that you have valid
|
25572 |
| - // UTF-8 input, so we are going to go back from the end counting 4 leading bytes, |
| 25571 | + // UTF-8 input, so we are going to go back from the end counting 8 leading bytes, |
25573 | 25572 | // to give us a good margin.
|
25574 | 25573 | size_t leading_byte = 0;
|
25575 | 25574 | size_t margin = size;
|
25576 |
| - for(; margin > 0 && leading_byte < 4; margin--) { |
| 25575 | + for(; margin > 0 && leading_byte < 8; margin--) { |
25577 | 25576 | leading_byte += (int8_t(in[margin-1]) > -65);
|
25578 | 25577 | }
|
25579 | 25578 | // If the input is long enough, then we have that margin-1 is the fourth last leading byte.
|
@@ -26887,10 +26886,10 @@ simdutf_unused simdutf_really_inline simd8<bool> must_be_continuation(const simd
|
26887 | 26886 | }
|
26888 | 26887 |
|
26889 | 26888 | simdutf_really_inline simd8<bool> must_be_2_3_continuation(const simd8<uint8_t> prev2, const simd8<uint8_t> prev3) {
|
26890 |
| - simd8<uint8_t> is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0 |
26891 |
| - simd8<uint8_t> is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0 |
| 26889 | + simd8<uint8_t> is_third_byte = prev2.saturating_sub(0xe0u-0x80); // Only 111_____ will be >= 0x80 |
| 26890 | + simd8<uint8_t> is_fourth_byte = prev3.saturating_sub(0xf0u-0x80); // Only 1111____ will be >= 0x80 |
26892 | 26891 | // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine.
|
26893 |
| - return simd8<int8_t>(is_third_byte | is_fourth_byte) > int8_t(0); |
| 26892 | + return simd8<bool>(is_third_byte | is_fourth_byte); |
26894 | 26893 | }
|
26895 | 26894 |
|
26896 | 26895 | } // unnamed namespace
|
@@ -27867,11 +27866,11 @@ using namespace simd;
|
27867 | 27866 | // 8 bytes when calling convert_masked_utf8_to_utf32. If you skip the last 16 bytes,
|
27868 | 27867 | // and if the data is valid, then it is entirely safe because 16 UTF-8 bytes generate
|
27869 | 27868 | // much more than 8 bytes. However, you cannot generally assume that you have valid
|
27870 |
| - // UTF-8 input, so we are going to go back from the end counting 4 leading bytes, |
| 27869 | + // UTF-8 input, so we are going to go back from the end counting 8 leading bytes, |
27871 | 27870 | // to give us a good margin.
|
27872 | 27871 | size_t leading_byte = 0;
|
27873 | 27872 | size_t margin = size;
|
27874 |
| - for(; margin > 0 && leading_byte < 4; margin--) { |
| 27873 | + for(; margin > 0 && leading_byte < 8; margin--) { |
27875 | 27874 | leading_byte += (int8_t(in[margin-1]) > -65);
|
27876 | 27875 | }
|
27877 | 27876 | // If the input is long enough, then we have that margin-1 is the fourth last leading byte.
|
@@ -27941,11 +27940,11 @@ using namespace simd;
|
27941 | 27940 | // 8 bytes when calling convert_masked_utf8_to_utf32. If you skip the last 16 bytes,
|
27942 | 27941 | // and if the data is valid, then it is entirely safe because 16 UTF-8 bytes generate
|
27943 | 27942 | // much more than 8 bytes. However, you cannot generally assume that you have valid
|
27944 |
| - // UTF-8 input, so we are going to go back from the end counting 4 leading bytes, |
| 27943 | + // UTF-8 input, so we are going to go back from the end counting 8 leading bytes, |
27945 | 27944 | // to give us a good margin.
|
27946 | 27945 | size_t leading_byte = 0;
|
27947 | 27946 | size_t margin = size;
|
27948 |
| - for(; margin > 0 && leading_byte < 4; margin--) { |
| 27947 | + for(; margin > 0 && leading_byte < 8; margin--) { |
27949 | 27948 | leading_byte += (int8_t(in[margin-1]) > -65);
|
27950 | 27949 | }
|
27951 | 27950 | // If the input is long enough, then we have that margin-1 is the fourth last leading byte.
|
@@ -28407,10 +28406,9 @@ simdutf_unused simdutf_really_inline simd8<bool> must_be_continuation(const simd
|
28407 | 28406 | }
|
28408 | 28407 |
|
28409 | 28408 | simdutf_really_inline simd8<bool> must_be_2_3_continuation(const simd8<uint8_t> prev2, const simd8<uint8_t> prev3) {
|
28410 |
| - simd8<uint8_t> is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0 |
28411 |
| - simd8<uint8_t> is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0 |
28412 |
| - // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine. |
28413 |
| - return simd8<int8_t>(is_third_byte | is_fourth_byte) > int8_t(0); |
| 28409 | + simd8<uint8_t> is_third_byte = prev2.saturating_sub(0xe0u-0x80); // Only 111_____ will be >= 0x80 |
| 28410 | + simd8<uint8_t> is_fourth_byte = prev3.saturating_sub(0xf0u-0x80); // Only 1111____ will be >= 0x80 |
| 28411 | + return simd8<bool>(is_third_byte | is_fourth_byte); |
28414 | 28412 | }
|
28415 | 28413 |
|
28416 | 28414 | /* begin file src/westmere/internal/loader.cpp */
|
@@ -31874,11 +31872,11 @@ using namespace simd;
|
31874 | 31872 | // 8 bytes when calling convert_masked_utf8_to_utf32. If you skip the last 16 bytes,
|
31875 | 31873 | // and if the data is valid, then it is entirely safe because 16 UTF-8 bytes generate
|
31876 | 31874 | // much more than 8 bytes. However, you cannot generally assume that you have valid
|
31877 |
| - // UTF-8 input, so we are going to go back from the end counting 4 leading bytes, |
| 31875 | + // UTF-8 input, so we are going to go back from the end counting 8 leading bytes, |
31878 | 31876 | // to give us a good margin.
|
31879 | 31877 | size_t leading_byte = 0;
|
31880 | 31878 | size_t margin = size;
|
31881 |
| - for(; margin > 0 && leading_byte < 4; margin--) { |
| 31879 | + for(; margin > 0 && leading_byte < 8; margin--) { |
31882 | 31880 | leading_byte += (int8_t(in[margin-1]) > -65);
|
31883 | 31881 | }
|
31884 | 31882 | // If the input is long enough, then we have that margin-1 is the fourth last leading byte.
|
@@ -31948,11 +31946,11 @@ using namespace simd;
|
31948 | 31946 | // 8 bytes when calling convert_masked_utf8_to_utf32. If you skip the last 16 bytes,
|
31949 | 31947 | // and if the data is valid, then it is entirely safe because 16 UTF-8 bytes generate
|
31950 | 31948 | // much more than 8 bytes. However, you cannot generally assume that you have valid
|
31951 |
| - // UTF-8 input, so we are going to go back from the end counting 4 leading bytes, |
| 31949 | + // UTF-8 input, so we are going to go back from the end counting 8 leading bytes, |
31952 | 31950 | // to give us a good margin.
|
31953 | 31951 | size_t leading_byte = 0;
|
31954 | 31952 | size_t margin = size;
|
31955 |
| - for(; margin > 0 && leading_byte < 4; margin--) { |
| 31953 | + for(; margin > 0 && leading_byte < 8; margin--) { |
31956 | 31954 | leading_byte += (int8_t(in[margin-1]) > -65);
|
31957 | 31955 | }
|
31958 | 31956 | // If the input is long enough, then we have that margin-1 is the fourth last leading byte.
|
|
0 commit comments