ayo/kitty
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Use the new SIMD utf-8 decoder

Browse source
This commit is contained in:
Kovid Goyal 2024-01-14 14:12:19 +05:30
parent e783eccc97
commit b3ca5d51fb
No known key found for this signature in database
GPG key ID: 06BC317B515ACE7C
2 changed files with 7 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
kitty/simd-string.c
View file

@ -139,14 +139,14 @@ init_simd(void *x) {
if (has_avx2) {
A(has_avx2, True);
find_either_of_two_bytes_impl = find_either_of_two_bytes_256;
/* utf8_decode_to_esc_impl = utf8_decode_to_esc_256; */
utf8_decode_to_esc_impl = utf8_decode_to_esc_256;
} else {
A(has_avx2, False);
}
if (has_sse4_2) {
A(has_sse4_2, True);
if (find_either_of_two_bytes_impl == find_either_of_two_bytes_scalar) find_either_of_two_bytes_impl = find_either_of_two_bytes_128;
/* if (utf8_decode_to_esc_impl == utf8_decode_to_esc_scalar) utf8_decode_to_esc_impl = utf8_decode_to_esc_128; */
if (utf8_decode_to_esc_impl == utf8_decode_to_esc_scalar) utf8_decode_to_esc_impl = utf8_decode_to_esc_128;
} else {
A(has_sse4_2, False);
}

10
kitty_tests/parser.py
View file

@ -220,8 +220,6 @@ def double_test(x):
x('abcd1234efgh5678')
x('abc\x1bd1234efgh5678')
x('abcd1234efgh5678ijklABCDmnopEFGH')
x('ニチ')
x('\x84\x85')
for which in (2, 3):
x = partial(t, which=which)
@ -235,15 +233,17 @@ def double_test(x):
x(b'abcd\xe2', b'\x89\xa41234')
x(b'abcd\xe2\x89', b'\xa41234')
def test_invalid(src, expected, which=2):
def test_expected(src, expected, which=2):
reset_state()
_, actual = test_utf8_decode_to_sentinel(b'filler' + asbytes(src), which)
expected = 'filler' + expected
self.ae(expected, actual, f'Failed for: {src!r} with {which=}')
# various invalid input
for which in (1, 2, 3):
pb = partial(test_invalid, which=which)
pb = partial(test_expected, which=which)
pb('ニチ', 'ニチ')
pb('\x84\x85', '\x84\x85')
# various invalid input
pb(b'abcd\xf51234', 'abcd\ufffd1234') # bytes > 0xf4
pb(b'abcd\xff1234', 'abcd\ufffd1234') # bytes > 0xf4
pb(b'"\xbf"', '"\ufffd"')