ayo/kitty
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Code to easily compare SIMD and scalar decode in a live instance

Browse source 
Also remove -mtune=intel as it fails with clang
This commit is contained in:
Kovid Goyal 2024-02-02 11:35:51 +05:30
parent 561712090d
commit f1fe0bf40a
No known key found for this signature in database
GPG key ID: 06BC317B515ACE7C
4 changed files with 26 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
kitty/simd-string-impl.h
View file

@ -423,6 +423,11 @@ scalar_decode_all(UTF8Decoder *d, const uint8_t *src, size_t src_sz) {
bool
FUNC(utf8_decode_to_esc)(UTF8Decoder *d, const uint8_t *src_data, size_t src_len) {
// Based on the algorithm described in: https://woboq.com/blog/utf-8-processing-using-simd.html
#ifdef compare_with_scalar
UTF8Decoder debugdec ={0};
memcpy(&debugdec.state, &d->state, sizeof(debugdec.state));
bool scalar_sentinel_found = utf8_decode_to_esc_scalar(&debugdec, src_data, src_len);
#endif
d->output.pos = 0; d->num_consumed = 0;
if (d->state.cur != UTF8_ACCEPT) {
// Finish the trailing sequence only
@ -618,6 +623,25 @@ start_classification:
FUNC(output_unicode)(d, output1, output2, output3, num_codepoints);
handle_trailing_bytes();
}
#ifdef compare_with_scalar
if (debugdec.output.pos != d->output.pos || debugdec.num_consumed != d->num_consumed ||
memcmp(d->output.storage, debugdec.output.storage, d->output.pos * sizeof(d->output.storage[0])) != 0 ||
sentinel_found != scalar_sentinel_found || debugdec.state.cur != d->state.cur
) {
fprintf(stderr, "vector decode output differs from scalar: input_sz=%zu consumed=(%u %u) output_sz=(%u %u) sentinel=(%d %d) state_changed: %d output_different: %d\n",
src_len, debugdec.num_consumed, d->num_consumed, debugdec.output.pos, d->output.pos, scalar_sentinel_found, sentinel_found,
debugdec.state.cur != d->state.cur,
memcmp(d->output.storage, debugdec.output.storage, MIN(d->output.pos, debugdec.output.pos) * sizeof(d->output.storage[0]))
);
fprintf(stderr, "\"");
for (unsigned i = 0; i < src_len; i++) {
if (32 <= src_data[i] && src_data[i] < 0x7f && src_data[i] != '"') fprintf(stderr, "%c", src_data[i]);
else fprintf(stderr, "\\x%x", src_data[i]);
}
fprintf(stderr, "\"\n");
}
utf8_decoder_free(&debugdec);
#endif
zero_upper();
return sentinel_found;
#undef abort_with_invalid_utf8

2
kitty/simd-string.c
View file

@ -29,7 +29,7 @@ find_either_of_two_bytes(const uint8_t *haystack, const size_t sz, const uint8_t
// UTF-8 {{{
static bool
bool
utf8_decode_to_esc_scalar(UTF8Decoder *d, const uint8_t *src, const size_t src_sz) {
d->output.pos = 0; d->num_consumed = 0;
utf8_decoder_ensure_capacity(d, src_sz);

1
kitty/simd-string.h
View file

@ -22,6 +22,7 @@ typedef struct UTF8Decoder {
static inline void utf8_decoder_reset(UTF8Decoder *self) { zero_at_ptr(&self->state); }
bool utf8_decode_to_esc(UTF8Decoder *d, const uint8_t *src, size_t src_sz);
bool utf8_decode_to_esc_scalar(UTF8Decoder *d, const uint8_t *src, const size_t src_sz);
static inline void utf8_decoder_ensure_capacity(UTF8Decoder *d, unsigned sz) {
if (d->output.pos + sz > d->output.capacity) {

4
setup.py
View file

@ -712,15 +712,11 @@ def get_source_specific_cflags(env: Env, src: str) -> List[str]:
if src in ('kitty/simd-string-128.c', 'kitty/simd-string-256.c'):
if env.binary_arch.isa in (ISA.AMD64, ISA.X86):
ans.append('-msse4.2' if '128' in src else '-mavx2')
if not env.native_optimizations:
ans.append('-mtune=intel')
elif env.binary_arch.isa != ISA.ARM64:
ans.append('-DKITTY_NO_SIMD')
elif src.startswith('3rdparty/base64/lib/arch/'):
if env.binary_arch.isa in (ISA.AMD64, ISA.X86):
q = src.split(os.path.sep)
if not env.native_optimizations:
ans.append('-mtune=intel')
if 'sse3' in q:
ans.append('-msse3')
elif 'sse41' in q: