Speedup unicode character property lookup
Use libunistring instead of building predicates from the unicode database
This commit is contained in:
Kovid Goyal
parent
63882e1fdc
commit
9bea1001f9
5 changed files with 16 additions and 64 deletions
|
|
@ -80,6 +80,7 @@ the following dependencies are installed first.
|
|||
|
||||
* python >= 3.5
|
||||
* glfw >= 3.2
|
||||
* libunistring
|
||||
* glew >= 2.0 (not needed on macOS)
|
||||
* fontconfig (not needed on macOS)
|
||||
* xrdb and xsel (only on X11 based systems)
|
||||
|
|
|
|||
|
|
@ -1,2 +1,2 @@
|
|||
#!/bin/bash
|
||||
cloc --exclude-list-file <(echo -e 'kitty/wcwidth9.h\nkitty/unicode-data.h\nkitty/gl.c\nkitty/glfw.c\nkitty/glfw.h\nkitty/charsets.c\nkitty/key_encoding.py\nkitty/rgb.py') kitty
|
||||
cloc --exclude-list-file <(echo -e 'kitty/wcwidth9.h\nkitty/gl.c\nkitty/glfw.c\nkitty/glfw.h\nkitty/charsets.c\nkitty/key_encoding.py\nkitty/rgb.py') kitty
|
||||
|
|
|
|||
|
|
@ -1,57 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# vim:fileencoding=utf-8
|
||||
# License: GPL v3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
|
||||
|
||||
import unicodedata
|
||||
import itertools
|
||||
import sys
|
||||
|
||||
IGNORED_CATEGORIES = ('Cc', 'Cf', 'Cs')
|
||||
|
||||
|
||||
def ranges(i):
|
||||
for a, b in itertools.groupby(enumerate(i), lambda r: r[1] - r[0]):
|
||||
b = list(b)
|
||||
yield b[0][1], b[-1][1]
|
||||
|
||||
|
||||
def generate_data(chars):
|
||||
points, cranges = [], []
|
||||
for l, r in ranges(chars):
|
||||
if l == r:
|
||||
points.append(l)
|
||||
else:
|
||||
cranges.append((l, r))
|
||||
return points, cranges
|
||||
|
||||
|
||||
def generate_predicate(name, chars):
|
||||
points, cranges = generate_data(chars)
|
||||
cranges = ['(0x%x %s ch && ch <= 0x%x)' % (l, '<' if l == 0 else '<=', r) for l, r in cranges]
|
||||
points = ['(ch == 0x%x)' % p for p in points]
|
||||
return '''
|
||||
static inline bool %s(uint32_t ch) {
|
||||
return %s || %s;
|
||||
}
|
||||
''' % (name, '||'.join(cranges), '||'.join(points))
|
||||
|
||||
|
||||
def main():
|
||||
combining_chars = []
|
||||
igchars = []
|
||||
for c in map(chr, range(sys.maxunicode + 1)):
|
||||
if unicodedata.category(c) in IGNORED_CATEGORIES:
|
||||
igchars.append(ord(c))
|
||||
if unicodedata.combining(c):
|
||||
combining_chars.append(ord(c))
|
||||
|
||||
cc = generate_predicate('is_combining_char', combining_chars)
|
||||
ig = generate_predicate('is_ignored_char', igchars)
|
||||
with open('kitty/unicode-data.h', 'w') as f:
|
||||
print('#pragma once', file=f)
|
||||
print(cc, file=f)
|
||||
print(ig, file=f)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
@ -1,11 +1,19 @@
|
|||
#pragma once
|
||||
|
||||
static inline bool is_combining_char(uint32_t ch) {
|
||||
return (0x300 <= ch && ch <= 0x34e)||(0x350 <= ch && ch <= 0x36f)||(0x483 <= ch && ch <= 0x487)||(0x591 <= ch && ch <= 0x5bd)||(0x5c1 <= ch && ch <= 0x5c2)||(0x5c4 <= ch && ch <= 0x5c5)||(0x610 <= ch && ch <= 0x61a)||(0x64b <= ch && ch <= 0x65f)||(0x6d6 <= ch && ch <= 0x6dc)||(0x6df <= ch && ch <= 0x6e4)||(0x6e7 <= ch && ch <= 0x6e8)||(0x6ea <= ch && ch <= 0x6ed)||(0x730 <= ch && ch <= 0x74a)||(0x7eb <= ch && ch <= 0x7f3)||(0x816 <= ch && ch <= 0x819)||(0x81b <= ch && ch <= 0x823)||(0x825 <= ch && ch <= 0x827)||(0x829 <= ch && ch <= 0x82d)||(0x859 <= ch && ch <= 0x85b)||(0x8e3 <= ch && ch <= 0x8ff)||(0x951 <= ch && ch <= 0x954)||(0xc55 <= ch && ch <= 0xc56)||(0xe38 <= ch && ch <= 0xe3a)||(0xe48 <= ch && ch <= 0xe4b)||(0xeb8 <= ch && ch <= 0xeb9)||(0xec8 <= ch && ch <= 0xecb)||(0xf18 <= ch && ch <= 0xf19)||(0xf71 <= ch && ch <= 0xf72)||(0xf7a <= ch && ch <= 0xf7d)||(0xf82 <= ch && ch <= 0xf84)||(0xf86 <= ch && ch <= 0xf87)||(0x1039 <= ch && ch <= 0x103a)||(0x135d <= ch && ch <= 0x135f)||(0x1939 <= ch && ch <= 0x193b)||(0x1a17 <= ch && ch <= 0x1a18)||(0x1a75 <= ch && ch <= 0x1a7c)||(0x1ab0 <= ch && ch <= 0x1abd)||(0x1b6b <= ch && ch <= 0x1b73)||(0x1baa <= ch && ch <= 0x1bab)||(0x1bf2 <= ch && ch <= 0x1bf3)||(0x1cd0 <= ch && ch <= 0x1cd2)||(0x1cd4 <= ch && ch <= 0x1ce0)||(0x1ce2 <= ch && ch <= 0x1ce8)||(0x1cf8 <= ch && ch <= 0x1cf9)||(0x1dc0 <= ch && ch <= 0x1df5)||(0x1dfc <= ch && ch <= 0x1dff)||(0x20d0 <= ch && ch <= 0x20dc)||(0x20e5 <= ch && ch <= 0x20f0)||(0x2cef <= ch && ch <= 0x2cf1)||(0x2de0 <= ch && ch <= 0x2dff)||(0x302a <= ch && ch <= 0x302f)||(0x3099 <= ch && ch <= 0x309a)||(0xa674 <= ch && ch <= 0xa67d)||(0xa69e <= ch && ch <= 0xa69f)||(0xa6f0 <= ch && ch <= 0xa6f1)||(0xa8e0 <= ch && ch <= 0xa8f1)||(0xa92b <= ch && ch <= 0xa92d)||(0xaab2 <= ch && ch <= 0xaab4)||(0xaab7 <= ch && ch <= 0xaab8)||(0xaabe <= ch && ch <= 0xaabf)||(0xfe20 <= ch && ch <= 0xfe2f)||(0x10376 <= ch && ch <= 0x1037a)||(0x10a38 <= ch && ch <= 0x10a3a)||(0x10ae5 <= ch && ch <= 0x10ae6)||(0x110b9 <= ch && ch <= 0x110ba)||(0x11100 <= ch && ch <= 0x11102)||(0x11133 <= ch && ch <= 0x11134)||(0x11235 <= ch && ch <= 0x11236)||(0x112e9 <= ch && ch <= 0x112ea)||(0x11366 <= ch && ch <= 0x1136c)||(0x11370 <= ch && ch <= 0x11374)||(0x114c2 <= ch && ch <= 0x114c3)||(0x115bf <= ch && ch <= 0x115c0)||(0x116b6 <= ch && ch <= 0x116b7)||(0x16af0 <= ch && ch <= 0x16af4)||(0x16b30 <= ch && ch <= 0x16b36)||(0x1d165 <= ch && ch <= 0x1d169)||(0x1d16d <= ch && ch <= 0x1d172)||(0x1d17b <= ch && ch <= 0x1d182)||(0x1d185 <= ch && ch <= 0x1d18b)||(0x1d1aa <= ch && ch <= 0x1d1ad)||(0x1d242 <= ch && ch <= 0x1d244)||(0x1e8d0 <= ch && ch <= 0x1e8d6) || (ch == 0x5bf)||(ch == 0x5c7)||(ch == 0x670)||(ch == 0x711)||(ch == 0x93c)||(ch == 0x94d)||(ch == 0x9bc)||(ch == 0x9cd)||(ch == 0xa3c)||(ch == 0xa4d)||(ch == 0xabc)||(ch == 0xacd)||(ch == 0xb3c)||(ch == 0xb4d)||(ch == 0xbcd)||(ch == 0xc4d)||(ch == 0xcbc)||(ch == 0xccd)||(ch == 0xd4d)||(ch == 0xdca)||(ch == 0xf35)||(ch == 0xf37)||(ch == 0xf39)||(ch == 0xf74)||(ch == 0xf80)||(ch == 0xfc6)||(ch == 0x1037)||(ch == 0x108d)||(ch == 0x1714)||(ch == 0x1734)||(ch == 0x17d2)||(ch == 0x17dd)||(ch == 0x18a9)||(ch == 0x1a60)||(ch == 0x1a7f)||(ch == 0x1b34)||(ch == 0x1b44)||(ch == 0x1be6)||(ch == 0x1c37)||(ch == 0x1ced)||(ch == 0x1cf4)||(ch == 0x20e1)||(ch == 0x2d7f)||(ch == 0xa66f)||(ch == 0xa806)||(ch == 0xa8c4)||(ch == 0xa953)||(ch == 0xa9b3)||(ch == 0xa9c0)||(ch == 0xaab0)||(ch == 0xaac1)||(ch == 0xaaf6)||(ch == 0xabed)||(ch == 0xfb1e)||(ch == 0x101fd)||(ch == 0x102e0)||(ch == 0x10a0d)||(ch == 0x10a0f)||(ch == 0x10a3f)||(ch == 0x11046)||(ch == 0x1107f)||(ch == 0x11173)||(ch == 0x111c0)||(ch == 0x111ca)||(ch == 0x1133c)||(ch == 0x1134d)||(ch == 0x1163f)||(ch == 0x1172b)||(ch == 0x1bc9e);
|
||||
#include <unictype.h>
|
||||
|
||||
static inline bool
|
||||
is_combining_char(uint32_t ch) {
|
||||
return uc_combining_class(ch) != UC_CCC_NR;
|
||||
}
|
||||
|
||||
|
||||
static inline bool is_ignored_char(uint32_t ch) {
|
||||
return (0x0 < ch && ch <= 0x1f)||(0x7f <= ch && ch <= 0x9f)||(0x600 <= ch && ch <= 0x605)||(0x200b <= ch && ch <= 0x200f)||(0x202a <= ch && ch <= 0x202e)||(0x2060 <= ch && ch <= 0x2064)||(0x2066 <= ch && ch <= 0x206f)||(0xd800 <= ch && ch <= 0xdfff)||(0xfff9 <= ch && ch <= 0xfffb)||(0x1bca0 <= ch && ch <= 0x1bca3)||(0x1d173 <= ch && ch <= 0x1d17a)||(0xe0020 <= ch && ch <= 0xe007f) || (ch == 0xad)||(ch == 0x61c)||(ch == 0x6dd)||(ch == 0x70f)||(ch == 0x180e)||(ch == 0xfeff)||(ch == 0x110bd)||(ch == 0xe0001);
|
||||
static inline bool
|
||||
is_ignored_char(uint32_t ch) {
|
||||
return uc_is_general_category_withtable(ch, UC_CATEGORY_MASK_Cc | UC_CATEGORY_MASK_Cf | UC_CATEGORY_MASK_Cs);
|
||||
}
|
||||
|
||||
static inline bool
|
||||
is_word_char(uint32_t ch) {
|
||||
return uc_is_general_category_withtable(ch, UC_CATEGORY_MASK_L | UC_CATEGORY_MASK_N);
|
||||
}
|
||||
|
||||
|
|
|
|||
2
setup.py
2
setup.py
|
|
@ -170,7 +170,7 @@ def init_env(debug=False, sanitize=False, native_optimizations=True, profile=Fal
|
|||
else:
|
||||
glfw_ldflags = pkg_config('glfw3', '--libs')
|
||||
glew_libs = pkg_config('glew', '--libs')
|
||||
ldpaths = pylib + glew_libs + font_libs + glfw_ldflags
|
||||
ldpaths = pylib + glew_libs + font_libs + glfw_ldflags + ['-lunistring']
|
||||
|
||||
try:
|
||||
os.mkdir(build_dir)
|
||||
|
|
|
|||
Loading…
Reference in a new issue