Use a multi-stage lookup table for grapheme segmentation
This commit is contained in:
Kovid Goyal
parent
72a1aabafd
commit
66856e7b52
9 changed files with 1873 additions and 474 deletions
236
gen/wcwidth.py
236
gen/wcwidth.py
|
|
@ -278,6 +278,7 @@ def parse_eaw() -> None:
|
|||
|
||||
def parse_grapheme_segmentation() -> None:
|
||||
global extended_pictographic, grapheme_break_as_int, incb_as_int, int_as_grapheme_break, int_as_incb
|
||||
global seg_props_from_int, seg_props_as_int
|
||||
grapheme_segmentation_maps['AtStart'] # this is used by the segmentation algorithm, no character has it
|
||||
grapheme_segmentation_maps['None'] # this is used by the segmentation algorithm, no character has it
|
||||
for line in get_data('ucd/auxiliary/GraphemeBreakProperty.txt'):
|
||||
|
|
@ -302,6 +303,8 @@ def parse_grapheme_segmentation() -> None:
|
|||
chars, category = split_two(line)
|
||||
if 'Extended_Pictographic#' == category:
|
||||
extended_pictographic |= chars
|
||||
seg_props_from_int = {'grapheme_break': int_as_grapheme_break, 'indic_conjunct_break': int_as_incb}
|
||||
seg_props_as_int = {'grapheme_break': grapheme_break_as_int, 'indic_conjunct_break': incb_as_int}
|
||||
|
||||
|
||||
class GraphemeSegmentationTest(TypedDict):
|
||||
|
|
@ -524,7 +527,7 @@ def gen_multistage_table(
|
|||
c(f'static const {ctype_t2} {name}_t2[{len(t2)}] = ''{')
|
||||
c(f'\t{", ".join(map(str, t2))}')
|
||||
c('};')
|
||||
items = '\n\t'.join(x.as_c + ',' for x in t3)
|
||||
items = '\n\t'.join(x.as_c + f', // {i}' for i, x in enumerate(t3))
|
||||
c(f'static const {name} {name}_t3[{len(t3)}] = ''{')
|
||||
c(f'\t{items}')
|
||||
c('};')
|
||||
|
|
@ -538,7 +541,7 @@ def gen_multistage_table(
|
|||
g(f'var {lname}_t2 = [{len(t2)}]{gotype_t2}''{')
|
||||
g(f'\t{", ".join(map(str, t2))},')
|
||||
g('}')
|
||||
items = '\n\t'.join(x.as_go + ',' for x in t3)
|
||||
items = '\n\t'.join(x.as_go + f', // {i}' for i, x in enumerate(t3))
|
||||
g(f'var {lname}_t3 = [{len(t3)}]{name}''{')
|
||||
g(f'\t{items}')
|
||||
g('}')
|
||||
|
|
@ -566,7 +569,7 @@ def clamped_bitsize(val: int) -> int:
|
|||
def bitfield_from_int(
|
||||
fields: dict[str, int], x: int, int_to_str: dict[str, tuple[str, ...]]
|
||||
) -> dict[str, str | bool]:
|
||||
# first field is most significant, last field is least significant
|
||||
# first field is least significant, last field is most significant
|
||||
args: dict[str, str | bool] = {}
|
||||
for f, shift in fields.items():
|
||||
mask = mask_for(shift)
|
||||
|
|
@ -579,26 +582,24 @@ def bitfield_from_int(
|
|||
return args
|
||||
|
||||
|
||||
def bit_field_as_int(
|
||||
field_name: str, fields: dict[str, int], for_go: bool = False, function_name_suffix: str = '_as_integer',
|
||||
) -> str:
|
||||
# first field is most significant, last field is least significant
|
||||
bsz = clamped_bitsize(sum(fields.values()))
|
||||
parts = []
|
||||
total_shift = 0
|
||||
base_type = f'uint{bsz}_t'
|
||||
for f, shift in reversed(fields.items()):
|
||||
parts.append(f'(({base_type})((x).{f}) << {shift})')
|
||||
total_shift += shift
|
||||
expr = ' | '.join(parts)
|
||||
return f'''\
|
||||
static inline {base_type} {field_name}{function_name_suffix}({field_name} x) {{
|
||||
return {expr};
|
||||
}}
|
||||
'''
|
||||
def bitfield_as_int(
|
||||
fields: dict[str, int], vals: Sequence[bool | str], str_maps: dict[str, dict[str, int]]
|
||||
) -> int:
|
||||
# first field is least significant, last field is most significant
|
||||
ans = shift = 0
|
||||
for i, (f, width) in enumerate(fields.items()):
|
||||
qval = vals[i]
|
||||
if isinstance(qval, str):
|
||||
val = str_maps[f][qval]
|
||||
else:
|
||||
val = int(qval)
|
||||
ans |= val << shift
|
||||
shift += width
|
||||
return ans
|
||||
|
||||
|
||||
seg_props_as_int = {'grapheme_break': int_as_grapheme_break, 'indic_conjunct_break': int_as_incb}
|
||||
seg_props_from_int: dict[str, tuple[str, ...]] = {}
|
||||
seg_props_as_int: dict[str, dict[str, int]] = {}
|
||||
|
||||
|
||||
class GraphemeSegmentationProps(NamedTuple):
|
||||
|
|
@ -621,14 +622,11 @@ def fields(cls) -> dict[str, int]:
|
|||
|
||||
@classmethod
|
||||
def from_int(cls, x: int) -> 'GraphemeSegmentationProps':
|
||||
args = bitfield_from_int(cls.fields(), x, seg_props_as_int)
|
||||
args = bitfield_from_int(cls.fields(), x, seg_props_from_int)
|
||||
return cls(**args) # type: ignore
|
||||
|
||||
def __int__(self) -> int:
|
||||
gbwidth = int(self._field_defaults['grapheme_break'])
|
||||
incbwidth = int(self._field_defaults['indic_conjunct_break'])
|
||||
return grapheme_break_as_int[self.grapheme_break] | incb_as_int[self.indic_conjunct_break] << gbwidth | int(
|
||||
self.is_extended_pictographic) << (gbwidth + incbwidth)
|
||||
return bitfield_as_int(self.fields(), self, seg_props_as_int)
|
||||
|
||||
|
||||
control_grapheme_breaks = 'CR', 'LF', 'Control'
|
||||
|
|
@ -636,7 +634,6 @@ def __int__(self) -> int:
|
|||
|
||||
|
||||
def bitfield_declaration_as_c(name: str, fields: dict[str, int], *alternate_fields: dict[str, int]) -> str:
|
||||
# empty in MSB, then top to bottom with bottom at LSB
|
||||
base_size = clamped_bitsize(sum(fields.values()))
|
||||
base_type = f'uint{base_size}_t'
|
||||
ans = [f'// {name}Declaration: uses {sum(fields.values())} bits {{''{{', f'typedef union {name} {{']
|
||||
|
|
@ -645,12 +642,12 @@ def struct(fields: dict[str, int]) -> Iterator[str]:
|
|||
return
|
||||
empty = base_size - sum(fields.values())
|
||||
yield ' struct __attribute__((packed)) {'
|
||||
yield '#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__'
|
||||
yield '#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__'
|
||||
for f, width in reversed(fields.items()):
|
||||
yield f' uint{clamped_bitsize(width)}_t {f} : {width};'
|
||||
if empty:
|
||||
yield f' uint{clamped_bitsize(empty)}_t : {empty};'
|
||||
yield '#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__'
|
||||
yield '#elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__'
|
||||
if empty:
|
||||
yield f' uint{clamped_bitsize(empty)}_t : {empty};'
|
||||
for f, width in fields.items():
|
||||
|
|
@ -695,6 +692,9 @@ def from_int(cls, x: int) -> 'GraphemeSegmentationState':
|
|||
args = bitfield_from_int(cls.fields(), x, {'grapheme_break': int_as_grapheme_break})
|
||||
return cls(**args) # type: ignore
|
||||
|
||||
def __int__(self) -> int:
|
||||
return bitfield_as_int(self.fields(), self, seg_props_as_int)
|
||||
|
||||
@classmethod
|
||||
def c_declaration(cls) -> str:
|
||||
return bitfield_declaration_as_c(cls.__name__, cls.fields())
|
||||
|
|
@ -764,7 +764,7 @@ def add_to_current_cell(self, p: GraphemeSegmentationProps) -> 'GraphemeSegmenta
|
|||
), add_to_cell)
|
||||
|
||||
|
||||
def split_into_graphemes(text: str, props: Sequence[GraphemeSegmentationProps]) -> Iterator[str]:
|
||||
def split_into_graphemes(props: Sequence[GraphemeSegmentationProps], text: str) -> Iterator[str]:
|
||||
s = GraphemeSegmentationState.make()
|
||||
pos = 0
|
||||
for i, ch in enumerate(text):
|
||||
|
|
@ -777,10 +777,25 @@ def split_into_graphemes(text: str, props: Sequence[GraphemeSegmentationProps])
|
|||
yield text[pos:]
|
||||
|
||||
|
||||
def test_grapheme_segmentation(props: Sequence[GraphemeSegmentationProps]) -> None:
|
||||
def split_into_graphemes_with_table(
|
||||
props: Sequence['GraphemeSegmentationProps'], table: Sequence['GraphemeSegmentationResult'], text: str,
|
||||
) -> Iterator[str]:
|
||||
s = GraphemeSegmentationResult.make()
|
||||
pos = 0
|
||||
for i, ch in enumerate(text):
|
||||
k = int(GraphemeSegmentationKey(s.new_state, props[ord(ch)]))
|
||||
s = table[k]
|
||||
if not s.add_to_current_cell:
|
||||
yield text[pos:i]
|
||||
pos = i
|
||||
if pos < len(text):
|
||||
yield text[pos:]
|
||||
|
||||
|
||||
def test_grapheme_segmentation(split_into_graphemes: Callable[[str], Iterator[str]]) -> None:
|
||||
for test in grapheme_segmentation_tests:
|
||||
expected = test['data']
|
||||
actual = tuple(split_into_graphemes(''.join(test['data']), props))
|
||||
actual = tuple(split_into_graphemes(''.join(test['data'])))
|
||||
if expected != actual:
|
||||
def as_codepoints(text: str) -> str:
|
||||
return ' '.join(hex(ord(x))[2:] for x in text)
|
||||
|
|
@ -795,39 +810,96 @@ class GraphemeSegmentationKey(NamedTuple):
|
|||
|
||||
@classmethod
|
||||
def from_int(cls, x: int) -> 'GraphemeSegmentationKey':
|
||||
shift = cls.char.used_bits()
|
||||
shift = GraphemeSegmentationProps.used_bits()
|
||||
mask = mask_for(shift)
|
||||
state = GraphemeSegmentationState.from_int(x >> shift)
|
||||
char = GraphemeSegmentationProps.from_int(x & mask)
|
||||
return GraphemeSegmentationKey(state, char)
|
||||
|
||||
def __int__(self) -> int:
|
||||
shift = GraphemeSegmentationProps.used_bits()
|
||||
return int(self.state) << shift | int(self.char)
|
||||
|
||||
def result(self) -> 'GraphemeSegmentationResult':
|
||||
return self.state.add_to_current_cell(self.char)
|
||||
|
||||
@classmethod
|
||||
def as_int(cls, for_go: bool = False) -> str:
|
||||
lines = []
|
||||
shift = cls.char.used_bits()
|
||||
base_type = f'uint{cls.state.bitsize()}_t'
|
||||
lines.append(f'static inline {base_type} {cls.__name__}(GraphemeSegmentation state, CharProps ch)' '{')
|
||||
lines.append(f'\treturn (state.val << {shift}) | ch.grapheme_segmentation_property;')
|
||||
lines.append('}')
|
||||
def code_to_convert_to_int(cls, for_go: bool = False) -> str:
|
||||
lines: list[str] = []
|
||||
a = lines.append
|
||||
shift = GraphemeSegmentationProps.used_bits()
|
||||
if for_go:
|
||||
base_type = f'uint{GraphemeSegmentationState.bitsize()}'
|
||||
a(f'func grapheme_segmentation_key(r GraphemeSegmentationResult, ch CharProps) ({base_type}) ''{')
|
||||
a(f'\treturn (r.State() << {shift}) | ch.GraphemeSegmentationProperty()')
|
||||
a('}')
|
||||
else:
|
||||
base_type = f'uint{GraphemeSegmentationState.bitsize()}_t'
|
||||
a(f'static inline {base_type} {cls.__name__}(GraphemeSegmentationResult r, CharProps ch)' '{')
|
||||
a(f'\treturn (r.state << {shift}) | ch.grapheme_segmentation_property;')
|
||||
a('}')
|
||||
return '\n'.join(lines)
|
||||
|
||||
|
||||
class GraphemeSegmentationResult(NamedTuple):
|
||||
new_state: GraphemeSegmentationState
|
||||
add_to_current_cell: bool
|
||||
new_state: GraphemeSegmentationState = GraphemeSegmentationState()
|
||||
add_to_current_cell: bool = True
|
||||
|
||||
@classmethod
|
||||
def used_bits(cls) -> int:
|
||||
return sum(int(cls.new_state._field_defaults[f]) for f in cls.new_state._fields) + 1
|
||||
return sum(int(GraphemeSegmentationState._field_defaults[f]) for f in GraphemeSegmentationState._fields) + 1
|
||||
|
||||
@classmethod
|
||||
def bitsize(cls) -> int:
|
||||
return clamped_bitsize(cls.used_bits())
|
||||
|
||||
@classmethod
|
||||
def make(cls) -> 'GraphemeSegmentationResult':
|
||||
return GraphemeSegmentationResult(GraphemeSegmentationState.make(), False)
|
||||
|
||||
@classmethod
|
||||
def go_fields(cls) -> Sequence[str]:
|
||||
ans = []
|
||||
ans.append('add_to_current_cell 1')
|
||||
for f, width in reversed(GraphemeSegmentationState.fields().items()):
|
||||
ans.append(f'{f} {width}')
|
||||
return tuple(ans)
|
||||
|
||||
@property
|
||||
def as_go(self) -> str:
|
||||
shift = 0
|
||||
parts = []
|
||||
for f in reversed(GraphemeSegmentationResult.go_fields()):
|
||||
f = f.partition(' ')[0]
|
||||
if f != 'add_to_current_cell':
|
||||
bits = int(self.new_state._field_defaults[f])
|
||||
x = getattr(self.new_state, f)
|
||||
if f == 'grapheme_break':
|
||||
x = f'GraphemeSegmentationResult(GBP_{x})'
|
||||
else:
|
||||
x = int(x)
|
||||
else:
|
||||
bits = 1
|
||||
x = int(self.add_to_current_cell)
|
||||
mask = '0b' + '1' * bits
|
||||
parts.append(f'(({x} & {mask}) << {shift})')
|
||||
shift += bits
|
||||
return ' | '.join(parts)
|
||||
|
||||
@classmethod
|
||||
def go_extra(cls) -> str:
|
||||
bits = GraphemeSegmentationState.used_bits()
|
||||
base_type = f'uint{GraphemeSegmentationState.bitsize()}'
|
||||
return f'''
|
||||
func (r GraphemeSegmentationResult) State() (ans {base_type}) {{
|
||||
return {base_type}(r) & {mask_for(bits)}
|
||||
}}
|
||||
'''
|
||||
|
||||
@property
|
||||
def as_c(self) -> str:
|
||||
parts = []
|
||||
for f in self.new_state._fields:
|
||||
for f in GraphemeSegmentationState._fields:
|
||||
x = getattr(self.new_state, f)
|
||||
match f:
|
||||
case 'grapheme_break':
|
||||
|
|
@ -835,27 +907,16 @@ def as_c(self) -> str:
|
|||
case _:
|
||||
x = int(x)
|
||||
parts.append(f'.{f}={x}')
|
||||
parts.append(f'.add_to_current_cell={self.add_to_current_cell}')
|
||||
parts.append(f'.add_to_current_cell={int(self.add_to_current_cell)}')
|
||||
return '{' + ', '.join(parts) + '}'
|
||||
|
||||
@classmethod
|
||||
def c_declaration(cls) -> str:
|
||||
base_type = f'uint{cls.bitsize()}_t'
|
||||
leftover = cls.bitsize() - cls.used_bits()
|
||||
bits = sum(int(cls._field_defaults[f]) for f in cls._fields)
|
||||
ans = ['// GraphemeSegmentationResultDeclaration', f'// Uses {bits} bits', 'typedef union GraphemeSegmentationResult {',
|
||||
' struct {']
|
||||
for f in cls.new_state._fields:
|
||||
ans.append(f' uint8_t {f} : {int(cls.new_state._field_defaults[f])};')
|
||||
ans.append(' uint8_t add_to_current_cell : 1;')
|
||||
ans.append(f' uint8_t : {leftover};')
|
||||
ans.append(' };')
|
||||
ans.append(f' struct {{ uint16_t state : 9; uint16_t : {1 + leftover}; }};')
|
||||
ans.append(f' {base_type} val;')
|
||||
ans.append('} GraphemeSegmentationResult;')
|
||||
ans.append(f'static_assert(sizeof(GraphemeSegmentationResult) == sizeof({base_type}), "Fix the ordering of GraphemeSegmentationResult");')
|
||||
ans.append('// EndGraphemeSegmentationResultDeclaration')
|
||||
return '\n'.join(ans)
|
||||
fields = {'add_to_current_cell': 1}
|
||||
sfields = GraphemeSegmentationState.fields()
|
||||
fields.update(sfields)
|
||||
bits = sum(sfields.values())
|
||||
return bitfield_declaration_as_c('GraphemeSegmentationResult', fields, {'state': bits})
|
||||
|
||||
|
||||
|
||||
|
|
@ -884,11 +945,11 @@ def bitsize(cls) -> int:
|
|||
ans = sum(int(cls._field_defaults[f]) for f in cls._fields)
|
||||
return clamped_bitsize(ans)
|
||||
|
||||
@property
|
||||
def go_fields(self) -> Iterable[str]:
|
||||
@classmethod
|
||||
def go_fields(cls) -> Sequence[str]:
|
||||
ans = []
|
||||
for f in self._fields:
|
||||
bits = int(self._field_defaults[f])
|
||||
for f in cls._fields:
|
||||
bits = int(cls._field_defaults[f])
|
||||
if f == 'width':
|
||||
f = 'shifted_width'
|
||||
ans.append(f'{f} {bits}')
|
||||
|
|
@ -917,6 +978,21 @@ def as_go(self) -> str:
|
|||
shift += bits
|
||||
return ' | '.join(parts)
|
||||
|
||||
@classmethod
|
||||
def go_extra(cls) -> str:
|
||||
base_type = f'uint{GraphemeSegmentationState.bitsize()}'
|
||||
f = GraphemeSegmentationProps.fields()
|
||||
s = f['grapheme_break'] + f['indic_conjunct_break']
|
||||
return f'''
|
||||
func (s CharProps) Width() int {{
|
||||
return int(s.Shifted_width()) - {width_shift}
|
||||
}}
|
||||
|
||||
func (s CharProps) GraphemeSegmentationProperty() {base_type} {{
|
||||
return {base_type}(s.Grapheme_break() | (s.Indic_conjunct_break() << {f["grapheme_break"]}) | (s.Is_extended_pictographic() << {s}))
|
||||
}}
|
||||
'''
|
||||
|
||||
@property
|
||||
def as_c(self) -> str:
|
||||
parts = []
|
||||
|
|
@ -943,6 +1019,7 @@ def fields(cls) -> dict[str, int]:
|
|||
|
||||
@classmethod
|
||||
def c_declaration(cls) -> str:
|
||||
# Dont know if grapheme_segmentation_property in alternate works on big endian
|
||||
alternate = {
|
||||
'grapheme_segmentation_property': sum(int(cls._field_defaults[f]) for f in GraphemeSegmentationProps._fields)
|
||||
}
|
||||
|
|
@ -1033,9 +1110,13 @@ def aw(s: Iterable[int], width: int) -> None:
|
|||
gsprops = tuple(GraphemeSegmentationProps(
|
||||
grapheme_break=x.grapheme_break, indic_conjunct_break=x.indic_conjunct_break,
|
||||
is_extended_pictographic=x.is_extended_pictographic) for x in prop_array)
|
||||
test_grapheme_segmentation(gsprops)
|
||||
t1, t2, t3, shift, bytesz = splitbins(prop_array, CharProps.bitsize() // 8)
|
||||
test_grapheme_segmentation(partial(split_into_graphemes, gsprops))
|
||||
gseg_results = tuple(GraphemeSegmentationKey.from_int(i).result() for i in range(1 << 16))
|
||||
test_grapheme_segmentation(partial(split_into_graphemes_with_table, gsprops, gseg_results))
|
||||
t1, t2, t3, t_shift, bytesz = splitbins(prop_array, CharProps.bitsize() // 8)
|
||||
print(f'Size of character properties table: {bytesz/1024:.1f}KB')
|
||||
g1, g2, g3, g_shift, bytesz = splitbins(gseg_results, GraphemeSegmentationResult.bitsize() // 8)
|
||||
print(f'Size of grapheme segmentation table: {bytesz/1024:.1f}KB')
|
||||
|
||||
from .bitfields import make_bitfield
|
||||
buf = StringIO()
|
||||
|
|
@ -1045,21 +1126,24 @@ def aw(s: Iterable[int], width: int) -> None:
|
|||
gp('package wcswidth')
|
||||
generate_enum(c, gp, 'GraphemeBreakProperty', *grapheme_segmentation_maps, prefix='GBP_')
|
||||
generate_enum(c, gp, 'IndicConjunctBreak', *incb_map, prefix='ICB_')
|
||||
cen('// UCBDeclaration')
|
||||
cen('// UCBDeclaration {{''{')
|
||||
generate_enum(cen, gp, 'UnicodeCategory', 'Cn', *class_maps, prefix='UC_')
|
||||
cen('// EndUCBDeclaration')
|
||||
bf = make_bitfield('tools/wcswidth', 'CharProps', *CharProps().go_fields, add_package=False)[1]
|
||||
gp(bf)
|
||||
gp(f'''
|
||||
func (s CharProps) Width() int {{
|
||||
return int(s.Shifted_width()) - {width_shift}
|
||||
}}''')
|
||||
gen_multistage_table(c, gp, t1, t2, t3, shift)
|
||||
cen('// EndUCBDeclaration }}''}')
|
||||
gp(make_bitfield('tools/wcswidth', 'CharProps', *CharProps.go_fields(), add_package=False)[1])
|
||||
gp(make_bitfield('tools/wcswidth', 'GraphemeSegmentationResult', *GraphemeSegmentationResult.go_fields(), add_package=False)[1])
|
||||
gp(CharProps.go_extra())
|
||||
gp(GraphemeSegmentationResult.go_extra())
|
||||
gen_multistage_table(c, gp, t1, t2, t3, t_shift)
|
||||
gen_multistage_table(c, gp, g1, g2, g3, g_shift)
|
||||
c(GraphemeSegmentationKey.code_to_convert_to_int())
|
||||
c(GraphemeSegmentationState.c_declaration())
|
||||
gp(GraphemeSegmentationKey.code_to_convert_to_int(for_go=True))
|
||||
gofmt(gof.name)
|
||||
with open('kitty/char-props.h', 'r+') as f:
|
||||
raw = f.read()
|
||||
nraw = re.sub(r'\d+/\*=width_shift\*/', f'{width_shift}/*=width_shift*/', raw)
|
||||
nraw = patch_declaration('CharProps', CharProps.c_declaration(), nraw)
|
||||
nraw = patch_declaration('GraphemeSegmentationResult', GraphemeSegmentationResult.c_declaration(), nraw)
|
||||
nraw = patch_declaration('UCB', buf.getvalue(), nraw)
|
||||
if nraw != raw:
|
||||
f.seek(0)
|
||||
|
|
|
|||
888
kitty/char-props-data.h
generated
888
kitty/char-props-data.h
generated
File diff suppressed because one or more lines are too long
|
|
@ -9,63 +9,21 @@
|
|||
#include "char-props-data.h"
|
||||
|
||||
|
||||
#define is_linker_or_extend(incb) ((incb) == ICB_Linker || (incb) == ICB_Extend)
|
||||
|
||||
CharProps
|
||||
char_props_for(char_type ch) {
|
||||
return CharProps_t3[CharProps_t2[(CharProps_t1[ch >> CharProps_shift] << CharProps_shift) + (ch & CharProps_mask)]];
|
||||
}
|
||||
|
||||
void
|
||||
grapheme_segmentation_reset(GraphemeSegmentationState *s) {
|
||||
*s = (GraphemeSegmentationState){0};
|
||||
grapheme_segmentation_reset(GraphemeSegmentationResult *s) {
|
||||
s->val = 0;
|
||||
}
|
||||
|
||||
bool
|
||||
grapheme_segmentation_step(GraphemeSegmentationState *s, CharProps ch) {
|
||||
// Grapheme segmentation as per UAX29-C1-1 as defined in https://www.unicode.org/reports/tr29/
|
||||
// Returns true iff ch should be added to the current cell based on s which
|
||||
// must reflect the state of the current cell. s is updated by ch.
|
||||
GraphemeBreakProperty prop = ch.grapheme_break;
|
||||
IndicConjunctBreak incb = ch.indic_conjunct_break;
|
||||
bool add_to_cell = false;
|
||||
if (s->last_char_prop == GBP_AtStart) {
|
||||
add_to_cell = true;
|
||||
} else {
|
||||
/* No break between CR and LF (GB3). */
|
||||
if (s->last_char_prop == GBP_CR && prop == GBP_LF) add_to_cell = true;
|
||||
/* Break before and after newlines (GB4, GB5). */
|
||||
else if ((s->last_char_prop == GBP_CR || s->last_char_prop == GBP_LF || s->last_char_prop == GBP_Control)
|
||||
|| (prop == GBP_CR || prop == GBP_LF || prop == GBP_Control)
|
||||
) {}
|
||||
/* No break between Hangul syllable sequences (GB6, GB7, GB8). */
|
||||
else if ((s->last_char_prop == GBP_L && (prop == GBP_L || prop == GBP_V || prop == GBP_LV || prop == GBP_LVT))
|
||||
|| ((s->last_char_prop == GBP_LV || s->last_char_prop == GBP_V) && (prop == GBP_V || prop == GBP_T))
|
||||
|| ((s->last_char_prop == GBP_LVT || s->last_char_prop == GBP_T) && prop == GBP_T)
|
||||
) add_to_cell = true;
|
||||
/* No break before: extending characters or ZWJ (GB9), SpacingMarks (GB9a), Prepend characters (GB9b) */
|
||||
else if (prop == GBP_Extend || prop == GBP_ZWJ || prop == GBP_SpacingMark || s->last_char_prop == GBP_Prepend) add_to_cell = true;
|
||||
/* No break within certain combinations of Indic_Conjunct_Break values:
|
||||
* Between consonant {extend|linker}* linker {extend|linker}* and consonant (GB9c). */
|
||||
else if (s->incb_consonant_extended_linker_extended && incb == ICB_Consonant) add_to_cell = true;
|
||||
/* No break within emoji modifier sequences or emoji zwj sequences (GB11). */
|
||||
else if (s->last_char_prop == GBP_ZWJ && s->emoji_modifier_sequence_before_last_char && ch.is_extended_pictographic) add_to_cell = true;
|
||||
/* No break between RI if there is an odd number of RI characters before (GB12, GB13). */
|
||||
else if (prop == GBP_Regional_Indicator && (s->ri_count % 2) != 0) add_to_cell = true;
|
||||
/* Break everywhere else */
|
||||
else {}
|
||||
}
|
||||
|
||||
s->incb_consonant_extended_linker = s->incb_consonant_extended && incb == ICB_Linker;
|
||||
s->incb_consonant_extended_linker_extended = (s->incb_consonant_extended_linker || (
|
||||
s->incb_consonant_extended_linker_extended && is_linker_or_extend(incb)));
|
||||
s->incb_consonant_extended = (incb == ICB_Consonant || (
|
||||
s->incb_consonant_extended && is_linker_or_extend(incb)));
|
||||
s->emoji_modifier_sequence_before_last_char = s->emoji_modifier_sequence;
|
||||
s->emoji_modifier_sequence = (s->emoji_modifier_sequence && prop == GBP_Extend) || ch.is_extended_pictographic;
|
||||
s->last_char_prop = prop;
|
||||
|
||||
if (prop == GBP_Regional_Indicator) s->ri_count++; else s->ri_count = 0;
|
||||
return add_to_cell;
|
||||
GraphemeSegmentationResult
|
||||
grapheme_segmentation_step(GraphemeSegmentationResult r, CharProps ch) {
|
||||
unsigned key = GraphemeSegmentationKey(r, ch);
|
||||
unsigned t1 = ((unsigned)GraphemeSegmentationResult_t1[key >> GraphemeSegmentationResult_shift]) << GraphemeSegmentationResult_shift;
|
||||
GraphemeSegmentationResult ans = GraphemeSegmentationResult_t3[GraphemeSegmentationResult_t2[t1 + (key & GraphemeSegmentationResult_mask)]];
|
||||
// printf("state: %u gsp: %u -> key: %u t1: %u -> add_to_cell: %u\n", r.state, ch.grapheme_segmentation_property, key, t1, ans.add_to_current_cell);
|
||||
return ans;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -9,16 +9,29 @@
|
|||
|
||||
#include "data-types.h"
|
||||
|
||||
// CharPropsDeclaration
|
||||
// Uses 23 bits
|
||||
// CharPropsDeclaration: uses 23 bits {{{
|
||||
typedef union CharProps {
|
||||
struct {
|
||||
uint8_t shifted_width : 3;
|
||||
struct __attribute__((packed)) {
|
||||
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
|
||||
uint8_t is_extended_pictographic : 1;
|
||||
uint8_t grapheme_break : 4;
|
||||
uint8_t indic_conjunct_break : 2;
|
||||
uint8_t grapheme_break : 4;
|
||||
uint8_t is_punctuation : 1;
|
||||
uint8_t is_word_char : 1;
|
||||
uint8_t is_combining_char : 1;
|
||||
uint8_t is_symbol : 1;
|
||||
uint8_t is_non_rendered : 1;
|
||||
uint8_t is_invalid : 1;
|
||||
uint8_t is_emoji_presentation_base : 1;
|
||||
uint8_t category : 5;
|
||||
uint8_t is_emoji : 1;
|
||||
uint8_t shifted_width : 3;
|
||||
uint16_t : 9;
|
||||
#elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
|
||||
uint16_t : 9;
|
||||
uint8_t shifted_width : 3;
|
||||
uint8_t is_emoji : 1;
|
||||
uint8_t category : 5;
|
||||
uint8_t is_emoji_presentation_base : 1;
|
||||
uint8_t is_invalid : 1;
|
||||
uint8_t is_non_rendered : 1;
|
||||
|
|
@ -26,14 +39,71 @@ typedef union CharProps {
|
|||
uint8_t is_combining_char : 1;
|
||||
uint8_t is_word_char : 1;
|
||||
uint8_t is_punctuation : 1;
|
||||
uint8_t grapheme_break : 4;
|
||||
uint8_t indic_conjunct_break : 2;
|
||||
uint8_t is_extended_pictographic : 1;
|
||||
#else
|
||||
#error "Unsupported endianness"
|
||||
#endif
|
||||
};
|
||||
struct __attribute__((packed)) {
|
||||
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
|
||||
uint8_t grapheme_segmentation_property : 7;
|
||||
uint32_t : 25;
|
||||
#elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
|
||||
uint32_t : 25;
|
||||
uint8_t grapheme_segmentation_property : 7;
|
||||
#else
|
||||
#error "Unsupported endianness"
|
||||
#endif
|
||||
};
|
||||
uint32_t val;
|
||||
} CharProps;
|
||||
static_assert(sizeof(CharProps) == sizeof(uint32_t), "Fix the ordering of CharProps");
|
||||
// EndCharPropsDeclaration
|
||||
// EndCharPropsDeclaration }}}
|
||||
|
||||
// GraphemeSegmentationResultDeclaration: uses 10 bits {{{
|
||||
typedef union GraphemeSegmentationResult {
|
||||
struct __attribute__((packed)) {
|
||||
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
|
||||
uint8_t emoji_modifier_sequence_before_last_char : 1;
|
||||
uint8_t emoji_modifier_sequence : 1;
|
||||
uint8_t incb_consonant_extended_linker_extended : 1;
|
||||
uint8_t incb_consonant_extended_linker : 1;
|
||||
uint8_t incb_consonant_extended : 1;
|
||||
uint8_t grapheme_break : 4;
|
||||
uint8_t add_to_current_cell : 1;
|
||||
uint8_t : 6;
|
||||
#elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
|
||||
uint8_t : 6;
|
||||
uint8_t add_to_current_cell : 1;
|
||||
uint8_t grapheme_break : 4;
|
||||
uint8_t incb_consonant_extended : 1;
|
||||
uint8_t incb_consonant_extended_linker : 1;
|
||||
uint8_t incb_consonant_extended_linker_extended : 1;
|
||||
uint8_t emoji_modifier_sequence : 1;
|
||||
uint8_t emoji_modifier_sequence_before_last_char : 1;
|
||||
#else
|
||||
#error "Unsupported endianness"
|
||||
#endif
|
||||
};
|
||||
struct __attribute__((packed)) {
|
||||
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
|
||||
uint16_t state : 9;
|
||||
uint8_t : 7;
|
||||
#elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
|
||||
uint8_t : 7;
|
||||
uint16_t state : 9;
|
||||
#else
|
||||
#error "Unsupported endianness"
|
||||
#endif
|
||||
};
|
||||
uint16_t val;
|
||||
} GraphemeSegmentationResult;
|
||||
static_assert(sizeof(GraphemeSegmentationResult) == sizeof(uint16_t), "Fix the ordering of GraphemeSegmentationResult");
|
||||
// EndGraphemeSegmentationResultDeclaration }}}
|
||||
|
||||
// UCBDeclaration
|
||||
// UCBDeclaration {{{
|
||||
typedef enum UnicodeCategory {
|
||||
UC_Cn,
|
||||
UC_Cc,
|
||||
|
|
@ -67,37 +137,12 @@ typedef enum UnicodeCategory {
|
|||
UC_Co,
|
||||
} UnicodeCategory;
|
||||
|
||||
// EndUCBDeclaration
|
||||
// EndUCBDeclaration }}}
|
||||
|
||||
|
||||
typedef struct GraphemeSegmentationState {
|
||||
int last_char_prop;
|
||||
|
||||
/* True if the last character ends a sequence of Indic_Conjunct_Break
|
||||
values: consonant {extend|linker}* */
|
||||
bool incb_consonant_extended;
|
||||
/* True if the last character ends a sequence of Indic_Conjunct_Break
|
||||
values: consonant {extend|linker}* linker */
|
||||
bool incb_consonant_extended_linker;
|
||||
/* True if the last character ends a sequence of Indic_Conjunct_Break
|
||||
values: consonant {extend|linker}* linker {extend|linker}* */
|
||||
bool incb_consonant_extended_linker_extended;
|
||||
|
||||
/* True if the last character ends an emoji modifier sequence
|
||||
\p{Extended_Pictographic} Extend*. */
|
||||
bool emoji_modifier_sequence;
|
||||
/* True if the last character was immediately preceded by an
|
||||
emoji modifier sequence \p{Extended_Pictographic} Extend*. */
|
||||
bool emoji_modifier_sequence_before_last_char;
|
||||
|
||||
/* Number of consecutive regional indicator (RI) characters seen
|
||||
immediately before the current point. */
|
||||
size_t ri_count;
|
||||
} GraphemeSegmentationState;
|
||||
|
||||
CharProps char_props_for(char_type ch);
|
||||
void grapheme_segmentation_reset(GraphemeSegmentationState *s);
|
||||
bool grapheme_segmentation_step(GraphemeSegmentationState *s, CharProps ch);
|
||||
void grapheme_segmentation_reset(GraphemeSegmentationResult *s);
|
||||
GraphemeSegmentationResult grapheme_segmentation_step(GraphemeSegmentationResult r, CharProps ch);
|
||||
static inline int wcwidth_std(CharProps ch) { return (int)ch.shifted_width - 4/*=width_shift*/; }
|
||||
static inline bool is_private_use(CharProps ch) { return ch.category == UC_Co; }
|
||||
static inline const char* char_category(CharProps cp) {
|
||||
|
|
|
|||
|
|
@ -139,11 +139,11 @@ split_into_graphemes(PyObject UNUSED *self, PyObject *src) {
|
|||
int kind = PyUnicode_KIND(src); char *data = PyUnicode_DATA(src);
|
||||
RAII_PyObject(ans, PyList_New(0));
|
||||
if (!ans) return NULL;
|
||||
GraphemeSegmentationState s; grapheme_segmentation_reset(&s);
|
||||
GraphemeSegmentationResult s; grapheme_segmentation_reset(&s);
|
||||
Py_ssize_t pos = 0;
|
||||
for (Py_ssize_t i = 0; i < PyUnicode_GET_LENGTH(src); i++) {
|
||||
char_type ch = PyUnicode_READ(kind, data, i);
|
||||
if (!grapheme_segmentation_step(&s, char_props_for(ch))) {
|
||||
if (!(s = grapheme_segmentation_step(s, char_props_for(ch))).add_to_current_cell) {
|
||||
RAII_PyObject(u, PyUnicode_FromKindAndData(kind, data + kind * pos, i - pos));
|
||||
if (!u || PyList_Append(ans, u) != 0) return NULL;
|
||||
pos = i;
|
||||
|
|
|
|||
|
|
@ -638,6 +638,7 @@ def test_shlex_split(self):
|
|||
|
||||
def test_split_into_graphemes(self):
|
||||
self.assertEqual(char_props_for('\ue000')['category'], 'Co')
|
||||
self.ae(split_into_graphemes('ab'), ['a', 'b'])
|
||||
for i, test in enumerate(json.loads(read_kitty_resource('GraphemeBreakTest.json', __name__.rpartition('.')[0]))):
|
||||
expected = test['data']
|
||||
actual = split_into_graphemes(''.join(expected))
|
||||
|
|
|
|||
2
setup.py
2
setup.py
|
|
@ -595,6 +595,8 @@ def add_lpath(which: str, name: str, val: Optional[str]) -> None:
|
|||
ccver=ccver, ldpaths=ldpaths, vcs_rev=vcs_rev,
|
||||
)
|
||||
ans.has_copy_file_range = bool(has_copy_file_range)
|
||||
if ans.compiler_type is CompilerType.gcc:
|
||||
cflags.append('-Wno-packed-bitfield-compat')
|
||||
if verbose:
|
||||
print(ans.cc_version_string.strip())
|
||||
print('Detected:', ans.compiler_type)
|
||||
|
|
|
|||
938
tools/wcswidth/char-props-data.go
generated
938
tools/wcswidth/char-props-data.go
generated
File diff suppressed because one or more lines are too long
|
|
@ -7,41 +7,16 @@ import (
|
|||
|
||||
var _ = fmt.Print
|
||||
|
||||
type GraphemeSegmentationState struct {
|
||||
last_char_prop GraphemeBreakProperty
|
||||
|
||||
/* True if the last character ends a sequence of Indic_Conjunct_Break
|
||||
values: consonant {extend|linker}* */
|
||||
incb_consonant_extended bool
|
||||
/* True if the last character ends a sequence of Indic_Conjunct_Break
|
||||
values: consonant {extend|linker}* linker */
|
||||
incb_consonant_extended_linker bool
|
||||
/* True if the last character ends a sequence of Indic_Conjunct_Break
|
||||
values: consonant {extend|linker}* linker {extend|linker}* */
|
||||
incb_consonant_extended_linker_extended bool
|
||||
|
||||
/* True if the last character ends an emoji modifier sequence
|
||||
\p{Extended_Pictographic} Extend*. */
|
||||
emoji_modifier_sequence bool
|
||||
/* True if the last character was immediately preceded by an
|
||||
emoji modifier sequence \p{Extended_Pictographic} Extend*. */
|
||||
emoji_modifier_sequence_before_last_char bool
|
||||
|
||||
/* Number of consecutive regional indicator (RI) characters seen
|
||||
immediately before the current point. */
|
||||
ri_count uint
|
||||
}
|
||||
|
||||
func CharPropsFor(ch rune) CharProps {
|
||||
return charprops_t3[charprops_t2[(rune(charprops_t1[ch>>charprops_shift])<<charprops_shift)+(ch&charprops_mask)]]
|
||||
}
|
||||
|
||||
func IteratorOverGraphemes(text string) iter.Seq[string] {
|
||||
s := GraphemeSegmentationState{}
|
||||
var s GraphemeSegmentationResult
|
||||
start_pos := 0
|
||||
return func(yield func(string) bool) {
|
||||
for pos, ch := range text {
|
||||
if !s.Step(CharPropsFor(ch)) {
|
||||
if s = s.Step(CharPropsFor(ch)); s.Add_to_current_cell() == 0 {
|
||||
if !yield(text[start_pos:pos]) {
|
||||
return
|
||||
}
|
||||
|
|
@ -62,73 +37,17 @@ func SplitIntoGraphemes(text string) []string {
|
|||
return ans
|
||||
}
|
||||
|
||||
func (i IndicConjunctBreak) is_linker_or_extend() bool {
|
||||
return i == ICB_Linker || i == ICB_Extend
|
||||
func (s *GraphemeSegmentationResult) Reset() {
|
||||
*s = 0
|
||||
}
|
||||
|
||||
func (s *GraphemeSegmentationState) Reset() {
|
||||
*s = GraphemeSegmentationState{}
|
||||
}
|
||||
|
||||
func (s *GraphemeSegmentationState) Step(ch CharProps) bool {
|
||||
// Grapheme segmentation as per UAX29-C1-1 as defined in https://www.unicode.org/reports/tr29/
|
||||
// Returns true iff ch should be added to the current cell based on s which
|
||||
// must reflect the state of the current cell. s is updated by ch.
|
||||
prop := GraphemeBreakProperty(ch.Grapheme_break())
|
||||
incb := IndicConjunctBreak(ch.Indic_conjunct_break())
|
||||
add_to_cell := false
|
||||
if s.last_char_prop == GBP_AtStart {
|
||||
add_to_cell = true
|
||||
} else {
|
||||
/* No break between CR and LF (GB3). */
|
||||
if s.last_char_prop == GBP_CR && prop == GBP_LF {
|
||||
add_to_cell = true
|
||||
} else if
|
||||
/* Break before and after newlines (GB4, GB5). */
|
||||
(s.last_char_prop == GBP_CR || s.last_char_prop == GBP_LF || s.last_char_prop == GBP_Control) ||
|
||||
(prop == GBP_CR || prop == GBP_LF || prop == GBP_Control) {
|
||||
} else if
|
||||
/* No break between Hangul syllable sequences (GB6, GB7, GB8). */
|
||||
(s.last_char_prop == GBP_L && (prop == GBP_L || prop == GBP_V || prop == GBP_LV || prop == GBP_LVT)) ||
|
||||
((s.last_char_prop == GBP_LV || s.last_char_prop == GBP_V) && (prop == GBP_V || prop == GBP_T)) ||
|
||||
((s.last_char_prop == GBP_LVT || s.last_char_prop == GBP_T) && prop == GBP_T) {
|
||||
add_to_cell = true
|
||||
} else if
|
||||
/* No break before: extending characters or ZWJ (GB9), SpacingMarks (GB9a), Prepend characters (GB9b) */
|
||||
prop == GBP_Extend || prop == GBP_ZWJ || prop == GBP_SpacingMark || s.last_char_prop == GBP_Prepend {
|
||||
add_to_cell = true
|
||||
} else if
|
||||
/* No break within certain combinations of Indic_Conjunct_Break values:
|
||||
* Between consonant {extend|linker}* linker {extend|linker}* and consonant (GB9c). */
|
||||
s.incb_consonant_extended_linker_extended && incb == ICB_Consonant {
|
||||
add_to_cell = true
|
||||
} else if
|
||||
/* No break within emoji modifier sequences or emoji zwj sequences (GB11). */
|
||||
s.last_char_prop == GBP_ZWJ && s.emoji_modifier_sequence_before_last_char && (ch.Is_extended_pictographic() == 1) {
|
||||
add_to_cell = true
|
||||
} else if
|
||||
/* No break between RI if there is an odd number of RI characters before (GB12, GB13). */
|
||||
prop == GBP_Regional_Indicator && (s.ri_count%2) != 0 {
|
||||
add_to_cell = true
|
||||
} else
|
||||
/* Break everywhere else */
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
s.incb_consonant_extended_linker = s.incb_consonant_extended && incb == ICB_Linker
|
||||
s.incb_consonant_extended_linker_extended = (s.incb_consonant_extended_linker || (s.incb_consonant_extended_linker_extended && incb.is_linker_or_extend()))
|
||||
s.incb_consonant_extended = (incb == ICB_Consonant || (s.incb_consonant_extended && incb.is_linker_or_extend()))
|
||||
s.emoji_modifier_sequence_before_last_char = s.emoji_modifier_sequence
|
||||
s.emoji_modifier_sequence = (s.emoji_modifier_sequence && prop == GBP_Extend) || (ch.Is_extended_pictographic() == 1)
|
||||
s.last_char_prop = prop
|
||||
|
||||
if prop == GBP_Regional_Indicator {
|
||||
s.ri_count++
|
||||
} else {
|
||||
s.ri_count = 0
|
||||
}
|
||||
return add_to_cell
|
||||
func (s GraphemeSegmentationResult) Step(ch CharProps) GraphemeSegmentationResult {
|
||||
key := grapheme_segmentation_key(s, ch)
|
||||
t1 := uint16(graphemesegmentationresult_t1[key>>graphemesegmentationresult_shift]) << graphemesegmentationresult_shift
|
||||
t2 := graphemesegmentationresult_t2[t1+key&graphemesegmentationresult_mask]
|
||||
ans := graphemesegmentationresult_t3[t2]
|
||||
// fmt.Printf("state: %d gsp: %d -> key: %d t1: %d -> add_to_cell: %d\n", s.State(), ch.GraphemeSegmentationProperty(), key, t1, ans.Add_to_current_cell())
|
||||
return ans
|
||||
}
|
||||
|
||||
func Runewidth(code rune) int {
|
||||
|
|
|
|||
Loading…
Reference in a new issue