diff options
Diffstat (limited to 'dfa.c')
-rw-r--r-- | dfa.c | 13 |
1 files changed, 6 insertions, 7 deletions
@@ -1847,13 +1847,12 @@ copytoks (size_t tindex, size_t ntokens) { size_t i; - for (i = 0; i < ntokens; ++i) - { - addtok (dfa->tokens[tindex + i]); - /* Update index into multibyte csets. */ - if (MB_CUR_MAX > 1 && dfa->tokens[tindex + i] == MBCSET) - dfa->multibyte_prop[dfa->tindex - 1] = dfa->multibyte_prop[tindex + i]; - } + if (MB_CUR_MAX > 1) + for (i = 0; i < ntokens; ++i) + addtok_mb(dfa->tokens[tindex + i], dfa->multibyte_prop[tindex + i]); + else + for (i = 0; i < ntokens; ++i) + addtok_mb(dfa->tokens[tindex + i], 3); } static void |