mbyte: clean up charset.c for assumed encoding=utf-8

This commit is contained in:
Björn Linse 2015-06-26 17:37:09 +02:00
parent 6115029496
commit 2eeabf9a43

View File

@ -59,12 +59,8 @@ static char_u g_chartab[256];
/// Depends on the option settings 'iskeyword', 'isident', 'isfname',
/// 'isprint' and 'encoding'.
///
/// The index in g_chartab[] depends on 'encoding':
/// - For non-multi-byte index with the byte (same as the character).
/// - For DBCS index with the first byte.
/// - For UTF-8 index with the character (when first byte is up to 0x80 it is
/// the same as the character, if the first byte is 0x80 and above it depends
/// on further bytes).
/// The index in g_chartab[] is the character when first byte is up to 0x80,
/// if the first byte is 0x80 and above it depends on further bytes.
///
/// The contents of g_chartab[]:
/// - The lower two bits, masked by CT_CELL_MASK, give the number of display
@ -118,15 +114,9 @@ int buf_init_chartab(buf_T *buf, int global)
}
while (c < 256) {
if (enc_utf8 && (c >= 0xa0)) {
if (c >= 0xa0) {
// UTF-8: bytes 0xa0 - 0xff are printable (latin1)
g_chartab[c++] = CT_PRINT_CHAR + 1;
} else if ((enc_dbcs == DBCS_JPNU) && (c == 0x8e)) {
// euc-jp characters starting with 0x8e are single width
g_chartab[c++] = CT_PRINT_CHAR + 1;
} else if ((enc_dbcs != 0) && (MB_BYTE2LEN(c) == 2)) {
// other double-byte chars can be printable AND double-width
g_chartab[c++] = CT_PRINT_CHAR + 2;
} else {
// the rest is unprintable by default
g_chartab[c++] = (dy_flags & DY_UHEX) ? 4 : 2;
@ -134,10 +124,8 @@ int buf_init_chartab(buf_T *buf, int global)
}
// Assume that every multi-byte char is a filename character.
for (c = 1; c < 256; ++c) {
if (((enc_dbcs != 0) && (MB_BYTE2LEN(c) > 1))
|| ((enc_dbcs == DBCS_JPNU) && (c == 0x8e))
|| (enc_utf8 && (c >= 0xa0))) {
for (c = 1; c < 256; c++) {
if (c >= 0xa0) {
g_chartab[c] |= CT_FNAME_CHAR;
}
}
@ -146,15 +134,6 @@ int buf_init_chartab(buf_T *buf, int global)
// Init word char flags all to false
memset(buf->b_chartab, 0, (size_t)32);
if (enc_dbcs != 0) {
for (c = 0; c < 256; ++c) {
// double-byte characters are probably word characters
if (MB_BYTE2LEN(c) == 2) {
SET_CHARTAB(buf, c);
}
}
}
// In lisp mode the '-' character is included in keywords.
if (buf->b_p_lisp) {
SET_CHARTAB(buf, '-');
@ -189,10 +168,8 @@ int buf_init_chartab(buf_T *buf, int global)
if (ascii_isdigit(*p)) {
c = getdigits_int(&p);
} else if (has_mbyte) {
c = mb_ptr2char_adv(&p);
} else {
c = *p++;
c = mb_ptr2char_adv(&p);
}
c2 = -1;
@ -201,10 +178,8 @@ int buf_init_chartab(buf_T *buf, int global)
if (ascii_isdigit(*p)) {
c2 = getdigits_int(&p);
} else if (has_mbyte) {
c2 = mb_ptr2char_adv(&p);
} else {
c2 = *p++;
c2 = mb_ptr2char_adv(&p);
}
}
@ -251,8 +226,7 @@ int buf_init_chartab(buf_T *buf, int global)
// that we can detect it from the first byte.
if (((c < ' ')
|| (c > '~')
|| (p_altkeymap && (F_isalpha(c) || F_isdigit(c))))
&& !(enc_dbcs && (MB_BYTE2LEN(c) == 2))) {
|| (p_altkeymap && (F_isalpha(c) || F_isdigit(c))))) {
if (tilde) {
g_chartab[c] = (uint8_t)((g_chartab[c] & ~CT_CELL_MASK)
+ ((dy_flags & DY_UHEX) ? 4 : 2));
@ -313,7 +287,7 @@ void trans_characters(char_u *buf, int bufsize)
while (*buf != 0) {
// Assume a multi-byte character doesn't need translation.
if (has_mbyte && ((trs_len = (*mb_ptr2len)(buf)) > 1)) {
if ((trs_len = (*mb_ptr2len)(buf)) > 1) {
len -= trs_len;
} else {
trs = transchar_byte(*buf);
@ -347,44 +321,40 @@ char_u *transstr(char_u *s) FUNC_ATTR_NONNULL_RET
size_t l;
char_u hexbuf[11];
if (has_mbyte) {
// Compute the length of the result, taking account of unprintable
// multi-byte characters.
size_t len = 0;
p = s;
// Compute the length of the result, taking account of unprintable
// multi-byte characters.
size_t len = 0;
p = s;
while (*p != NUL) {
if ((l = (size_t)(*mb_ptr2len)(p)) > 1) {
c = (*mb_ptr2char)(p);
p += l;
while (*p != NUL) {
if ((l = (size_t)(*mb_ptr2len)(p)) > 1) {
c = (*mb_ptr2char)(p);
p += l;
if (vim_isprintc(c)) {
len += l;
} else {
transchar_hex(hexbuf, c);
len += STRLEN(hexbuf);
}
if (vim_isprintc(c)) {
len += l;
} else {
l = (size_t)byte2cells(*p++);
transchar_hex(hexbuf, c);
len += STRLEN(hexbuf);
}
} else {
l = (size_t)byte2cells(*p++);
if (l > 0) {
len += l;
} else {
// illegal byte sequence
len += 4;
}
if (l > 0) {
len += l;
} else {
// illegal byte sequence
len += 4;
}
}
res = xmallocz(len);
} else {
res = xmallocz((size_t)vim_strsize(s));
}
res = xmallocz(len);
*res = NUL;
p = s;
while (*p != NUL) {
if (has_mbyte && ((l = (size_t)(*mb_ptr2len)(p)) > 1)) {
if ((l = (size_t)(*mb_ptr2len)(p)) > 1) {
c = (*mb_ptr2char)(p);
if (vim_isprintc(c)) {
@ -443,59 +413,49 @@ char_u* str_foldcase(char_u *str, int orglen, char_u *buf, int buflen)
// Make each character lower case.
i = 0;
while (STR_CHAR(i) != NUL) {
if (enc_utf8 || (has_mbyte && (MB_BYTE2LEN(STR_CHAR(i)) > 1))) {
if (enc_utf8) {
int c = utf_ptr2char(STR_PTR(i));
int olen = utf_ptr2len(STR_PTR(i));
int lc = utf_tolower(c);
int c = utf_ptr2char(STR_PTR(i));
int olen = utf_ptr2len(STR_PTR(i));
int lc = utf_tolower(c);
// Only replace the character when it is not an invalid
// sequence (ASCII character or more than one byte) and
// utf_tolower() doesn't return the original character.
if (((c < 0x80) || (olen > 1)) && (c != lc)) {
int nlen = utf_char2len(lc);
// Only replace the character when it is not an invalid
// sequence (ASCII character or more than one byte) and
// utf_tolower() doesn't return the original character.
if (((c < 0x80) || (olen > 1)) && (c != lc)) {
int nlen = utf_char2len(lc);
// If the byte length changes need to shift the following
// characters forward or backward.
if (olen != nlen) {
if (nlen > olen) {
if (buf == NULL) {
ga_grow(&ga, nlen - olen + 1);
} else {
if (len + nlen - olen >= buflen) {
// out of memory, keep old char
lc = c;
nlen = olen;
}
}
}
if (olen != nlen) {
if (buf == NULL) {
STRMOVE(GA_PTR(i) + nlen, GA_PTR(i) + olen);
ga.ga_len += nlen - olen;
} else {
STRMOVE(buf + i + nlen, buf + i + olen);
len += nlen - olen;
}
// If the byte length changes need to shift the following
// characters forward or backward.
if (olen != nlen) {
if (nlen > olen) {
if (buf == NULL) {
ga_grow(&ga, nlen - olen + 1);
} else {
if (len + nlen - olen >= buflen) {
// out of memory, keep old char
lc = c;
nlen = olen;
}
}
(void)utf_char2bytes(lc, STR_PTR(i));
}
if (olen != nlen) {
if (buf == NULL) {
STRMOVE(GA_PTR(i) + nlen, GA_PTR(i) + olen);
ga.ga_len += nlen - olen;
} else {
STRMOVE(buf + i + nlen, buf + i + olen);
len += nlen - olen;
}
}
}
// skip to next multi-byte char
i += (*mb_ptr2len)(STR_PTR(i));
} else {
if (buf == NULL) {
GA_CHAR(i) = (char_u)TOLOWER_LOC(GA_CHAR(i));
} else {
buf[i] = (char_u)TOLOWER_LOC(buf[i]);
}
++i;
(void)utf_char2bytes(lc, STR_PTR(i));
}
// skip to next multi-byte char
i += (*mb_ptr2len)(STR_PTR(i));
}
if (buf == NULL) {
return (char_u *)ga.ga_data;
}
@ -545,7 +505,7 @@ char_u* transchar(int c)
/// @return pointer to translated character in transchar_buf.
char_u* transchar_byte(int c)
{
if (enc_utf8 && (c >= 0x80)) {
if (c >= 0x80) {
transchar_nonprint(transchar_buf, c);
return transchar_buf;
}
@ -578,7 +538,7 @@ void transchar_nonprint(char_u *buf, int c)
buf[1] = (char_u)(c ^ 0x40);
buf[2] = NUL;
} else if (enc_utf8 && (c >= 0x80)) {
} else if (c >= 0x80) {
transchar_hex(buf, c);
} else if ((c >= ' ' + 0x80) && (c <= '~' + 0x80)) {
// 0xa0 - 0xfe
@ -632,15 +592,15 @@ static unsigned nr2hex(unsigned c)
/// Caller must make sure 0 <= b <= 255.
/// For multi-byte mode "b" must be the first byte of a character.
/// A TAB is counted as two cells: "^I".
/// For UTF-8 mode this will return 0 for bytes >= 0x80, because the number of
/// cells depends on further bytes.
/// This will return 0 for bytes >= 0x80, because the number of
/// cells depends on further bytes in UTF-8.
///
/// @param b
///
/// @reeturn Number of display cells.
int byte2cells(int b)
{
if (enc_utf8 && (b >= 0x80)) {
if (b >= 0x80) {
return 0;
}
return g_chartab[b] & CT_CELL_MASK;
@ -662,18 +622,7 @@ int char2cells(int c)
if (c >= 0x80) {
// UTF-8: above 0x80 need to check the value
if (enc_utf8) {
return utf_char2cells(c);
}
// DBCS: double-byte means double-width, except for euc-jp with first
// byte 0x8e
if ((enc_dbcs != 0) && (c >= 0x100)) {
if ((enc_dbcs == DBCS_JPNU) && (((unsigned)c >> 8) == 0x8e)) {
return 1;
}
return 2;
}
return utf_char2cells(c);
}
return g_chartab[c & 0xff] & CT_CELL_MASK;
}
@ -687,7 +636,7 @@ int char2cells(int c)
int ptr2cells(char_u *p)
{
// For UTF-8 we need to look at more bytes if the first byte is >= 0x80.
if (enc_utf8 && (*p >= 0x80)) {
if (*p >= 0x80) {
return utf_ptr2cells(p);
}
@ -722,14 +671,10 @@ int vim_strnsize(char_u *s, int len)
assert(s != NULL);
int size = 0;
while (*s != NUL && --len >= 0) {
if (has_mbyte) {
int l = (*mb_ptr2len)(s);
size += ptr2cells(s);
s += l;
len -= l - 1;
} else {
size += byte2cells(*s++);
}
int l = (*mb_ptr2len)(s);
size += ptr2cells(s);
s += l;
len -= l - 1;
}
return size;
}
@ -840,13 +785,7 @@ bool vim_iswordc_buf(int c, buf_T *buf)
FUNC_ATTR_PURE FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_NONNULL_ARG(2)
{
if (c >= 0x100) {
if (enc_dbcs != 0) {
return dbcs_class((unsigned)c >> 8, (unsigned)(c & 0xff)) >= 2;
}
if (enc_utf8) {
return utf_class(c) >= 2;
}
return utf_class(c) >= 2;
}
return c > 0 && c < 0x100 && GET_CHARTAB(buf, c) != 0;
}
@ -859,7 +798,7 @@ bool vim_iswordc_buf(int c, buf_T *buf)
bool vim_iswordp(char_u *p)
FUNC_ATTR_PURE FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_NONNULL_ALL
{
if (has_mbyte && (MB_BYTE2LEN(*p) > 1)) {
if (MB_BYTE2LEN(*p) > 1) {
return mb_get_class(p) >= 2;
}
return GET_CHARTAB(curbuf, *p) != 0;
@ -875,7 +814,7 @@ bool vim_iswordp(char_u *p)
bool vim_iswordp_buf(char_u *p, buf_T *buf)
FUNC_ATTR_PURE FUNC_ATTR_WARN_UNUSED_RESULT FUNC_ATTR_NONNULL_ALL
{
if (has_mbyte && (MB_BYTE2LEN(*p) > 1)) {
if (MB_BYTE2LEN(*p) > 1) {
return mb_get_class(p) >= 2;
}
return GET_CHARTAB(buf, *p) != 0;
@ -913,7 +852,7 @@ bool vim_isfilec_or_wc(int c)
bool vim_isprintc(int c)
FUNC_ATTR_PURE FUNC_ATTR_WARN_UNUSED_RESULT
{
if (enc_utf8 && (c >= 0x100)) {
if (c >= 0x100) {
return utf_printable(c);
}
return c >= 0x100 || (c > 0 && (g_chartab[c] & CT_PRINT_CHAR));
@ -928,14 +867,10 @@ bool vim_isprintc(int c)
bool vim_isprintc_strict(int c)
FUNC_ATTR_PURE FUNC_ATTR_WARN_UNUSED_RESULT
{
if ((enc_dbcs != 0) && (c < 0x100) && (MB_BYTE2LEN(c) > 1)) {
return false;
}
if (enc_utf8 && (c >= 0x100)) {
if (c >= 0x100) {
return utf_printable(c);
}
return c >= 0x100 || (c > 0 && (g_chartab[c] & CT_PRINT_CHAR));
return c > 0 && (g_chartab[c] & CT_PRINT_CHAR);
}
/// like chartabsize(), but also check for line breaks on the screen
@ -1052,8 +987,7 @@ int win_lbr_chartabsize(win_T *wp, char_u *line, char_u *s, colnr_T col, int *he
break;
}
}
} else if (has_mbyte
&& (size == 2)
} else if ((size == 2)
&& (MB_BYTE2LEN(*s) > 1)
&& wp->w_p_wrap
&& in_win_border(wp, col)) {
@ -1251,28 +1185,24 @@ void getvcol(win_T *wp, pos_T *pos, colnr_T *start, colnr_T *cursor,
if (c == TAB) {
incr = ts - (vcol % ts);
} else {
if (has_mbyte) {
// For utf-8, if the byte is >= 0x80, need to look at
// further bytes to find the cell width.
if (enc_utf8 && (c >= 0x80)) {
incr = utf_ptr2cells(ptr);
} else {
incr = g_chartab[c] & CT_CELL_MASK;
}
// If a double-cell char doesn't fit at the end of a line
// it wraps to the next line, it's like this char is three
// cells wide.
if ((incr == 2)
&& wp->w_p_wrap
&& (MB_BYTE2LEN(*ptr) > 1)
&& in_win_border(wp, vcol)) {
++incr;
head = 1;
}
// For utf-8, if the byte is >= 0x80, need to look at
// further bytes to find the cell width.
if (c >= 0x80) {
incr = utf_ptr2cells(ptr);
} else {
incr = g_chartab[c] & CT_CELL_MASK;
}
// If a double-cell char doesn't fit at the end of a line
// it wraps to the next line, it's like this char is three
// cells wide.
if ((incr == 2)
&& wp->w_p_wrap
&& (MB_BYTE2LEN(*ptr) > 1)
&& in_win_border(wp, vcol)) {
incr++;
head = 1;
}
}
if ((posptr != NULL) && (ptr >= posptr)) {
@ -1557,36 +1487,6 @@ char_u* skiptohex(char_u *q)
// islower()/toupper() etc. do not work properly: they crash when used with
// invalid values or can't handle latin1 when the locale is C.
// Speed is most important here.
#define LATIN1LOWER 'l'
#define LATIN1UPPER 'U'
static char_u latin1flags[257] =
" "
" UUUUUUUUUUUUUUUUUUUUUUUUUU llllllllllllllllllllllllll "
" "
"UUUUUUUUUUUUUUUUUUUUUUU UUUUUUUllllllllllllllllllllllll llllllll";
static char_u latin1upper[257] =
" !\"#$%&'()*+,-./0123456789:;<=>"
"?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`ABCDEFGHIJKLMNOPQRSTUVWXYZ{|}~"
"\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e"
"\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e"
"\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae"
"\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe"
"\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce"
"\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde"
"\xdf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce"
"\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xf7\xd8\xd9\xda\xdb\xdc\xdd\xde\xff";
static char_u latin1lower[257] =
" !\"#$%&'()*+,-./0123456789:;<=>"
"?@abcdefghijklmnopqrstuvwxyz[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~"
"\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e"
"\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e"
"\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae"
"\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe"
"\xbf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee"
"\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xd7\xf8\xf9\xfa\xfb\xfc\xfd\xfe"
"\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee"
"\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff";
/// Check that the character is lower-case
///
@ -1599,20 +1499,7 @@ bool vim_islower(int c)
}
if (c >= 0x80) {
if (enc_utf8) {
return utf_islower(c);
}
if (c >= 0x100) {
if (has_mbyte) {
return iswlower((wint_t)c);
}
// islower() can't handle these chars and may crash
return false;
}
return (latin1flags[c] & LATIN1LOWER) == LATIN1LOWER;
return utf_islower(c);
}
return islower(c);
}
@ -1628,20 +1515,7 @@ bool vim_isupper(int c)
}
if (c >= 0x80) {
if (enc_utf8) {
return utf_isupper(c);
}
if (c >= 0x100) {
if (has_mbyte) {
return iswupper((wint_t)c);
}
// isupper() can't handle these chars and may crash
return false;
}
return (latin1flags[c] & LATIN1UPPER) == LATIN1UPPER;
}
return isupper(c);
}
@ -1653,20 +1527,7 @@ int vim_toupper(int c)
}
if (c >= 0x80) {
if (enc_utf8) {
return utf_toupper(c);
}
if (c >= 0x100) {
if (has_mbyte) {
return (int)towupper((wint_t)c);
}
// toupper() can't handle these chars and may crash
return c;
}
return latin1upper[c];
return utf_toupper(c);
}
return TOUPPER_LOC(c);
}
@ -1678,20 +1539,7 @@ int vim_tolower(int c)
}
if (c >= 0x80) {
if (enc_utf8) {
return utf_tolower(c);
}
if (c >= 0x100) {
if (has_mbyte) {
return (int)towlower((wint_t)c);
}
// tolower() can't handle these chars and may crash
return c;
}
return latin1lower[c];
return utf_tolower(c);
}
return TOLOWER_LOC(c);
}