IntenseFinance/neovim
3
0
Fork 0
mirror of https://github.com/neovim/neovim.git synced 2025-02-25 18:55:25 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #7736 from jamessan/vim-8.0.0420

Browse Source 
[RFC] vim-patch:8.0.0420: text garbled when the system encoding differs from 'encoding'
This commit is contained in:
James McCoy 2017-12-17 20:15:12 -05:00 committed by GitHub
commit e6f8b105b0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 304 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
runtime/doc/options.txt
View File

@ -3809,6 +3809,23 @@ A jump table for the options with a short description can be found at |Q_op|.
This option cannot be set from a |modeline| or in the |sandbox|, for
security reasons.
*'makeencoding'* *'menc'*
'makeencoding' 'menc' string (default "")
global or local to buffer |global-local|
{only available when compiled with the |+multi_byte|
feature}
{not in Vi}
Encoding used for reading the output of external commands. When empty,
encoding is not converted.
This is used for `:make`, `:lmake`, `:grep`, `:lgrep`, `:grepadd`,
`:lgrepadd`, `:cfile`, `:cgetfile`, `:caddfile`, `:lfile`, `:lgetfile`,
and `:laddfile`.
This would be mostly useful when you use MS-Windows. If |+iconv| is
enabled and GNU libiconv is used, setting 'makeencoding' to "char" has
the same effect as setting to the system locale encoding. Example: >
:set makeencoding=char " system locale is used
<
*'makeprg'* *'mp'*
'makeprg' 'mp' string (default "make")
global or local to buffer |global-local|

21
runtime/doc/quickfix.txt
View File

@ -165,6 +165,9 @@ processing a quickfix or location list command, it will be aborted.
keep Vim running while compiling. If you give the
name of the errorfile, the 'errorfile' option will
be set to [errorfile]. See |:cc| for [!].
If the encoding of the error file differs from the
'encoding' option, you can use the 'makeencoding'
option to specify the encoding.
*:lf* *:lfile*
:lf[ile][!] [errorfile] Same as ":cfile", except the location list for the
@ -176,6 +179,9 @@ processing a quickfix or location list command, it will be aborted.
:cg[etfile] [errorfile] *:cg* *:cgetfile*
Read the error file. Just like ":cfile" but don't
jump to the first error.
If the encoding of the error file differs from the
'encoding' option, you can use the 'makeencoding'
option to specify the encoding.
:lg[etfile] [errorfile] *:lg* *:lgetfile*
@ -186,6 +192,9 @@ processing a quickfix or location list command, it will be aborted.
:caddf[ile] [errorfile] Read the error file and add the errors from the
errorfile to the current quickfix list. If a quickfix
list is not present, then a new list is created.
If the encoding of the error file differs from the
'encoding' option, you can use the 'makeencoding'
option to specify the encoding.
*:laddf* *:laddfile*
:laddf[ile] [errorfile] Same as ":caddfile", except the location list for the
@ -322,6 +331,7 @@ use this code: >
endfunction
au QuickfixCmdPost make call QfMakeConv()
Another option is using 'makeencoding'.
EXECUTE A COMMAND IN ALL THE BUFFERS IN QUICKFIX OR LOCATION LIST:
*:cdo*
@ -586,6 +596,9 @@ lists, use ":cnewer 99" first.
like |:cnext| and |:cprevious|, see above.
This command does not accept a comment, any "
characters are considered part of the arguments.
If the encoding of the program output differs from the
'encoding' option, you can use the 'makeencoding'
option to specify the encoding.
*:lmak* *:lmake*
:lmak[e][!] [arguments]
@ -645,6 +658,7 @@ read the error messages: >
au QuickfixCmdPost make call QfMakeConv()
(Example by Faque Cheng)
Another option is using 'makeencoding'.
==============================================================================
5. Using :vimgrep and :grep *grep* *lid*
@ -759,6 +773,9 @@ id-utils) in a similar way to its compiler integration (see |:make| above).
When 'grepprg' is "internal" this works like
|:vimgrep|. Note that the pattern needs to be
enclosed in separator characters then.
If the encoding of the program output differs from the
'encoding' option, you can use the 'makeencoding'
option to specify the encoding.
*:lgr* *:lgrep*
:lgr[ep][!] [arguments] Same as ":grep", except the location list for the
@ -783,6 +800,10 @@ id-utils) in a similar way to its compiler integration (see |:make| above).
\ | catch /E480:/
\ | endtry"
<
If the encoding of the program output differs from the
'encoding' option, you can use the 'makeencoding'
option to specify the encoding.
*:lgrepa* *:lgrepadd*
:lgrepa[dd][!] [arguments]
Same as ":grepadd", except the location list for the

1
runtime/doc/quickref.txt
View File

@ -762,6 +762,7 @@ Short explanation of each option: *option-list*
'loadplugins' 'lpl' load plugin scripts when starting up
'magic' changes special characters in search patterns
'makeef' 'mef' name of the errorfile for ":make"
'makeencoding' 'menc' encoding of external make/grep commands
'makeprg' 'mp' program to use for the ":make" command
'matchpairs' 'mps' pairs of characters that "%" can match
'matchtime' 'mat' tenths of a second to show matching paren

1
src/nvim/buffer.c
View File

@ -1808,6 +1808,7 @@ void free_buf_options(buf_T *buf, int free_p_ff)
buf->b_p_ul = NO_LOCAL_UNDOLEVEL;
clear_string_option(&buf->b_p_lw);
clear_string_option(&buf->b_p_bkc);
clear_string_option(&buf->b_p_menc);
}

1
src/nvim/buffer_defs.h
View File

@ -637,6 +637,7 @@ struct file_buffer {
uint32_t b_p_fex_flags; ///< flags for 'formatexpr'
char_u *b_p_kp; ///< 'keywordprg'
int b_p_lisp; ///< 'lisp'
char_u *b_p_menc; ///< 'makeencoding'
char_u *b_p_mps; ///< 'matchpairs'
int b_p_ml; ///< 'modeline'
int b_p_ml_nobin; ///< b_p_ml saved for binary mode

4
src/nvim/if_cscope.c
View File

@ -1012,9 +1012,9 @@ static int cs_find_common(char *opt, char *pat, int forceit, int verbose,
fclose(f);
if (use_ll) /* Use location list */
wp = curwin;
/* '-' starts a new error list */
// '-' starts a new error list
if (qf_init(wp, tmp, (char_u *)"%f%*\\t%l%*\\t%m",
*qfpos == '-', cmdline) > 0) {
*qfpos == '-', cmdline, NULL) > 0) {
if (postponed_split != 0) {
(void)win_split(postponed_split > 0 ? postponed_split : 0,
postponed_split_flags);

2
src/nvim/main.c
View File

@ -1369,7 +1369,7 @@ static void handle_quickfix(mparm_T *paramp)
set_string_option_direct((char_u *)"ef", -1,
paramp->use_ef, OPT_FREE, SID_CARG);
vim_snprintf((char *)IObuff, IOSIZE, "cfile %s", p_ef);
if (qf_init(NULL, p_ef, p_efm, true, IObuff) < 0) {
if (qf_init(NULL, p_ef, p_efm, true, IObuff, p_menc) < 0) {
ui_linefeed();
mch_exit(3);
}

12
src/nvim/option.c
View File

@ -2217,6 +2217,7 @@ void check_buf_options(buf_T *buf)
check_string_option(&buf->b_p_tsr);
check_string_option(&buf->b_p_lw);
check_string_option(&buf->b_p_bkc);
check_string_option(&buf->b_p_menc);
}
/*
@ -2635,8 +2636,8 @@ did_set_string_option (
else if (varp == &p_ei) {
if (check_ei() == FAIL)
errmsg = e_invarg;
/* 'encoding' and 'fileencoding' */
} else if (varp == &p_enc || gvarp == &p_fenc) {
// 'encoding', 'fileencoding' and 'makeencoding'
} else if (varp == &p_enc || gvarp == &p_fenc || gvarp == &p_menc) {
if (gvarp == &p_fenc) {
if (!MODIFIABLE(curbuf) && opt_flags != OPT_GLOBAL) {
errmsg = e_modifiable;
@ -5400,6 +5401,9 @@ void unset_global_local_option(char *name, void *from)
case PV_LW:
clear_string_option(&buf->b_p_lw);
break;
case PV_MENC:
clear_string_option(&buf->b_p_menc);
break;
}
}
@ -5433,6 +5437,7 @@ static char_u *get_varp_scope(vimoption_T *p, int opt_flags)
case PV_UL: return (char_u *)&(curbuf->b_p_ul);
case PV_LW: return (char_u *)&(curbuf->b_p_lw);
case PV_BKC: return (char_u *)&(curbuf->b_p_bkc);
case PV_MENC: return (char_u *)&(curbuf->b_p_menc);
}
return NULL; /* "cannot happen" */
}
@ -5488,6 +5493,8 @@ static char_u *get_varp(vimoption_T *p)
? (char_u *)&(curbuf->b_p_ul) : p->var;
case PV_LW: return *curbuf->b_p_lw != NUL
? (char_u *)&(curbuf->b_p_lw) : p->var;
case PV_MENC: return *curbuf->b_p_menc != NUL
? (char_u *)&(curbuf->b_p_menc) : p->var;
case PV_ARAB: return (char_u *)&(curwin->w_p_arab);
case PV_LIST: return (char_u *)&(curwin->w_p_list);
@ -5885,6 +5892,7 @@ void buf_copy_options(buf_T *buf, int flags)
buf->b_p_qe = vim_strsave(p_qe);
buf->b_p_udf = p_udf;
buf->b_p_lw = empty_option;
buf->b_p_menc = empty_option;
/*
* Don't copy the options set by ex_help(), use the saved values,

2
src/nvim/option_defs.h
View File

@ -489,6 +489,7 @@ EXTERN char_u *p_lcs; // 'listchars'
EXTERN int p_lz; // 'lazyredraw'
EXTERN int p_lpl; // 'loadplugins'
EXTERN int p_magic; // 'magic'
EXTERN char_u *p_menc; // 'makeencoding'
EXTERN char_u *p_mef; // 'makeef'
EXTERN char_u *p_mp; // 'makeprg'
EXTERN char_u *p_cc; // 'colorcolumn'
@ -736,6 +737,7 @@ enum {
, BV_KP
, BV_LISP
, BV_LW
, BV_MENC
, BV_MA
, BV_ML
, BV_MOD

7
src/nvim/options.lua
View File

@ -1456,6 +1456,13 @@ return {
varname='p_mef',
defaults={if_true={vi=""}}
},
{
full_name='makeencoding', abbreviation='menc',
type='string', scope={'global', 'buffer'},
vi_def=true,
varname='p_menc',
defaults={if_true={vi=""}}
},
{
full_name='makeprg', abbreviation='mp',
type='string', scope={'global', 'buffer'},

106
src/nvim/quickfix.c
View File

@ -160,6 +160,7 @@ typedef struct {
buf_T *buf;
linenr_T buflnum;
linenr_T lnumlast;
vimconv_T vc;
} qfstate_T;
typedef struct {
@ -193,19 +194,19 @@ typedef struct {
static char_u *qf_last_bufname = NULL;
static bufref_T qf_last_bufref = { NULL, 0, 0 };
/*
* Read the errorfile "efile" into memory, line by line, building the error
* list. Set the error list's title to qf_title.
* Return -1 for error, number of errors for success.
*/
int
qf_init (
win_T *wp,
char_u *efile,
char_u *errorformat,
int newlist, /* TRUE: start a new error list */
char_u *qf_title
)
/// Read the errorfile "efile" into memory, line by line, building the error
/// list. Set the error list's title to qf_title.
///
/// @params wp If non-NULL, make a location list
/// @params efile If non-NULL, errorfile to parse
/// @params errorformat 'errorformat' string used to parse the error lines
/// @params newlist If true, create a new error list
/// @params qf_title If non-NULL, title of the error list
/// @params enc If non-NULL, encoding used to parse errors
///
/// @returns -1 for error, number of errors for success.
int qf_init(win_T *wp, char_u *efile, char_u *errorformat, int newlist,
char_u *qf_title, char_u *enc)
{
qf_info_T *qi = &ql_info;
@ -214,8 +215,8 @@ qf_init (
}
return qf_init_ext(qi, efile, curbuf, NULL, errorformat, newlist,
(linenr_T)0, (linenr_T)0,
qf_title);
(linenr_T)0, (linenr_T)0,
qf_title, enc);
}
// Maximum number of bytes allowed per line while reading an errorfile.
@ -638,6 +639,22 @@ retry:
} else {
state->linebuf = IObuff;
}
// Convert a line if it contains a non-ASCII character
if (state->vc.vc_type != CONV_NONE && has_non_ascii(state->linebuf)) {
char_u *line = string_convert(&state->vc, state->linebuf, &state->linelen);
if (line != NULL) {
if (state->linelen < IOSIZE) {
STRLCPY(state->linebuf, line, state->linelen + 1);
xfree(line);
} else {
xfree(state->growbuf);
state->linebuf = state->growbuf = line;
state->growbufsiz = state->linelen < LINE_MAXLEN
? state->linelen : LINE_MAXLEN;
}
}
}
return QF_OK;
}
@ -976,15 +993,15 @@ qf_init_ext(
buf_T *buf,
typval_T *tv,
char_u *errorformat,
int newlist, /* TRUE: start a new error list */
linenr_T lnumfirst, /* first line number to use */
linenr_T lnumlast, /* last line number to use */
char_u *qf_title
int newlist, // TRUE: start a new error list
linenr_T lnumfirst, // first line number to use
linenr_T lnumlast, // last line number to use
char_u *qf_title,
char_u *enc
)
{
qfstate_T state = { NULL, 0, NULL, 0, NULL, NULL, NULL, NULL,
NULL, 0, 0 };
qffields_T fields = { NULL, NULL, 0, 0L, 0, false, NULL, 0, 0, 0 };
qfstate_T state;
qffields_T fields;
qfline_T *old_last = NULL;
bool adding = false;
static efm_T *fmt_first = NULL;
@ -997,6 +1014,13 @@ qf_init_ext(
xfree(qf_last_bufname);
qf_last_bufname = NULL;
memset(&state, 0, sizeof(state));
memset(&fields, 0, sizeof(fields));
state.vc.vc_type = CONV_NONE;
if (enc != NULL && *enc != NUL) {
convert_setup(&state.vc, enc, p_enc);
}
fields.namebuf = xmalloc(CMDBUFFSIZE + 1);
fields.errmsglen = CMDBUFFSIZE + 1;
fields.errmsg = xmalloc(fields.errmsglen);
@ -1147,6 +1171,10 @@ qf_init_end:
qf_update_buffer(qi, old_last);
if (state.vc.vc_type != CONV_NONE) {
convert_setup(&state.vc, NULL, NULL);
}
return retval;
}
@ -3024,6 +3052,7 @@ void ex_make(exarg_T *eap)
qf_info_T *qi = &ql_info;
int res;
char_u *au_name = NULL;
char_u *enc = (*curbuf->b_p_menc != NUL) ? curbuf->b_p_menc : p_menc;
/* Redirect ":grep" to ":vimgrep" if 'grepprg' is "internal". */
if (grep_internal(eap->cmdidx)) {
@ -3083,11 +3112,11 @@ void ex_make(exarg_T *eap)
res = qf_init(wp, fname, (eap->cmdidx != CMD_make
&& eap->cmdidx != CMD_lmake) ? p_gefm : p_efm,
(eap->cmdidx != CMD_grepadd
&& eap->cmdidx != CMD_lgrepadd),
*eap->cmdlinep);
if (wp != NULL)
(eap->cmdidx != CMD_grepadd && eap->cmdidx != CMD_lgrepadd),
*eap->cmdlinep, enc);
if (wp != NULL) {
qi = GET_LOC_LIST(wp);
}
if (au_name != NULL) {
apply_autocmds(EVENT_QUICKFIXCMDPOST, au_name,
curbuf->b_fname, TRUE, curbuf);
@ -3429,19 +3458,18 @@ void ex_cfile(exarg_T *eap)
if (*eap->arg != NUL)
set_string_option_direct((char_u *)"ef", -1, eap->arg, OPT_FREE, 0);
/*
* This function is used by the :cfile, :cgetfile and :caddfile
* commands.
* :cfile always creates a new quickfix list and jumps to the
* first error.
* :cgetfile creates a new quickfix list but doesn't jump to the
* first error.
* :caddfile adds to an existing quickfix list. If there is no
* quickfix list then a new list is created.
*/
char_u *enc = (*curbuf->b_p_menc != NUL) ? curbuf->b_p_menc : p_menc;
// This function is used by the :cfile, :cgetfile and :caddfile
// commands.
// :cfile always creates a new quickfix list and jumps to the
// first error.
// :cgetfile creates a new quickfix list but doesn't jump to the
// first error.
// :caddfile adds to an existing quickfix list. If there is no
// quickfix list then a new list is created.
if (qf_init(wp, p_ef, p_efm, (eap->cmdidx != CMD_caddfile
&& eap->cmdidx != CMD_laddfile),
*eap->cmdlinep) > 0
*eap->cmdlinep, enc) > 0
&& (eap->cmdidx == CMD_cfile
|| eap->cmdidx == CMD_lfile)) {
if (au_name != NULL)
@ -4338,7 +4366,7 @@ void ex_cbuffer(exarg_T *eap)
if (qf_init_ext(qi, NULL, buf, NULL, p_efm,
(eap->cmdidx != CMD_caddbuffer
&& eap->cmdidx != CMD_laddbuffer),
eap->line1, eap->line2, qf_title) > 0) {
eap->line1, eap->line2, qf_title, NULL) > 0) {
if (au_name != NULL) {
apply_autocmds(EVENT_QUICKFIXCMDPOST, (char_u *)au_name,
curbuf->b_fname, true, curbuf);
@ -4403,7 +4431,7 @@ void ex_cexpr(exarg_T *eap)
if (qf_init_ext(qi, NULL, NULL, &tv, p_efm,
(eap->cmdidx != CMD_caddexpr
&& eap->cmdidx != CMD_laddexpr),
(linenr_T)0, (linenr_T)0, *eap->cmdlinep) > 0) {
(linenr_T)0, (linenr_T)0, *eap->cmdlinep, NULL) > 0) {
if (au_name != NULL) {
apply_autocmds(EVENT_QUICKFIXCMDPOST, (char_u *)au_name,
curbuf->b_fname, true, curbuf);

1
src/nvim/testdir/Makefile
View File

@ -65,6 +65,7 @@ NEW_TESTS ?= \
test_increment_dbcs.res \
test_lambda.res \
test_langmap.res \
test_makeencoding.res \
test_marks.res \
test_match.res \
test_matchadd_conceal.res \

67
src/nvim/testdir/test_makeencoding.py Normal file
View File

@ -0,0 +1,67 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
# Test program for :make, :grep and :cgetfile.
from __future__ import print_function, unicode_literals
import locale
import io
import sys
def set_output_encoding(enc=None):
"""Set the encoding of stdout and stderr
arguments:
enc -- Encoding name.
If omitted, locale.getpreferredencoding() is used.
"""
if enc is None:
enc = locale.getpreferredencoding()
def get_text_writer(fo, **kwargs):
kw = dict(kwargs)
kw.setdefault('errors', 'backslashreplace') # use \uXXXX style
kw.setdefault('closefd', False)
if sys.version_info[0] < 3:
# Work around for Python 2.x
# New line conversion isn't needed here. Done in somewhere else.
writer = io.open(fo.fileno(), mode='w', newline='', **kw)
write = writer.write # save the original write() function
enc = locale.getpreferredencoding()
def convwrite(s):
if isinstance(s, bytes):
write(s.decode(enc)) # convert to unistr
else:
write(s)
try:
writer.flush() # needed on Windows
except IOError:
pass
writer.write = convwrite
else:
writer = io.open(fo.fileno(), mode='w', **kw)
return writer
sys.stdout = get_text_writer(sys.stdout, encoding=enc)
sys.stderr = get_text_writer(sys.stderr, encoding=enc)
def main():
enc = 'utf-8'
if len(sys.argv) > 1:
enc = sys.argv[1]
set_output_encoding(enc)
message_tbl = {
'utf-8': 'ÀÈÌÒÙ こんにちは 你好',
'latin1': 'ÀÈÌÒÙ',
'cp932': 'こんにちは',
'cp936': '你好',
}
print('Xfoobar.c(10) : %s (%s)' % (message_tbl[enc], enc))
if __name__ == "__main__":
main()

106
src/nvim/testdir/test_makeencoding.vim Normal file
View File

@ -0,0 +1,106 @@
" Tests for 'makeencoding'.
if !has('multi_byte')
finish
endif
source shared.vim
let s:python = PythonProg()
if s:python == ''
" Can't run this test.
finish
endif
let s:script = 'test_makeencoding.py'
let s:message_tbl = {
\ 'utf-8': 'ÀÈÌÒÙ こんにちは 你好',
\ 'latin1': 'ÀÈÌÒÙ',
\ 'cp932': 'こんにちは',
\ 'cp936': '你好',
\}
" Tests for :cgetfile and :lgetfile.
func Test_getfile()
set errorfile=Xerror.txt
set errorformat=%f(%l)\ :\ %m
" :cgetfile
for enc in keys(s:message_tbl)
let &makeencoding = enc
exec "silent !" . s:python . " " . s:script . " " . enc . " > " . &errorfile
cgetfile
copen
call assert_equal("Xfoobar.c|10| " . s:message_tbl[enc] . " (" . enc . ")",
\ getline('.'))
cclose
endfor
" :lgetfile
for enc in keys(s:message_tbl)
let &makeencoding = enc
exec "silent !" . s:python . " " . s:script . " " . enc . " > " . &errorfile
lgetfile
lopen
call assert_equal("Xfoobar.c|10| " . s:message_tbl[enc] . " (" . enc . ")",
\ getline('.'))
lclose
endfor
call delete(&errorfile)
endfunc
" Tests for :grep and :lgrep.
func Test_grep()
let &grepprg = s:python
set grepformat=%f(%l)\ :\ %m
" :grep
for enc in keys(s:message_tbl)
let &makeencoding = enc
exec "silent grep! " . s:script . " " . enc
copen
call assert_equal("Xfoobar.c|10| " . s:message_tbl[enc] . " (" . enc . ")",
\ getline('.'))
cclose
endfor
" :lgrep
for enc in keys(s:message_tbl)
let &makeencoding = enc
exec "silent lgrep! " . s:script . " " . enc
lopen
call assert_equal("Xfoobar.c|10| " . s:message_tbl[enc] . " (" . enc . ")",
\ getline('.'))
lclose
endfor
endfunc
" Tests for :make and :lmake.
func Test_make()
let &makeprg = s:python
set errorformat=%f(%l)\ :\ %m
" :make
for enc in keys(s:message_tbl)
let &makeencoding = enc
exec "silent make! " . s:script . " " . enc
copen
call assert_equal("Xfoobar.c|10| " . s:message_tbl[enc] . " (" . enc . ")",
\ getline('.'))
cclose
endfor
" :lmake
for enc in keys(s:message_tbl)
let &makeencoding = enc
exec "silent lmake! " . s:script . " " . enc
lopen
call assert_equal("Xfoobar.c|10| " . s:message_tbl[enc] . " (" . enc . ")",
\ getline('.'))
lclose
endfor
endfunc