vim-patch:8.0.0420: text garbled when the system encoding differs from 'encoding'

Problem:    When running :make the output may be in the system encoding,
            different from 'encoding'.
Solution:   Add the 'makeencoding' option. (Ken Takata)

2c7292dc5b

runtime/doc/options.txt

@@ -3809,6 +3809,23 @@ A jump table for the options with a short description can be found at |Q_op|.
 	This option cannot be set from a |modeline| or in the |sandbox|, for
 	security reasons.
 
+					*'makeencoding'* *'menc'*
+'makeencoding' 'menc'	string	(default "")
+			global or local to buffer |global-local|
+			{only available when compiled with the |+multi_byte|
+			feature}
+			{not in Vi}
+	Encoding used for reading the output of external commands.  When empty,
+	encoding is not converted.
+	This is used for `:make`, `:lmake`, `:grep`, `:lgrep`, `:grepadd`,
+	`:lgrepadd`, `:cfile`, `:cgetfile`, `:caddfile`, `:lfile`, `:lgetfile`,
+	and `:laddfile`.
+
+	This would be mostly useful when you use MS-Windows.  If |+iconv| is
+	enabled and GNU libiconv is used, setting 'makeencoding' to "char" has
+	the same effect as setting to the system locale encoding.  Example: >
+		:set makeencoding=char	" system locale is used
+<
 						*'makeprg'* *'mp'*
 'makeprg' 'mp'		string	(default "make")
 			global or local to buffer |global-local|

runtime/doc/quickfix.txt

@@ -165,6 +165,9 @@ processing a quickfix or location list command, it will be aborted.
 			keep Vim running while compiling.  If you give the
 			name of the errorfile, the 'errorfile' option will
 			be set to [errorfile].  See |:cc| for [!].
+			If the encoding of the error file differs from the
+			'encoding' option, you can use the 'makeencoding'
+			option to specify the encoding.
 
 							*:lf* *:lfile*
 :lf[ile][!] [errorfile]	Same as ":cfile", except the location list for the
@@ -176,6 +179,9 @@ processing a quickfix or location list command, it will be aborted.
 :cg[etfile] [errorfile]					*:cg* *:cgetfile*
 			Read the error file.  Just like ":cfile" but don't
 			jump to the first error.
+			If the encoding of the error file differs from the
+			'encoding' option, you can use the 'makeencoding'
+			option to specify the encoding.
 
 
 :lg[etfile] [errorfile]					*:lg* *:lgetfile*
@@ -186,6 +192,9 @@ processing a quickfix or location list command, it will be aborted.
 :caddf[ile] [errorfile]	Read the error file and add the errors from the
 			errorfile to the current quickfix list. If a quickfix
 			list is not present, then a new list is created.
+			If the encoding of the error file differs from the
+			'encoding' option, you can use the 'makeencoding'
+			option to specify the encoding.
 
 							*:laddf* *:laddfile*
 :laddf[ile] [errorfile]	Same as ":caddfile", except the location list for the
@@ -322,6 +331,7 @@ use this code: >
 	endfunction
 
 	au QuickfixCmdPost make call QfMakeConv()
+Another option is using 'makeencoding'.
 
 EXECUTE A COMMAND IN ALL THE BUFFERS IN QUICKFIX OR LOCATION LIST:
 							*:cdo*
@@ -586,6 +596,9 @@ lists, use ":cnewer 99" first.
 			   like |:cnext| and |:cprevious|, see above.
 			This command does not accept a comment, any "
 			characters are considered part of the arguments.
+			If the encoding of the program output differs from the
+			'encoding' option, you can use the 'makeencoding'
+			option to specify the encoding.
 
 							*:lmak* *:lmake*
 :lmak[e][!] [arguments]
@@ -645,6 +658,7 @@ read the error messages: >
 	au QuickfixCmdPost make call QfMakeConv()
 
 (Example by Faque Cheng)
+Another option is using 'makeencoding'.
 
 ==============================================================================
 5. Using :vimgrep and :grep				*grep* *lid*
@@ -759,6 +773,9 @@ id-utils) in a similar way to its compiler integration (see |:make| above).
 			When 'grepprg' is "internal" this works like
 			|:vimgrep|.  Note that the pattern needs to be
 			enclosed in separator characters then.
+			If the encoding of the program output differs from the
+			'encoding' option, you can use the 'makeencoding'
+			option to specify the encoding.
 
 							    *:lgr* *:lgrep*
 :lgr[ep][!] [arguments]	Same as ":grep", except the location list for the
@@ -783,6 +800,10 @@ id-utils) in a similar way to its compiler integration (see |:make| above).
 				  \ | catch /E480:/
 				  \ | endtry"
 <
+			If the encoding of the program output differs from the
+			'encoding' option, you can use the 'makeencoding'
+			option to specify the encoding.
+
 							*:lgrepa* *:lgrepadd*
 :lgrepa[dd][!] [arguments]
 			Same as ":grepadd", except the location list for the

runtime/doc/quickref.txt

@@ -762,6 +762,7 @@ Short explanation of each option:		*option-list*
 'loadplugins'	  'lpl'     load plugin scripts when starting up
 'magic'			    changes special characters in search patterns
 'makeef'	  'mef'     name of the errorfile for ":make"
+'makeencoding'	  'menc'    encoding of external make/grep commands
 'makeprg'	  'mp'	    program to use for the ":make" command
 'matchpairs'	  'mps'     pairs of characters that "%" can match
 'matchtime'	  'mat'     tenths of a second to show matching paren

src/nvim/buffer.c

@@ -1808,6 +1808,7 @@ void free_buf_options(buf_T *buf, int free_p_ff)
   buf->b_p_ul = NO_LOCAL_UNDOLEVEL;
   clear_string_option(&buf->b_p_lw);
   clear_string_option(&buf->b_p_bkc);
+  clear_string_option(&buf->b_p_menc);
 }
 
 

src/nvim/buffer_defs.h

@@ -637,6 +637,7 @@ struct file_buffer {
   uint32_t b_p_fex_flags;       ///< flags for 'formatexpr'
   char_u *b_p_kp;               ///< 'keywordprg'
   int b_p_lisp;                 ///< 'lisp'
+  char_u *b_p_menc;             ///< 'makeencoding'
   char_u *b_p_mps;              ///< 'matchpairs'
   int b_p_ml;                   ///< 'modeline'
   int b_p_ml_nobin;             ///< b_p_ml saved for binary mode

src/nvim/if_cscope.c

@@ -1012,9 +1012,9 @@ static int cs_find_common(char *opt, char *pat, int forceit, int verbose,
       fclose(f);
       if (use_ll)           /* Use location list */
         wp = curwin;
-      /* '-' starts a new error list */
+      // '-' starts a new error list
       if (qf_init(wp, tmp, (char_u *)"%f%*\\t%l%*\\t%m",
-              *qfpos == '-', cmdline) > 0) {
+                  *qfpos == '-', cmdline, NULL) > 0) {
         if (postponed_split != 0) {
           (void)win_split(postponed_split > 0 ? postponed_split : 0,
                           postponed_split_flags);

src/nvim/main.c

@@ -1369,7 +1369,7 @@ static void handle_quickfix(mparm_T *paramp)
       set_string_option_direct((char_u *)"ef", -1,
           paramp->use_ef, OPT_FREE, SID_CARG);
     vim_snprintf((char *)IObuff, IOSIZE, "cfile %s", p_ef);
-    if (qf_init(NULL, p_ef, p_efm, true, IObuff) < 0) {
+    if (qf_init(NULL, p_ef, p_efm, true, IObuff, p_menc) < 0) {
       ui_linefeed();
       mch_exit(3);
     }

src/nvim/option.c

@@ -2217,6 +2217,7 @@ void check_buf_options(buf_T *buf)
   check_string_option(&buf->b_p_tsr);
   check_string_option(&buf->b_p_lw);
   check_string_option(&buf->b_p_bkc);
+  check_string_option(&buf->b_p_menc);
 }
 
 /*
@@ -2635,8 +2636,8 @@ did_set_string_option (
   else if (varp == &p_ei) {
     if (check_ei() == FAIL)
       errmsg = e_invarg;
-  /* 'encoding' and 'fileencoding' */
-  } else if (varp == &p_enc || gvarp == &p_fenc) {
+  // 'encoding', 'fileencoding' and 'makeencoding'
+  } else if (varp == &p_enc || gvarp == &p_fenc || gvarp == &p_menc) {
     if (gvarp == &p_fenc) {
       if (!MODIFIABLE(curbuf) && opt_flags != OPT_GLOBAL) {
         errmsg = e_modifiable;
@@ -5400,6 +5401,9 @@ void unset_global_local_option(char *name, void *from)
     case PV_LW:
       clear_string_option(&buf->b_p_lw);
       break;
+    case PV_MENC:
+      clear_string_option(&buf->b_p_menc);
+      break;
   }
 }
 
@@ -5433,6 +5437,7 @@ static char_u *get_varp_scope(vimoption_T *p, int opt_flags)
     case PV_UL:   return (char_u *)&(curbuf->b_p_ul);
     case PV_LW:   return (char_u *)&(curbuf->b_p_lw);
     case PV_BKC:  return (char_u *)&(curbuf->b_p_bkc);
+    case PV_MENC: return (char_u *)&(curbuf->b_p_menc);
     }
     return NULL;     /* "cannot happen" */
   }
@@ -5488,6 +5493,8 @@ static char_u *get_varp(vimoption_T *p)
            ? (char_u *)&(curbuf->b_p_ul) : p->var;
   case PV_LW:   return *curbuf->b_p_lw != NUL
            ? (char_u *)&(curbuf->b_p_lw) : p->var;
+  case PV_MENC: return *curbuf->b_p_menc != NUL
+           ? (char_u *)&(curbuf->b_p_menc) : p->var;
 
   case PV_ARAB:   return (char_u *)&(curwin->w_p_arab);
   case PV_LIST:   return (char_u *)&(curwin->w_p_list);
@@ -5885,6 +5892,7 @@ void buf_copy_options(buf_T *buf, int flags)
       buf->b_p_qe = vim_strsave(p_qe);
       buf->b_p_udf = p_udf;
       buf->b_p_lw = empty_option;
+      buf->b_p_menc = empty_option;
 
       /*
        * Don't copy the options set by ex_help(), use the saved values,

src/nvim/option_defs.h

@@ -489,6 +489,7 @@ EXTERN char_u   *p_lcs;         // 'listchars'
 EXTERN int p_lz;                // 'lazyredraw'
 EXTERN int p_lpl;               // 'loadplugins'
 EXTERN int p_magic;             // 'magic'
+EXTERN char_u   *p_menc;        // 'makeencoding'
 EXTERN char_u   *p_mef;         // 'makeef'
 EXTERN char_u   *p_mp;          // 'makeprg'
 EXTERN char_u   *p_cc;          // 'colorcolumn'
@@ -736,6 +737,7 @@ enum {
   , BV_KP
   , BV_LISP
   , BV_LW
+  , BV_MENC
   , BV_MA
   , BV_ML
   , BV_MOD

src/nvim/options.lua

@@ -1456,6 +1456,13 @@ return {
       varname='p_mef',
       defaults={if_true={vi=""}}
     },
+    {
+      full_name='makeencoding', abbreviation='menc',
+      type='string', scope={'global', 'buffer'},
+      vi_def=true,
+      varname='p_menc',
+      defaults={if_true={vi=""}}
+    },
     {
       full_name='makeprg', abbreviation='mp',
       type='string', scope={'global', 'buffer'},

src/nvim/quickfix.c

@@ -160,6 +160,7 @@ typedef struct {
   buf_T      *buf;
   linenr_T buflnum;
   linenr_T lnumlast;
+  vimconv_T vc;
 } qfstate_T;
 
 typedef struct {
@@ -204,7 +205,8 @@ qf_init (
     char_u *efile,
     char_u *errorformat,
     int newlist,                            /* TRUE: start a new error list */
-    char_u *qf_title
+    char_u *qf_title,
+    char_u *enc
 )
 {
   qf_info_T       *qi = &ql_info;
@@ -214,8 +216,8 @@ qf_init (
   }
 
   return qf_init_ext(qi, efile, curbuf, NULL, errorformat, newlist,
-      (linenr_T)0, (linenr_T)0,
-      qf_title);
+                     (linenr_T)0, (linenr_T)0,
+                     qf_title, enc);
 }
 
 // Maximum number of bytes allowed per line while reading an errorfile.
@@ -638,6 +640,22 @@ retry:
   } else {
     state->linebuf = IObuff;
   }
+
+  // Convert a line if it contains a non-ASCII character
+  if (state->vc.vc_type != CONV_NONE && has_non_ascii(state->linebuf)) {
+    char_u *line = string_convert(&state->vc, state->linebuf, &state->linelen);
+    if (line != NULL) {
+      if (state->linelen < IOSIZE) {
+        STRLCPY(state->linebuf, line, state->linelen + 1);
+        xfree(line);
+      } else {
+        xfree(state->growbuf);
+        state->linebuf = state->growbuf = line;
+        state->growbufsiz = state->linelen < LINE_MAXLEN
+          ? state->linelen : LINE_MAXLEN;
+      }
+    }
+  }
   return QF_OK;
 }
 
@@ -979,12 +997,12 @@ qf_init_ext(
     int newlist,                            /* TRUE: start a new error list */
     linenr_T lnumfirst,                     /* first line number to use */
     linenr_T lnumlast,                      /* last line number to use */
-    char_u *qf_title
+    char_u *qf_title,
+    char_u *enc
 )
 {
-  qfstate_T state = { NULL, 0, NULL, 0, NULL, NULL, NULL, NULL,
-                      NULL, 0, 0 };
-  qffields_T fields = { NULL, NULL, 0, 0L, 0, false, NULL, 0, 0, 0 };
+  qfstate_T state;
+  qffields_T fields;
   qfline_T        *old_last = NULL;
   bool adding = false;
   static efm_T    *fmt_first = NULL;
@@ -997,6 +1015,13 @@ qf_init_ext(
   xfree(qf_last_bufname);
   qf_last_bufname = NULL;
 
+  memset(&state, 0, sizeof(state));
+  memset(&fields, 0, sizeof(fields));
+  state.vc.vc_type = CONV_NONE;
+  if (enc != NULL && *enc != NUL) {
+    convert_setup(&state.vc, enc, p_enc);
+  }
+
   fields.namebuf = xmalloc(CMDBUFFSIZE + 1);
   fields.errmsglen = CMDBUFFSIZE + 1;
   fields.errmsg = xmalloc(fields.errmsglen);
@@ -1147,6 +1172,10 @@ qf_init_end:
 
   qf_update_buffer(qi, old_last);
 
+  if (state.vc.vc_type != CONV_NONE) {
+    convert_setup(&state.vc, NULL, NULL);
+  }
+
   return retval;
 }
 
@@ -3024,6 +3053,7 @@ void ex_make(exarg_T *eap)
   qf_info_T   *qi = &ql_info;
   int res;
   char_u      *au_name = NULL;
+  char_u *enc = (*curbuf->b_p_menc != NUL) ? curbuf->b_p_menc : p_menc;
 
   /* Redirect ":grep" to ":vimgrep" if 'grepprg' is "internal". */
   if (grep_internal(eap->cmdidx)) {
@@ -3083,9 +3113,8 @@ void ex_make(exarg_T *eap)
 
   res = qf_init(wp, fname, (eap->cmdidx != CMD_make
                             && eap->cmdidx != CMD_lmake) ? p_gefm : p_efm,
-      (eap->cmdidx != CMD_grepadd
-       && eap->cmdidx != CMD_lgrepadd),
-      *eap->cmdlinep);
+                (eap->cmdidx != CMD_grepadd && eap->cmdidx != CMD_lgrepadd),
+                *eap->cmdlinep, enc);
   if (wp != NULL)
     qi = GET_LOC_LIST(wp);
   if (au_name != NULL) {
@@ -3429,6 +3458,7 @@ void ex_cfile(exarg_T *eap)
   if (*eap->arg != NUL)
     set_string_option_direct((char_u *)"ef", -1, eap->arg, OPT_FREE, 0);
 
+  char_u *enc = (*curbuf->b_p_menc != NUL) ? curbuf->b_p_menc : p_menc;
   /*
    * This function is used by the :cfile, :cgetfile and :caddfile
    * commands.
@@ -3441,7 +3471,7 @@ void ex_cfile(exarg_T *eap)
    */
   if (qf_init(wp, p_ef, p_efm, (eap->cmdidx != CMD_caddfile
                                 && eap->cmdidx != CMD_laddfile),
-          *eap->cmdlinep) > 0
+              *eap->cmdlinep, enc) > 0
       && (eap->cmdidx == CMD_cfile
           || eap->cmdidx == CMD_lfile)) {
     if (au_name != NULL)
@@ -4338,7 +4368,7 @@ void ex_cbuffer(exarg_T *eap)
       if (qf_init_ext(qi, NULL, buf, NULL, p_efm,
                       (eap->cmdidx != CMD_caddbuffer
                        && eap->cmdidx != CMD_laddbuffer),
-                      eap->line1, eap->line2, qf_title) > 0) {
+                      eap->line1, eap->line2, qf_title, NULL) > 0) {
         if (au_name != NULL) {
           apply_autocmds(EVENT_QUICKFIXCMDPOST, (char_u *)au_name,
                          curbuf->b_fname, true, curbuf);
@@ -4403,7 +4433,7 @@ void ex_cexpr(exarg_T *eap)
       if (qf_init_ext(qi, NULL, NULL, &tv, p_efm,
                       (eap->cmdidx != CMD_caddexpr
                        && eap->cmdidx != CMD_laddexpr),
-                      (linenr_T)0, (linenr_T)0, *eap->cmdlinep) > 0) {
+                      (linenr_T)0, (linenr_T)0, *eap->cmdlinep, NULL) > 0) {
         if (au_name != NULL) {
           apply_autocmds(EVENT_QUICKFIXCMDPOST, (char_u *)au_name,
                          curbuf->b_fname, true, curbuf);

src/nvim/testdir/Makefile

@@ -65,6 +65,7 @@ NEW_TESTS ?= \
 	    test_increment_dbcs.res \
 	    test_lambda.res \
 	    test_langmap.res \
+	    test_makeencoding.res \
 	    test_marks.res \
 	    test_match.res \
 	    test_matchadd_conceal.res \

src/nvim/testdir/test_makeencoding.py

@@ -0,0 +1,67 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# Test program for :make, :grep and :cgetfile.
+
+from __future__ import print_function, unicode_literals
+import locale
+import io
+import sys
+
+def set_output_encoding(enc=None):
+    """Set the encoding of stdout and stderr
+
+    arguments:
+      enc -- Encoding name.
+             If omitted, locale.getpreferredencoding() is used.
+    """
+    if enc is None:
+        enc = locale.getpreferredencoding()
+
+    def get_text_writer(fo, **kwargs):
+        kw = dict(kwargs)
+        kw.setdefault('errors', 'backslashreplace') # use \uXXXX style
+        kw.setdefault('closefd', False)
+
+        if sys.version_info[0] < 3:
+            # Work around for Python 2.x
+            # New line conversion isn't needed here. Done in somewhere else.
+            writer = io.open(fo.fileno(), mode='w', newline='', **kw)
+            write = writer.write    # save the original write() function
+            enc = locale.getpreferredencoding()
+            def convwrite(s):
+                if isinstance(s, bytes):
+                    write(s.decode(enc))    # convert to unistr
+                else:
+                    write(s)
+                try:
+                    writer.flush()  # needed on Windows
+                except IOError:
+                    pass
+            writer.write = convwrite
+        else:
+            writer = io.open(fo.fileno(), mode='w', **kw)
+        return writer
+
+    sys.stdout = get_text_writer(sys.stdout, encoding=enc)
+    sys.stderr = get_text_writer(sys.stderr, encoding=enc)
+
+
+def main():
+    enc = 'utf-8'
+    if len(sys.argv) > 1:
+        enc = sys.argv[1]
+    set_output_encoding(enc)
+
+    message_tbl = {
+            'utf-8': 'ÀÈÌÒÙ こんにちは 你好',
+            'latin1': 'ÀÈÌÒÙ',
+            'cp932': 'こんにちは',
+            'cp936': '你好',
+            }
+
+    print('Xfoobar.c(10) : %s (%s)' % (message_tbl[enc], enc))
+
+
+if __name__ == "__main__":
+    main()

src/nvim/testdir/test_makeencoding.vim

@@ -0,0 +1,106 @@
+" Tests for 'makeencoding'.
+if !has('multi_byte')
+  finish
+endif
+
+source shared.vim
+
+let s:python = PythonProg()
+if s:python == ''
+  " Can't run this test.
+  finish
+endif
+
+let s:script = 'test_makeencoding.py'
+
+let s:message_tbl = {
+      \ 'utf-8': 'ÀÈÌÒÙ こんにちは 你好',
+      \ 'latin1': 'ÀÈÌÒÙ',
+      \ 'cp932': 'こんにちは',
+      \ 'cp936': '你好',
+      \}
+
+
+" Tests for :cgetfile and :lgetfile.
+func Test_getfile()
+  set errorfile=Xerror.txt
+  set errorformat=%f(%l)\ :\ %m
+
+  " :cgetfile
+  for enc in keys(s:message_tbl)
+    let &makeencoding = enc
+    exec "silent !" . s:python . " " . s:script . " " . enc . " > " . &errorfile
+    cgetfile
+    copen
+    call assert_equal("Xfoobar.c|10| " . s:message_tbl[enc] . " (" . enc . ")",
+          \ getline('.'))
+    cclose
+  endfor
+
+  " :lgetfile
+  for enc in keys(s:message_tbl)
+    let &makeencoding = enc
+    exec "silent !" . s:python . " " . s:script . " " . enc . " > " . &errorfile
+    lgetfile
+    lopen
+    call assert_equal("Xfoobar.c|10| " . s:message_tbl[enc] . " (" . enc . ")",
+          \ getline('.'))
+    lclose
+  endfor
+
+  call delete(&errorfile)
+endfunc
+
+
+" Tests for :grep and :lgrep.
+func Test_grep()
+  let &grepprg = s:python
+  set grepformat=%f(%l)\ :\ %m
+
+  " :grep
+  for enc in keys(s:message_tbl)
+    let &makeencoding = enc
+    exec "silent grep! " . s:script . " " . enc
+    copen
+    call assert_equal("Xfoobar.c|10| " . s:message_tbl[enc] . " (" . enc . ")",
+          \ getline('.'))
+    cclose
+  endfor
+
+  " :lgrep
+  for enc in keys(s:message_tbl)
+    let &makeencoding = enc
+    exec "silent lgrep! " . s:script . " " . enc
+    lopen
+    call assert_equal("Xfoobar.c|10| " . s:message_tbl[enc] . " (" . enc . ")",
+          \ getline('.'))
+    lclose
+  endfor
+endfunc
+
+
+" Tests for :make and :lmake.
+func Test_make()
+  let &makeprg = s:python
+  set errorformat=%f(%l)\ :\ %m
+
+  " :make
+  for enc in keys(s:message_tbl)
+    let &makeencoding = enc
+    exec "silent make! " . s:script . " " . enc
+    copen
+    call assert_equal("Xfoobar.c|10| " . s:message_tbl[enc] . " (" . enc . ")",
+          \ getline('.'))
+    cclose
+  endfor
+
+  " :lmake
+  for enc in keys(s:message_tbl)
+    let &makeencoding = enc
+    exec "silent lmake! " . s:script . " " . enc
+    lopen
+    call assert_equal("Xfoobar.c|10| " . s:message_tbl[enc] . " (" . enc . ")",
+          \ getline('.'))
+    lclose
+  endfor
+endfunc