vim-patch:8.0.0251: not easy to select Python 2 or 3 (#9173)

Problem: It is not so easy to write a script that works with both Python 2 and Python 3, even when the Python code works with both.
Solution: Add 'pyxversion', :pyx, etc. (Marc Weber, Ken Takata)

f42dd3c390

runtime/doc/eval.txt

@@ -2209,6 +2209,7 @@ printf({fmt}, {expr1}...)	String	format text
 pumvisible()			Number	whether popup menu is visible
 pyeval({expr})			any	evaluate |Python| expression
 py3eval({expr})			any	evaluate |python3| expression
+pyxeval({expr})			any	evaluate |python_x| expression
 range({expr} [, {max} [, {stride}]])
 				List	items from {expr} to {max}
 readfile({fname} [, {binary} [, {max}]])
@@ -6146,6 +6147,12 @@ pyeval({expr})						*pyeval()*
 		non-string keys result in error.
 		{only available when compiled with the |+python| feature}
 
+pyxeval({expr})						*pyxeval()*
+		Evaluate Python expression {expr} and return its result
+		converted to Vim data structures.
+		Uses Python 2 or 3, see |python_x| and 'pyxversion'.
+		See also: |pyeval()|, |py3eval()|
+
 							*E726* *E727*
 range({expr} [, {max} [, {stride}]])				*range()*
 		Returns a |List| with Numbers:
@@ -8517,6 +8524,7 @@ printer			Compiled with |:hardcopy| support.
 profile			Compiled with |:profile| support.
 python			Legacy Vim Python 2.x API is available. |has-python|
 python3			Legacy Vim Python 3.x API is available. |has-python|
+pythonx			Compiled with |python_x| interface. |has-pythonx|
 quickfix		Compiled with |quickfix| support.
 reltime			Compiled with |reltime()| support.
 rightleft		Compiled with 'rightleft' support.

runtime/doc/if_pyth.txt

@@ -692,6 +692,7 @@ vim.Function object				*python-Function*
 
 To facilitate bi-directional interface, you can use |pyeval()| and |py3eval()| 
 functions to evaluate Python expressions and pass their values to Vim script.
+|pyxeval()| is also available.
 
 ==============================================================================
 9. Python 3						*python3*
@@ -722,5 +723,58 @@ You can test what Python version is available with: >
 	  echo 'there is Python 3.x'
 	endif
 
+==============================================================================
+10. Python X						*python_x* *pythonx*
+
+Because most python code can be written so that it works with Python 2.6+ and
+Python 3, the pyx* functions and commands have been written.  They work the
+same as the Python 2 and 3 variants, but select the Python version using the
+'pyxversion' setting.
+
+Set 'pyxversion' in your |vimrc| to prefer Python 2 or Python 3 for Python
+commands. Changing this setting at runtime risks losing the state of plugins
+(such as initialization).
+
+If you want to use a module, you can put it in the {rtp}/pythonx directory.
+See |pythonx-directory|.
+
+							*:pyx* *:pythonx*
+`:pyx` and `:pythonx` work similar to `:python`.  To check if `:pyx` works: >
+	:pyx print("Hello")
+
+To see what version of Python is being used: >
+	:pyx import sys
+	:pyx print(sys.version)
+<
+					*:pyxfile* *python_x-special-comments*
+`:pyxfile` works similar to `:pyfile`.  But you can add a "shebang" comment to
+force Vim to use `:pyfile` or `:py3file`: >
+  #!/any string/python2		" Shebang. Must be the first line of the file.
+  #!/any string/python3		" Shebang. Must be the first line of the file.
+  # requires python 2.x		" Maximum lines depend on 'modelines'.
+  # requires python 3.x		" Maximum lines depend on 'modelines'.
+Unlike normal modelines, the bottom of the file is not checked.
+If none of them are found, the 'pyxversion' option is used.
+							*W20* *W21*
+If Vim does not support the selected Python version a silent message will be
+printed.  Use `:messages` to read them.
+
+							*:pyxdo*
+`:pyxdo` works similar to `:pydo`.
+
+							*has-pythonx*
+To check if pyx* functions and commands are available: >
+	if has('pythonx')
+	  echo 'pyx* commands are available. (Python ' . &pyx . ')'
+	endif
+
+If you prefer Python 2 and want to fallback to Python 3, set 'pyxversion'
+explicitly in your |.vimrc|.  Example: >
+	if has('python')
+	  set pyx=2
+	elseif has('python3')
+	  set pyx=3
+	endif
+
 ==============================================================================
  vim:tw=78:ts=8:noet:ft=help:norl:

runtime/doc/index.txt

@@ -1418,6 +1418,10 @@ tag	      command	      action ~
 |:python|	:py[thon]	execute Python command
 |:pydo|		:pyd[o]		execute Python command for each line
 |:pyfile|	:pyf[ile]	execute Python script file
+|:pyx|		:pyx		execute |python_x| command
+|:pythonx|	:pythonx	same as :pyx
+|:pyxdo|	:pyxd[o]	execute |python_x| command for each line
+|:pyxfile|	:pyxf[ile]	execute |python_x| script file
 |:quit|		:q[uit]		quit current window (when one window quit Vim)
 |:quitall|	:quita[ll]	quit Vim
 |:qall|		:qa[ll]		quit Vim

runtime/doc/options.txt

@@ -4471,6 +4471,30 @@ A jump table for the options with a short description can be found at |Q_op|.
 	Insert mode completion.  When zero as much space as available is used.
 	|ins-completion-menu|.
 
+						*'pyxversion'* *'pyx'*
+'pyxversion' 'pyx'	number	(default depends on the build)
+			global
+	Specifies the python version used for pyx* functions and commands
+	|python_x|.  The default value is as follows:
+
+		|provider| installed	    Default ~
+		|+python| and |+python3|	0
+		only |+python|			2
+		only |+python3|			3
+
+	Available values are 0, 2 and 3.
+	If 'pyxversion' is 0, it is set to 2 or 3 after the first execution of
+	any python2/3 commands or functions.  E.g. `:py` sets to 2, and `:py3`
+	sets to 3. `:pyx` sets it to 3 if Python 3 is available, otherwise sets
+	to 2 if Python 2 is available.
+	See also: |has-pythonx|
+
+	If only |+python| or |+python3| are available,
+	'pyxversion' has no effect. The pyx* functions and commands are
+	always the same as the installed version.
+
+	This option cannot be set from a |modeline| or in the |sandbox|, for
+	security reasons.
 
 						*'quoteescape'* *'qe'*
 'quoteescape' 'qe'	string	(default "\")

runtime/doc/quickref.txt

@@ -813,6 +813,7 @@ Short explanation of each option:		*option-list*
 'pumwidth'	  'pw'	    minimum width of the popup menu
 'pythondll'		    name of the Python 2 dynamic library
 'pythonthreedll'	    name of the Python 3 dynamic library
+'pyxversion'	  'pyx'	    Python version used for pyx* commands
 'quoteescape'	  'qe'	    escape characters used in a string
 'readonly'	  'ro'	    disallow writing the buffer
 'redrawtime'	  'rdt'     timeout for 'hlsearch' and |:match| highlighting

runtime/optwin.vim

@@ -905,7 +905,7 @@ if has("folding")
   call append("$", "foldmarker\tmarkers used when 'foldmethod' is \"marker\"")
   call append("$", "\t(local to window)")
   call <SID>OptionL("fmr")
-  call append("$", "foldnestmax\tmaximum fold depth for when 'foldmethod is \"indent\" or \"syntax\"")
+  call append("$", "foldnestmax\tmaximum fold depth for when 'foldmethod' is \"indent\" or \"syntax\"")
   call append("$", "\t(local to window)")
   call <SID>OptionL("fdn")
 endif
@@ -1295,6 +1295,10 @@ if exists("&mzschemedll")
   call append("$", "mzschemegcdll\tname of the Tcl GC dynamic library")
   call <SID>OptionG("mzschemegcdll", &mzschemegcdll)
 endif
+if has('pythonx')
+  call append("$", "pyxversion\twhether to use Python 2 or 3")
+  call append("$", " \tset pyx=" . &wd)
+endif
 
 " Install autocommands to enable mappings in option-window
 noremap <silent> <buffer> <CR> <C-\><C-N>:call <SID>CR()<CR>

src/nvim/eval.c

@@ -10704,6 +10704,7 @@ static void f_has(typval_T *argvars, typval_T *rettv, FunPtr fptr)
     "postscript",
     "printer",
     "profile",
+    "pythonx",
     "reltime",
     "quickfix",
     "rightleft",
@@ -13026,6 +13027,10 @@ static void f_pumvisible(typval_T *argvars, typval_T *rettv, FunPtr fptr)
  */
 static void f_pyeval(typval_T *argvars, typval_T *rettv, FunPtr fptr)
 {
+  if (p_pyx == 0) {
+      p_pyx = 2;
+  }
+
   script_host_eval("python", argvars, rettv);
 }
 
@@ -13034,9 +13039,24 @@ static void f_pyeval(typval_T *argvars, typval_T *rettv, FunPtr fptr)
  */
 static void f_py3eval(typval_T *argvars, typval_T *rettv, FunPtr fptr)
 {
+  if (p_pyx == 0) {
+      p_pyx = 3;
+  }
+
   script_host_eval("python3", argvars, rettv);
 }
 
+// "pyxeval()" function
+static void f_pyxeval(typval_T *argvars, typval_T *rettv, FunPtr fptr)
+{
+  init_pyxversion();
+  if (p_pyx == 2) {
+    f_pyeval(argvars, rettv, NULL);
+  } else {
+    f_py3eval(argvars, rettv, NULL);
+  }
+}
+
 /*
  * "range()" function
  */
@@ -20133,7 +20153,9 @@ void ex_function(exarg_T *eap)
       arg = skipwhite(skiptowhite(p));
       if (arg[0] == '<' && arg[1] =='<'
           && ((p[0] == 'p' && p[1] == 'y'
-               && (!ASCII_ISALPHA(p[2]) || p[2] == 't'))
+               && (!ASCII_ISALNUM(p[2]) || p[2] == 't'
+                   || ((p[2] == '3' || p[2] == 'x')
+                       && !ASCII_ISALPHA(p[3]))))
               || (p[0] == 'p' && p[1] == 'e'
                   && (!ASCII_ISALPHA(p[2]) || p[2] == 'r'))
               || (p[0] == 't' && p[1] == 'c'

src/nvim/eval.lua

@@ -226,6 +226,7 @@ return {
     pumvisible={},
     py3eval={args=1},
     pyeval={args=1},
+    pyxeval={args=1},
     range={args={1, 3}},
     readfile={args={1, 3}},
     reltime={args={0, 2}},

src/nvim/ex_cmds.lua

@@ -2066,6 +2066,30 @@ return {
     addr_type=ADDR_LINES,
     func='ex_py3file',
   },
+  {
+    command='pyx',
+    flags=bit.bor(RANGE, EXTRA, NEEDARG, CMDWIN),
+    addr_type=ADDR_LINES,
+    func='ex_pyx',
+  },
+  {
+    command='pyxdo',
+    flags=bit.bor(RANGE, DFLALL, EXTRA, NEEDARG, CMDWIN),
+    addr_type=ADDR_LINES,
+    func='ex_pyxdo',
+  },
+  {
+    command='pythonx',
+    flags=bit.bor(RANGE, EXTRA, NEEDARG, CMDWIN),
+    addr_type=ADDR_LINES,
+    func='ex_pyx',
+  },
+  {
+    command='pyxfile',
+    flags=bit.bor(RANGE, FILE1, NEEDARG, CMDWIN),
+    addr_type=ADDR_LINES,
+    func='ex_pyxfile',
+  },
   {
     command='quit',
     flags=bit.bor(BANG, RANGE, COUNT, NOTADR, TRLBAR, CMDWIN),

src/nvim/ex_cmds2.c

@@ -2809,6 +2809,126 @@ void ex_options(exarg_T *eap)
   cmd_source((char_u *)SYS_OPTWIN_FILE, NULL);
 }
 
+// Detect Python 3 or 2, and initialize 'pyxversion'.
+void init_pyxversion(void)
+{
+  if (p_pyx == 0) {
+    if (!eval_has_provider("python3")) {
+      p_pyx = 3;
+    } else if (!eval_has_provider("python")) {
+      p_pyx = 2;
+    }
+  }
+}
+
+// Does a file contain one of the following strings at the beginning of any
+// line?
+// "#!(any string)python2"  => returns 2
+// "#!(any string)python3"  => returns 3
+// "# requires python 2.x"  => returns 2
+// "# requires python 3.x"  => returns 3
+// otherwise return 0.
+static int requires_py_version(char_u *filename)
+{
+  FILE      *file;
+  int       requires_py_version = 0;
+  int       i, lines;
+
+  lines = (int)p_mls;
+  if (lines < 0) {
+    lines = 5;
+  }
+
+  file = mch_fopen((char *)filename, "r");
+  if (file != NULL) {
+    for (i = 0; i < lines; i++) {
+      if (vim_fgets(IObuff, IOSIZE, file)) {
+        break;
+      }
+      if (i == 0 && IObuff[0] == '#' && IObuff[1] == '!') {
+        // Check shebang.
+        if (strstr((char *)IObuff + 2, "python2") != NULL) {
+          requires_py_version = 2;
+          break;
+        }
+        if (strstr((char *)IObuff + 2, "python3") != NULL) {
+          requires_py_version = 3;
+          break;
+        }
+      }
+      IObuff[21] = '\0';
+      if (STRCMP("# requires python 2.x", IObuff) == 0) {
+        requires_py_version = 2;
+        break;
+      }
+      if (STRCMP("# requires python 3.x", IObuff) == 0) {
+        requires_py_version = 3;
+        break;
+      }
+    }
+    fclose(file);
+  }
+  return requires_py_version;
+}
+
+
+// Source a python file using the requested python version.
+static void source_pyx_file(exarg_T *eap, char_u *fname)
+{
+  exarg_T ex;
+  long int v = requires_py_version(fname);
+
+  init_pyxversion();
+  if (v == 0) {
+    // user didn't choose a preference, 'pyx' is used
+    v = p_pyx;
+  }
+
+  // now source, if required python version is not supported show
+  // unobtrusive message.
+  if (eap == NULL) {
+    memset(&ex, 0, sizeof(ex));
+  } else {
+    ex = *eap;
+  }
+  ex.arg = fname;
+  ex.cmd = (char_u *)(v == 2 ? "pyfile" : "pyfile3");
+
+  if (v == 2) {
+    ex_pyfile(&ex);
+  } else {
+    ex_py3file(&ex);
+  }
+}
+
+// ":pyxfile {fname}"
+void ex_pyxfile(exarg_T *eap)
+{
+  source_pyx_file(eap, eap->arg);
+}
+
+// ":pyx"
+void ex_pyx(exarg_T *eap)
+{
+  init_pyxversion();
+  if (p_pyx == 2) {
+    ex_python(eap);
+  } else {
+    ex_python3(eap);
+  }
+}
+
+// ":pyxdo"
+void ex_pyxdo(exarg_T *eap)
+{
+  init_pyxversion();
+  if (p_pyx == 2) {
+    ex_pydo(eap);
+  } else {
+    ex_pydo3(eap);
+  }
+}
+
 /// ":source {fname}"
 void ex_source(exarg_T *eap)
 {

src/nvim/ex_docmd.c

@@ -2158,6 +2158,10 @@ static char_u * do_one_cmd(char_u **cmdlinep,
     case CMD_python:
     case CMD_py3:
     case CMD_python3:
+    case CMD_pythonx:
+    case CMD_pyx:
+    case CMD_pyxdo:
+    case CMD_pyxfile:
     case CMD_return:
     case CMD_rightbelow:
     case CMD_ruby:

src/nvim/option.c

@@ -4322,6 +4322,10 @@ static char *set_num_option(int opt_idx, char_u *varp, long value,
     if (p_uc && !old_value) {
       ml_open_files();
     }
+  } else if (pp == &p_pyx) {
+    if (p_pyx != 0 && p_pyx != 2 && p_pyx != 3) {
+      errmsg = e_invarg;
+    }
   } else if (pp == &p_ul || pp == &curbuf->b_p_ul) {
     // sync undo before 'undolevels' changes
     // use the old value, otherwise u_sync() may not work properly

src/nvim/option_defs.h

@@ -516,6 +516,7 @@ EXTERN char_u   *p_pex;         // 'patchexpr'
 EXTERN char_u   *p_pm;          // 'patchmode'
 EXTERN char_u   *p_path;        // 'path'
 EXTERN char_u   *p_cdpath;      // 'cdpath'
+EXTERN long p_pyx;              // 'pyxversion'
 EXTERN long p_rdt;              // 'redrawtime'
 EXTERN int p_remap;             // 'remap'
 EXTERN long p_re;               // 'regexpengine'

src/nvim/options.lua

@@ -1809,6 +1809,14 @@ return {
       varname='p_ph',
       defaults={if_true={vi=0}}
     },
+    {
+      full_name='pyxversion', abbreviation='pyx',
+      type='number', scope={'global'},
+      secure=true,
+      vi_def=true,
+      varname='p_pyx',
+      defaults={if_true={vi=0}}
+    },
     {
       full_name='quoteescape', abbreviation='qe',
       type='string', scope={'buffer'},

src/nvim/testdir/pyxfile/py2_magic.py

@@ -0,0 +1,4 @@
+# requires python 2.x
+
+import sys
+print(sys.version)

src/nvim/testdir/pyxfile/py2_shebang.py

@@ -0,0 +1,4 @@
+#!/usr/bin/python2
+
+import sys
+print(sys.version)

src/nvim/testdir/pyxfile/py3_magic.py

@@ -0,0 +1,4 @@
+# requires python 3.x
+
+import sys
+print(sys.version)

src/nvim/testdir/pyxfile/py3_shebang.py

@@ -0,0 +1,4 @@
+#!/usr/bin/python3
+
+import sys
+print(sys.version)

src/nvim/testdir/pyxfile/pyx.py

@@ -0,0 +1,2 @@
+import sys
+print(sys.version)

src/nvim/testdir/test_pyx2.vim

@@ -0,0 +1,74 @@
+" Test for pyx* commands and functions with Python 2.
+
+set pyx=2
+if !has('python')
+  finish
+endif
+
+let s:py2pattern = '^2\.[0-7]\.\d\+'
+let s:py3pattern = '^3\.\d\+\.\d\+'
+
+
+func Test_has_pythonx()
+  call assert_true(has('pythonx'))
+endfunc
+
+
+func Test_pyx()
+  redir => var
+  pyx << EOF
+import sys
+print(sys.version)
+EOF
+  redir END
+  call assert_match(s:py2pattern, split(var)[0])
+endfunc
+
+
+func Test_pyxdo()
+  pyx import sys
+  enew
+  pyxdo return sys.version.split("\n")[0]
+  call assert_match(s:py2pattern, split(getline('.'))[0])
+endfunc
+
+
+func Test_pyxeval()
+  pyx import sys
+  call assert_match(s:py2pattern, split(pyxeval('sys.version'))[0])
+endfunc
+
+
+func Test_pyxfile()
+  " No special comments nor shebangs
+  redir => var
+  pyxfile pyxfile/pyx.py
+  redir END
+  call assert_match(s:py2pattern, split(var)[0])
+
+  " Python 2 special comment
+  redir => var
+  pyxfile pyxfile/py2_magic.py
+  redir END
+  call assert_match(s:py2pattern, split(var)[0])
+
+  " Python 2 shebang
+  redir => var
+  pyxfile pyxfile/py2_shebang.py
+  redir END
+  call assert_match(s:py2pattern, split(var)[0])
+
+  if has('python3')
+    " Python 3 special comment
+    redir => var
+    pyxfile pyxfile/py3_magic.py
+    redir END
+    call assert_match(s:py3pattern, split(var)[0])
+
+    " Python 3 shebang
+    redir => var
+    pyxfile pyxfile/py3_shebang.py
+    redir END
+    call assert_match(s:py3pattern, split(var)[0])
+  endif
+endfunc

src/nvim/testdir/test_pyx3.vim

@@ -0,0 +1,74 @@
+" Test for pyx* commands and functions with Python 3.
+
+set pyx=3
+if !has('python3')
+  finish
+endif
+
+let s:py2pattern = '^2\.[0-7]\.\d\+'
+let s:py3pattern = '^3\.\d\+\.\d\+'
+
+
+func Test_has_pythonx()
+  call assert_true(has('pythonx'))
+endfunc
+
+
+func Test_pyx()
+  redir => var
+  pyx << EOF
+import sys
+print(sys.version)
+EOF
+  redir END
+  call assert_match(s:py3pattern, split(var)[0])
+endfunc
+
+
+func Test_pyxdo()
+  pyx import sys
+  enew
+  pyxdo return sys.version.split("\n")[0]
+  call assert_match(s:py3pattern, split(getline('.'))[0])
+endfunc
+
+
+func Test_pyxeval()
+  pyx import sys
+  call assert_match(s:py3pattern, split(pyxeval('sys.version'))[0])
+endfunc
+
+
+func Test_pyxfile()
+  " No special comments nor shebangs
+  redir => var
+  pyxfile pyxfile/pyx.py
+  redir END
+  call assert_match(s:py3pattern, split(var)[0])
+
+  " Python 3 special comment
+  redir => var
+  pyxfile pyxfile/py3_magic.py
+  redir END
+  call assert_match(s:py3pattern, split(var)[0])
+
+  " Python 3 shebang
+  redir => var
+  pyxfile pyxfile/py3_shebang.py
+  redir END
+  call assert_match(s:py3pattern, split(var)[0])
+
+  if has('python')
+    " Python 2 special comment
+    redir => var
+    pyxfile pyxfile/py2_magic.py
+    redir END
+    call assert_match(s:py2pattern, split(var)[0])
+
+    " Python 2 shebang
+    redir => var
+    pyxfile pyxfile/py2_shebang.py
+    redir END
+    call assert_match(s:py2pattern, split(var)[0])
+  endif
+endfunc