Use lua generator in place of ex_cmds_defs header trick

Closes #788
Fixes #379
Ref #549

scripts/genex_cmds.lua

@@ -0,0 +1,87 @@
+local nvimsrcdir = arg[1]
+local includedir = arg[2]
+local autodir = arg[3]
+
+if nvimsrcdir == '--help' then
+  print ([[
+Usage:
+  lua genex_cmds.lua src/nvim build/include build/src/nvim/auto
+
+Will generate files build/include/ex_cmds_enum.generated.h with cmdidx_T 
+enum and build/src/nvim/auto/ex_cmds_defs.generated.h with main Ex commands 
+definitions.
+]])
+  os.exit(0)
+end
+
+package.path = nvimsrcdir .. '/?.lua;' .. package.path
+
+local enumfname = includedir .. '/ex_cmds_enum.generated.h'
+local defsfname = autodir .. '/ex_cmds_defs.generated.h'
+
+local enumfile = io.open(enumfname, 'w')
+local defsfile = io.open(defsfname, 'w')
+
+local defs = require('ex_cmds')
+local lastchar = nil
+
+local i
+local cmd
+local first = true
+local prevfirstchar = nil
+
+local byte_a = string.byte('a')
+local byte_z = string.byte('z')
+
+local cmdidxs = string.format([[
+static cmdidx_T cmdidxs[%u] = {
+]], byte_z - byte_a + 2)
+
+enumfile:write([[
+typedef enum CMD_index {
+]])
+defsfile:write([[
+static CommandDefinition cmdnames[] = {
+]])
+for i, cmd in ipairs(defs) do
+  local enumname = cmd.enum or ('CMD_' .. cmd.command)
+  firstchar = string.byte(cmd.command)
+  if firstchar ~= prevfirstchar then
+    if (not prevfirstchar
+        or (byte_a <= firstchar      and firstchar     <= byte_z)
+        or (byte_a <= prevfirstchar  and prevfirstchar <= byte_z)) then
+      if not first then
+        cmdidxs = cmdidxs .. ',\n'
+      end
+      cmdidxs = cmdidxs .. '  ' .. enumname
+    end
+    prevfirstchar = firstchar
+  end
+  if first then
+    first = false
+  else
+    defsfile:write(',\n')
+  end
+  enumfile:write('  ' .. enumname .. ',\n')
+  defsfile:write(string.format([[
+  [%s] = {
+    .cmd_name = (char_u *) "%s",
+    .cmd_func = &%s,
+    .cmd_argt = %uL
+  }]], enumname, cmd.command, cmd.func, cmd.flags))
+end
+defsfile:write([[
+
+};
+]])
+enumfile:write([[
+  CMD_SIZE,
+  CMD_USER = -1,
+  CMD_USER_BUF = -2
+} cmdidx_T;
+]])
+cmdidxs = cmdidxs .. [[
+
+};
+]]
+defsfile:write(cmdidxs)

scripts/transform_ex_cmds_defs.pl

@@ -0,0 +1,70 @@
+#!/usr/bin/perl
+
+use strict;
+use warnings;
+no utf8;
+
+my $srcfname = shift @ARGV;
+my $tgtfname = shift @ARGV;
+
+if ($srcfname eq "--help") {
+  print << "EOF";
+$0 path/to/ex_cmds_defs.h path/to/ex_cmds.lua
+
+  Translate Ex commands defines from old format to new one.
+EOF
+}
+
+my $flag_const = qr((?:[A-Z]+[A-Z1-9]));
+
+my $F;
+open $F, "<", "$srcfname"
+  or die "Failed to open $srcfname";
+
+my $T;
+open $T, ">", "$tgtfname"
+  or die "Failed to open $tgtfname";
+
+my ($enum_name, $cmd_name, $cmd_func);
+my ($flags);
+my $did_header = 0;
+while (<$F>) {
+  if (/^#define\s+($flag_const)\s+(.*)/) {
+    my $const_name = $1;
+    my $const_value = $2;
+    $const_value =~ s@/\*.*|//.*@@g;
+    $const_value =~ s/\s+//g;
+    $const_value =~ s/\|/+/g;
+    $const_value =~ s/(?<=\d)L//g;
+    print $T "local $const_name = $const_value\n";
+  } elsif (/^\{/../^\}/) {
+    if (/^\s*EX\(\s*CMD_(\w+)\s*,\s*"((?:\\.|[^"\\])+)"\s*,\s*(\w+)\s*,\s*$/) {
+      $enum_name = $1;
+      $cmd_name = $2;
+      $cmd_func = $3;
+    } elsif (defined $enum_name and /^\s*($flag_const(?:\|$flag_const)*|0)/) {
+      if (not $did_header) {
+        print $T "return {\n";
+        $did_header = 1;
+      }
+      $flags = $1;
+      $flags =~ s/\|/+/g;
+      local $\ = "\n";
+      print $T "  {";
+      print $T "    command='$cmd_name',";
+      print $T "    enum='CMD_$enum_name',"
+        if ($cmd_name ne $enum_name);
+      print $T "    flags=$flags,";
+      print $T "    func='$cmd_func',";
+      print $T "  },";
+      undef $enum_name;
+      undef $cmd_name;
+      undef $cmd_func;
+      undef $flags;
+    } else {
+      next;
+    }
+  }
+}
+
+print $T "}\n";

src/nvim/CMakeLists.txt

@@ -7,6 +7,10 @@ set(MSGPACK_RPC_HEADER ${PROJECT_SOURCE_DIR}/src/nvim/os/msgpack_rpc.h)
 set(MSGPACK_DISPATCH ${GENERATED_DIR}/msgpack_dispatch.c)
 set(HEADER_GENERATOR ${PROJECT_SOURCE_DIR}/scripts/gendeclarations.lua)
 set(GENERATED_INCLUDES_DIR ${PROJECT_BINARY_DIR}/include)
+set(GENERATED_EX_CMDS_ENUM ${GENERATED_INCLUDES_DIR}/ex_cmds_enum.generated.h)
+set(GENERATED_EX_CMDS_DEFS ${GENERATED_DIR}/ex_cmds_defs.generated.h)
+set(EX_CMDS_GENERATOR ${PROJECT_SOURCE_DIR}/scripts/genex_cmds.lua)
+set(EX_CMDS_DEFS_FILE ${PROJECT_SOURCE_DIR}/src/nvim/ex_cmds.lua)
 
 include_directories(${GENERATED_DIR})
 include_directories(${GENERATED_INCLUDES_DIR})
@@ -124,11 +128,20 @@ add_custom_command(OUTPUT ${MSGPACK_DISPATCH}
     ${API_HEADERS}
     ${MSGPACK_RPC_HEADER}
     ${DISPATCH_GENERATOR}
-    )
+)
 
 list(APPEND NEOVIM_GENERATED_SOURCES
   "${PROJECT_BINARY_DIR}/config/auto/pathdef.c"
-  "${MSGPACK_DISPATCH}")
+  "${MSGPACK_DISPATCH}"
+  "${GENERATED_EX_CMDS_ENUM}"
+  "${GENERATED_EX_CMDS_DEFS}"
+)
+
+add_custom_command(OUTPUT ${GENERATED_EX_CMDS_ENUM} ${GENERATED_EX_CMDS_DEFS}
+  COMMAND ${LUA_PRG} ${EX_CMDS_GENERATOR}
+      ${PROJECT_SOURCE_DIR}/src/nvim ${GENERATED_INCLUDES_DIR} ${GENERATED_DIR}
+  DEPENDS ${EX_CMDS_GENERATOR} ${EX_CMDS_DEFS_FILE}
+)
 
 # Our dependencies come first.
 

src/nvim/ex_docmd.c

@@ -66,6 +66,7 @@
 #include "nvim/version.h"
 #include "nvim/window.h"
 #include "nvim/os/os.h"
+#include "nvim/ex_cmds_defs.h"
 
 static int quitmore = 0;
 static int ex_pressedreturn = FALSE;
@@ -156,42 +157,9 @@ static int did_lcd;             /* whether ":lcd" was produced for a session */
 /*
  * Declare cmdnames[].
  */
-#define DO_DECLARE_EXCMD
-#include "nvim/ex_cmds_defs.h"
-
-/*
- * Table used to quickly search for a command, based on its first character.
- */
-static cmdidx_T cmdidxs[27] =
-{
-  CMD_append,
-  CMD_buffer,
-  CMD_change,
-  CMD_delete,
-  CMD_edit,
-  CMD_file,
-  CMD_global,
-  CMD_help,
-  CMD_insert,
-  CMD_join,
-  CMD_k,
-  CMD_list,
-  CMD_move,
-  CMD_next,
-  CMD_open,
-  CMD_print,
-  CMD_quit,
-  CMD_read,
-  CMD_substitute,
-  CMD_t,
-  CMD_undo,
-  CMD_vglobal,
-  CMD_write,
-  CMD_xit,
-  CMD_yank,
-  CMD_z,
-  CMD_bang
-};
+#ifdef INCLUDE_GENERATED_DECLARATIONS
+# include "ex_cmds_defs.generated.h"
+#endif
 
 static char_u dollar_command[2] = {'$', 0};
 

src/nvim/ex_docmd.h

@@ -1,6 +1,8 @@
 #ifndef NVIM_EX_DOCMD_H
 #define NVIM_EX_DOCMD_H
 
+#include "nvim/ex_cmds_defs.h"
+
 /* flags for do_cmdline() */
 #define DOCMD_VERBOSE   0x01    /* included command in error message */
 #define DOCMD_NOWAIT    0x02    /* don't call wait_return() and friends */
@@ -17,8 +19,6 @@
 #define EXMODE_NORMAL           1
 #define EXMODE_VIM              2
 
-typedef char_u *(*LineGetter)(int, void *, int);
-
 #ifdef INCLUDE_GENERATED_DECLARATIONS
 # include "ex_docmd.h.generated.h"
 #endif