gen_vimdoc.py: get Lua docs via lua2dox.lua #9740

scripts/gen_vimdoc.py

@@ -48,26 +48,50 @@ DEBUG = ('DEBUG' in os.environ)
 INCLUDE_C_DECL = ('INCLUDE_C_DECL' in os.environ)
 INCLUDE_DEPRECATED = ('INCLUDE_DEPRECATED' in os.environ)
 
-doc_filename = 'api.txt'
+text_width = 78
-# String used to find the start of the generated part of the doc.
+script_path = os.path.abspath(__file__)
-section_start_token = '*api-global*'
+base_dir = os.path.dirname(os.path.dirname(script_path))
-# Required prefix for API function names.
+out_dir = os.path.join(base_dir, 'tmp-{mode}-doc')
-api_func_name_prefix = 'nvim_'
+filter_cmd = '%s %s' % (sys.executable, script_path)
+seen_funcs = set()
+lua2dox_filter = os.path.join(base_dir, 'scripts', 'lua2dox_filter')
 
+CONFIG = {
+  'api': {
+    'filename': 'api.txt',
+    # String used to find the start of the generated part of the doc.
+    'section_start_token': '*api-global*',
+    # Section ordering.
+    'section_order' : [
+      'vim.c',
+      'buffer.c',
+      'window.c',
+      'tabpage.c',
+      'ui.c',
+    ],
+    # List of files/directories for doxygen to read, separated by blanks
+    'files': os.path.join(base_dir, 'src/nvim/api'), 
+    # file patterns used by doxygen
+    'file_patterns': '*.h *.c',
+    # Only function with this prefix are considered
+    'func_name_prefix': 'nvim_',
+  },
+  'lua': {
+    'filename': 'if_lua.txt',
+    'section_start_token': '*vim-lua*',
+    'section_order' : [
+      'vim.lua',
+    ],
+    'files': os.path.join(base_dir, 'src/nvim/lua/vim.lua'), 
+    'file_patterns': '*.lua',
+    'func_name_prefix': '',
+  }
+}
 # Section name overrides.
 section_name = {
     'vim.c': 'Global',
 }
 
-# Section ordering.
-section_order = (
-    'vim.c',
-    'buffer.c',
-    'window.c',
-    'tabpage.c',
-    'ui.c',
-)
-
 param_exclude = (
     'channel_id',
 )
@@ -77,13 +101,6 @@ annotation_map = {
     'FUNC_API_ASYNC': '{async}',
 }
 
-text_width = 78
-script_path = os.path.abspath(__file__)
-base_dir = os.path.dirname(os.path.dirname(script_path))
-src_dir = os.path.join(base_dir, 'src/nvim/api')
-out_dir = os.path.join(base_dir, 'tmp-api-doc')
-filter_cmd = '%s %s' % (sys.executable, script_path)
-seen_funcs = set()
 
 # Tracks `xrefsect` titles.  As of this writing, used only for separating
 # deprecated functions.
@@ -400,7 +417,7 @@ def parse_parblock(parent, prefix='', width=62, indent=''):
 # }}}
 
 
-def parse_source_xml(filename):
+def parse_source_xml(filename, mode):
     """Collects API functions.
 
     Returns two strings:
@@ -415,9 +432,12 @@ def parse_source_xml(filename):
     deprecated_functions = []
 
     dom = minidom.parse(filename)
+    compoundname = get_text(dom.getElementsByTagName('compoundname')[0])
     for member in dom.getElementsByTagName('memberdef'):
         if member.getAttribute('static') == 'yes' or \
-                member.getAttribute('kind') != 'function':
+                member.getAttribute('kind') != 'function' or \
+                member.getAttribute('prot') == 'private' or \
+                get_text(get_child(member, 'name')).startswith('_'):
             continue
 
         loc = find_first(member, 'location')
@@ -444,7 +464,12 @@ def parse_source_xml(filename):
         annotations = filter(None, map(lambda x: annotation_map.get(x),
                                        annotations.split()))
 
-        vimtag = '*{}()*'.format(name)
+        if mode == 'lua':
+            fstem = compoundname.split('.')[0]
+            vimtag = '*%s.%s()*' % (fstem, name)
+        else:
+            vimtag = '*%s()*' % name
+
         params = []
         type_length = 0
 
@@ -454,6 +479,10 @@ def parse_source_xml(filename):
             declname = get_child(param, 'declname')
             if declname:
                 param_name = get_text(declname).strip()
+            elif mode == 'lua':
+                # that's how it comes out of lua2dox
+                param_name = param_type
+                param_type = ''
 
             if param_name in param_exclude:
                 continue
@@ -521,7 +550,7 @@ def parse_source_xml(filename):
 
         if 'Deprecated' in xrefs:
             deprecated_functions.append(func_doc)
-        elif name.startswith(api_func_name_prefix):
+        elif name.startswith(CONFIG[mode]['func_name_prefix']):
             functions.append(func_doc)
 
         xrefs.clear()
@@ -547,115 +576,138 @@ def gen_docs(config):
 
     Doxygen is called and configured through stdin.
     """
-    p = subprocess.Popen(['doxygen', '-'], stdin=subprocess.PIPE)
+    for mode in CONFIG:
-    p.communicate(config.format(input=src_dir, output=out_dir,
+        output_dir = out_dir.format(mode=mode)
-                                filter=filter_cmd).encode('utf8'))
+        p = subprocess.Popen(['doxygen', '-'], stdin=subprocess.PIPE)
-    if p.returncode:
+        p.communicate(
-        sys.exit(p.returncode)
+            config.format(
+                input=CONFIG[mode]['files'],
+                output=output_dir,
+                filter=filter_cmd,
+                file_patterns=CONFIG[mode]['file_patterns'])
+            .encode('utf8')
+        )
+        if p.returncode:
+            sys.exit(p.returncode)
 
-    sections = {}
+        sections = {}
-    intros = {}
+        intros = {}
-    sep = '=' * text_width
+        sep = '=' * text_width
 
-    base = os.path.join(out_dir, 'xml')
+        base = os.path.join(output_dir, 'xml')
-    dom = minidom.parse(os.path.join(base, 'index.xml'))
+        dom = minidom.parse(os.path.join(base, 'index.xml'))
 
-    # generate docs for section intros
+        # generate docs for section intros
-    for compound in dom.getElementsByTagName('compound'):
+        for compound in dom.getElementsByTagName('compound'):
-        if compound.getAttribute('kind') != 'group':
+            if compound.getAttribute('kind') != 'group':
-            continue
-
-        groupname = get_text(find_first(compound, 'name'))
-        groupxml = os.path.join(base, '%s.xml' % compound.getAttribute('refid'))
-
-        desc = find_first(minidom.parse(groupxml), 'detaileddescription')
-        if desc:
-            doc = parse_parblock(desc)
-            if doc:
-                intros[groupname] = doc
-
-    for compound in dom.getElementsByTagName('compound'):
-        if compound.getAttribute('kind') != 'file':
-            continue
-
-        filename = get_text(find_first(compound, 'name'))
-        if filename.endswith('.c'):
-            functions, deprecated = parse_source_xml(
-                os.path.join(base, '%s.xml' % compound.getAttribute('refid')))
-
-            if not functions and not deprecated:
                 continue
 
-            if functions or deprecated:
+            groupname = get_text(find_first(compound, 'name'))
-                name = os.path.splitext(os.path.basename(filename))[0]
+            groupxml = os.path.join(base, '%s.xml' % compound.getAttribute('refid'))
-                if name == 'ui':
-                    name = name.upper()
-                else:
-                    name = name.title()
-
-                doc = ''
-
-                intro = intros.get('api-%s' % name.lower())
-                if intro:
-                    doc += '\n\n' + intro
-
-                if functions:
-                    doc += '\n\n' + functions
-
-                if INCLUDE_DEPRECATED and deprecated:
-                    doc += '\n\n\nDeprecated %s Functions: ~\n\n' % name
-                    doc += deprecated
 
+            desc = find_first(minidom.parse(groupxml), 'detaileddescription')
+            if desc:
+                doc = parse_parblock(desc)
                 if doc:
-                    filename = os.path.basename(filename)
+                    intros[groupname] = doc
-                    name = section_name.get(filename, name)
-                    title = '%s Functions' % name
-                    helptag = '*api-%s*' % name.lower()
-                    sections[filename] = (title, helptag, doc)
 
-    if not sections:
+        for compound in dom.getElementsByTagName('compound'):
-        return
+            if compound.getAttribute('kind') != 'file':
+                continue
 
-    docs = ''
+            filename = get_text(find_first(compound, 'name'))
+            if filename.endswith('.c') or filename.endswith('.lua'):
+                functions, deprecated = parse_source_xml(
+                    os.path.join(base, '%s.xml' %
+                        compound.getAttribute('refid')), mode)
 
-    i = 0
+                if not functions and not deprecated:
-    for filename in section_order:
+                    continue
-        if filename not in sections:
-            continue
-        title, helptag, section_doc = sections.pop(filename)
 
-        i += 1
+                if functions or deprecated:
-        docs += sep
+                    name = os.path.splitext(os.path.basename(filename))[0]
-        docs += '\n%s%s' % (title, helptag.rjust(text_width - len(title)))
+                    if name == 'ui':
-        docs += section_doc
+                        name = name.upper()
-        docs += '\n\n\n'
+                    else:
+                        name = name.title()
+
+                    doc = ''
+
+                    intro = intros.get('api-%s' % name.lower())
+                    if intro:
+                        doc += '\n\n' + intro
+
+                    if functions:
+                        doc += '\n\n' + functions
+
+                    if INCLUDE_DEPRECATED and deprecated:
+                        doc += '\n\n\nDeprecated %s Functions: ~\n\n' % name
+                        doc += deprecated
+
+                    if doc:
+                        filename = os.path.basename(filename)
+                        name = section_name.get(filename, name)
+
+                        if mode == 'lua':
+                            title = '%s Lua Functions' % name
+                            helptag = '*%s-lua*' % name.lower()
+                        else:
+                            title = '%s Functions' % name
+                            helptag = '*api-%s*' % name.lower()
+                        sections[filename] = (title, helptag, doc)
+
+        if not sections:
+            return
+
+        docs = ''
+
+        i = 0
+        for filename in CONFIG[mode]['section_order']:
+            if filename not in sections:
+                continue
+            title, helptag, section_doc = sections.pop(filename)
 
-    if sections:
-        # In case new API sources are added without updating the order dict.
-        for title, helptag, section_doc in sections.values():
             i += 1
             docs += sep
             docs += '\n%s%s' % (title, helptag.rjust(text_width - len(title)))
             docs += section_doc
             docs += '\n\n\n'
 
-    docs = docs.rstrip() + '\n\n'
+        if sections:
-    docs += ' vim:tw=78:ts=8:ft=help:norl:\n'
+            # In case new API sources are added without updating the order dict.
+            for title, helptag, section_doc in sections.values():
+                i += 1
+                docs += sep
+                docs += '\n%s%s' % (title, helptag.rjust(text_width - len(title)))
+                docs += section_doc
+                docs += '\n\n\n'
 
-    doc_file = os.path.join(base_dir, 'runtime/doc', doc_filename)
+        docs = docs.rstrip() + '\n\n'
-    delete_lines_below(doc_file, section_start_token)
+        docs += ' vim:tw=78:ts=8:ft=help:norl:\n'
-    with open(doc_file, 'ab') as fp:
+
-        fp.write(docs.encode('utf8'))
+        doc_file = os.path.join(base_dir, 'runtime', 'doc',
-    shutil.rmtree(out_dir)
+                CONFIG[mode]['filename'])
+
+        delete_lines_below(doc_file, CONFIG[mode]['section_start_token'])
+        with open(doc_file, 'ab') as fp:
+            fp.write(docs.encode('utf8'))
+
+        shutil.rmtree(output_dir)
 
 
 def filter_source(filename):
-    """Filters the source to fix macros that confuse Doxygen."""
+    name, extension = os.path.splitext(filename)
-    with open(filename, 'rt') as fp:
+    if extension == '.lua':
-        print(re.sub(r'^(ArrayOf|DictionaryOf)(\(.*?\))',
+        p = subprocess.run([lua2dox_filter, filename], stdout=subprocess.PIPE)
-                     lambda m: m.group(1)+'_'.join(
+        op = ('?' if 0 != p.returncode else p.stdout.decode('utf-8'))
-                         re.split(r'[^\w]+', m.group(2))),
+        print(op)
-                     fp.read(), flags=re.M))
+    else:
+        """Filters the source to fix macros that confuse Doxygen."""
+        with open(filename, 'rt') as fp:
+            print(re.sub(r'^(ArrayOf|DictionaryOf)(\(.*?\))',
+                         lambda m: m.group(1)+'_'.join(
+                             re.split(r'[^\w]+', m.group(2))),
+                         fp.read(), flags=re.M))
 
 
 # Doxygen Config {{{
@@ -663,13 +715,15 @@ Doxyfile = '''
 OUTPUT_DIRECTORY       = {output}
 INPUT                  = {input}
 INPUT_ENCODING         = UTF-8
-FILE_PATTERNS          = *.h *.c
+FILE_PATTERNS          = {file_patterns}
 RECURSIVE              = YES
 INPUT_FILTER           = "{filter}"
 EXCLUDE                =
 EXCLUDE_SYMLINKS       = NO
 EXCLUDE_PATTERNS       = */private/*
 EXCLUDE_SYMBOLS        =
+EXTENSION_MAPPING      = lua=C
+EXTRACT_PRIVATE        = NO
 
 GENERATE_HTML          = NO
 GENERATE_DOCSET        = NO

scripts/lua2dox.lua

@@ -0,0 +1,666 @@
+--[[--------------------------------------------------------------------------
+--   Copyright (C) 2012 by Simon Dales   --
+--   simon@purrsoft.co.uk   --
+--                                                                         --
+--   This program is free software; you can redistribute it and/or modify  --
+--   it under the terms of the GNU General Public License as published by  --
+--   the Free Software Foundation; either version 2 of the License, or     --
+--   (at your option) any later version.                                   --
+--                                                                         --
+--   This program is distributed in the hope that it will be useful,       --
+--   but WITHOUT ANY WARRANTY; without even the implied warranty of        --
+--   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         --
+--   GNU General Public License for more details.                          --
+--                                                                         --
+--   You should have received a copy of the GNU General Public License     --
+--   along with this program; if not, write to the                         --
+--   Free Software Foundation, Inc.,                                       --
+--   59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.             --
+----------------------------------------------------------------------------]]
+--[[!
+\file
+\brief a hack lua2dox converter
+]]
+
+--[[!
+\mainpage
+
+Introduction
+------------
+
+A hack lua2dox converter
+Version 0.2
+
+This lets us make Doxygen output some documentation to let
+us develop this code.
+
+It is partially cribbed from the functionality of lua2dox
+(http://search.cpan.org/~alec/Doxygen-Lua-0.02/lib/Doxygen/Lua.pm).
+Found on CPAN when looking for something else; kinda handy.
+
+Improved from lua2dox to make the doxygen output more friendly.
+Also it runs faster in lua rather than Perl.
+
+Because this Perl based system is called "lua2dox"., I have decided to add ".lua" to the name
+to keep the two separate.
+
+Running
+-------
+
+<ol>
+<li>  Ensure doxygen is installed on your system and that you are familiar with its use.
+Best is to try to make and document some simple C/C++/PHP to see what it produces.
+You can experiment with the enclosed example code.
+
+<li> Run "doxygen -g" to create a default Doxyfile.
+
+Then alter it to let it recognise lua. Add the two following lines:
+
+\code{.bash}
+FILE_PATTERNS   = *.lua
+
+FILTER_PATTERNS = *.lua=lua2dox_filter
+\endcode
+
+
+Either add them to the end or find the appropriate entry in Doxyfile.
+
+There are other lines that you might like to alter, but see futher documentation for details.
+
+<li> When Doxyfile is edited run "doxygen"
+
+The core function reads the input file (filename or stdin) and outputs some pseudo C-ish language.
+It only has to be good enough for doxygen to see it as legal.
+Therefore our lua interpreter is fairly limited, but "good enough".
+
+One limitation is that each line is treated separately (except for long comments).
+The implication is that class and function declarations must be on the same line.
+Some functions can have their parameter lists extended over multiple lines to make it look neat.
+Managing this where there are also some comments is a bit more coding than I want to do at this stage,
+so it will probably not document accurately if we do do this.
+
+However I have put in a hack that will insert the "missing" close paren.
+The effect is that you will get the function documented, but not with the parameter list you might expect.
+</ol>
+
+Installation
+------------
+
+Here for linux or unix-like, for any other OS you need to refer to other documentation.
+
+This file is "lua2dox.lua". It gets called by "lua2dox_filter"(bash).
+Somewhere in your path (e.g. "~/bin" or "/usr/local/bin") put a link to "lua2dox_filter".
+
+Documentation
+-------------
+
+Read the external documentation that should be part of this package.
+For example look for the "README" and some .PDFs.
+
+]]
+
+-- we won't use our library code, so this becomes more portable
+
+-- require 'elijah_fix_require'
+-- require 'elijah_class'
+-- 
+--! \brief ``declare'' as class
+--! 
+--! use as:
+--! \code{.lua}
+--! TWibble = class()
+--! function TWibble.init(this,Str)
+--! 	this.str = Str
+--! 	-- more stuff here
+--! end
+--! \endcode
+--! 
+function class(BaseClass, ClassInitialiser)
+  local newClass = {}    -- a new class newClass
+  if not ClassInitialiser and type(BaseClass) == 'function' then
+    ClassInitialiser = BaseClass
+    BaseClass = nil
+  elseif type(BaseClass) == 'table' then
+    -- our new class is a shallow copy of the base class!
+    for i,v in pairs(BaseClass) do
+      newClass[i] = v
+    end
+    newClass._base = BaseClass
+  end
+  -- the class will be the metatable for all its newInstanceects,
+  -- and they will look up their methods in it.
+  newClass.__index = newClass
+
+  -- expose a constructor which can be called by <classname>(<args>)
+  local classMetatable = {}
+  classMetatable.__call = 
+  function(class_tbl, ...)
+    local newInstance = {}
+    setmetatable(newInstance,newClass)
+    --if init then
+    --	init(newInstance,...)
+    if class_tbl.init then
+      class_tbl.init(newInstance,...)
+    else 
+      -- make sure that any stuff from the base class is initialized!
+      if BaseClass and BaseClass.init then
+        BaseClass.init(newInstance, ...)
+      end
+    end
+    return newInstance
+  end
+  newClass.init = ClassInitialiser
+  newClass.is_a = 
+  function(this, klass)
+    local thisMetatable = getmetatable(this)
+    while thisMetatable do 
+      if thisMetatable == klass then
+        return true
+      end
+      thisMetatable = thisMetatable._base
+    end
+    return false
+  end
+  setmetatable(newClass, classMetatable)
+  return newClass
+end
+
+-- require 'elijah_clock'
+
+--! \class TCore_Clock
+--! \brief a clock
+TCore_Clock = class()
+
+--! \brief get the current time
+function TCore_Clock.GetTimeNow()
+  if os.gettimeofday then
+    return os.gettimeofday()
+  else
+    return os.time()
+  end
+end
+
+--! \brief constructor
+function TCore_Clock.init(this,T0)
+  if T0 then
+    this.t0 = T0
+  else
+    this.t0 = TCore_Clock.GetTimeNow()
+  end
+end
+
+--! \brief get time string
+function TCore_Clock.getTimeStamp(this,T0)
+  local t0
+  if T0 then
+    t0 = T0
+  else
+    t0 = this.t0
+  end
+  return os.date('%c %Z',t0)
+end
+
+
+--require 'elijah_io'
+
+--! \class TCore_IO
+--! \brief io to console
+--! 
+--! pseudo class (no methods, just to keep documentation tidy)
+TCore_IO = class()
+-- 
+--! \brief write to stdout
+function TCore_IO_write(Str)
+  if (Str) then
+    io.write(Str)
+  end
+end
+
+--! \brief write to stdout
+function TCore_IO_writeln(Str)
+  if (Str) then
+    io.write(Str)
+  end
+  io.write("\n")
+end
+
+
+--require 'elijah_string'
+
+--! \brief trims a string
+function string_trim(Str)
+  return Str:match("^%s*(.-)%s*$")
+end
+
+--! \brief split a string
+--! 
+--! \param Str
+--! \param Pattern
+--! \returns table of string fragments
+function string_split(Str, Pattern)
+  local splitStr = {}
+  local fpat = "(.-)" .. Pattern
+  local last_end = 1
+  local str, e, cap = string.find(Str,fpat, 1)
+  while str do
+    if str ~= 1 or cap ~= "" then
+      table.insert(splitStr,cap)
+    end
+    last_end = e+1
+    str, e, cap = string.find(Str,fpat, last_end)
+  end
+  if last_end <= #Str then
+    cap = string.sub(Str,last_end)
+    table.insert(splitStr, cap)
+  end
+  return splitStr
+end
+
+
+--require 'elijah_commandline'
+
+--! \class TCore_Commandline
+--! \brief reads/parses commandline
+TCore_Commandline = class()
+
+--! \brief constructor
+function TCore_Commandline.init(this)
+  this.argv = arg
+  this.parsed = {}
+  this.params = {}
+end
+
+--! \brief get value
+function TCore_Commandline.getRaw(this,Key,Default)
+  local val = this.argv[Key]
+  if not val then
+    val = Default
+  end
+  return val
+end
+
+
+--require 'elijah_debug'
+
+-------------------------------
+--! \brief file buffer
+--! 
+--! an input file buffer
+TStream_Read = class()
+
+--! \brief get contents of file
+--! 
+--! \param Filename name of file to read (or nil == stdin)
+function 	TStream_Read.getContents(this,Filename)
+  -- get lines from file
+  local filecontents
+  if Filename then
+    -- syphon lines to our table
+    --TCore_Debug_show_var('Filename',Filename)
+    filecontents={}
+    for line in io.lines(Filename) do
+      table.insert(filecontents,line)
+    end
+  else
+    -- get stuff from stdin as a long string (with crlfs etc)
+    filecontents=io.read('*a')
+    --  make it a table of lines
+    filecontents = TString_split(filecontents,'[\n]') -- note this only works for unix files.
+    Filename = 'stdin'
+  end
+
+  if filecontents then
+    this.filecontents = filecontents
+    this.contentsLen = #filecontents
+    this.currentLineNo = 1
+  end
+
+  return filecontents
+end
+
+--! \brief get lineno
+function TStream_Read.getLineNo(this)
+  return this.currentLineNo
+end
+
+--! \brief get a line
+function TStream_Read.getLine(this)
+  local line
+  if this.currentLine then
+    line = this.currentLine
+    this.currentLine = nil
+  else
+    -- get line
+    if this.currentLineNo<=this.contentsLen then
+      line = this.filecontents[this.currentLineNo]
+      this.currentLineNo = this.currentLineNo + 1
+    else
+      line = ''
+    end
+  end
+  return line
+end
+
+--! \brief save line fragment
+function TStream_Read.ungetLine(this,LineFrag)
+  this.currentLine = LineFrag
+end
+
+--! \brief is it eof?
+function TStream_Read.eof(this)
+  if this.currentLine or this.currentLineNo<=this.contentsLen then
+    return false
+  end
+  return true
+end
+
+--! \brief output stream
+TStream_Write = class()
+
+--! \brief constructor
+function TStream_Write.init(this)
+  this.tailLine = {}
+end
+
+--! \brief write immediately
+function TStream_Write.write(this,Str)
+  TCore_IO_write(Str)
+end
+
+--! \brief write immediately
+function TStream_Write.writeln(this,Str)
+  TCore_IO_writeln(Str)
+end
+
+--! \brief write immediately
+function TStream_Write.writelnComment(this,Str)
+  TCore_IO_write('// ZZ: ')
+  TCore_IO_writeln(Str)
+end
+
+--! \brief write to tail
+function TStream_Write.writelnTail(this,Line)
+  if not Line then
+    Line = ''
+  end
+  table.insert(this.tailLine,Line)
+end
+
+--! \brief outout tail lines
+function TStream_Write.write_tailLines(this)
+  for k,line in ipairs(this.tailLine) do
+    TCore_IO_writeln(line)
+  end
+  TCore_IO_write('// Lua2DoX new eof')
+end
+
+--! \brief input filter
+TLua2DoX_filter = class()
+
+--! \brief allow us to do errormessages
+function TLua2DoX_filter.warning(this,Line,LineNo,Legend)
+  this.outStream:writelnTail(
+  '//! \todo warning! ' .. Legend .. ' (@' .. LineNo .. ')"' .. Line .. '"'
+  )
+end
+
+--! \brief trim comment off end of string
+--!
+--! If the string has a comment on the end, this trims it off.
+--!
+local function TString_removeCommentFromLine(Line)
+  local pos_comment = string.find(Line,'%-%-')
+  local tailComment
+  if pos_comment then
+    Line = string.sub(Line,1,pos_comment-1)
+    tailComment = string.sub(Line,pos_comment)
+  end
+  return Line,tailComment
+end
+
+--! \brief get directive from magic
+local function getMagicDirective(Line)
+  local macro,tail
+  local macroStr = '[\\@]'
+  local pos_macro = string.find(Line,macroStr)
+  if pos_macro then
+    --! ....\\ macro...stuff
+    --! ....\@ macro...stuff
+    local line = string.sub(Line,pos_macro+1)
+    local space = string.find(line,'%s+')
+    if space then
+      macro = string.sub(line,1,space-1)
+      tail  = string_trim(string.sub(line,space+1))
+    else
+      macro = line
+      tail  = ''
+    end
+  end
+  return macro,tail
+end
+
+--! \brief check comment for fn
+local function checkComment4fn(Fn_magic,MagicLines)
+  local fn_magic = Fn_magic
+  --	TCore_IO_writeln('// checkComment4fn "' .. MagicLines .. '"')
+
+  local magicLines = string_split(MagicLines,'\n')
+
+  local macro,tail
+
+  for k,line in ipairs(magicLines) do
+    macro,tail = getMagicDirective(line)
+    if macro == 'fn' then
+      fn_magic = tail
+      --	TCore_IO_writeln('// found fn "' .. fn_magic .. '"')
+    else
+      --TCore_IO_writeln('// not found fn "' .. line .. '"')
+    end
+  end
+
+  return fn_magic
+end
+--! \brief run the filter
+function TLua2DoX_filter.readfile(this,AppStamp,Filename)
+  local err
+
+  local inStream = TStream_Read()
+  local outStream = TStream_Write()
+  this.outStream = outStream -- save to this obj
+
+  if (inStream:getContents(Filename)) then
+    -- output the file
+    local line
+    local fn_magic -- function name/def from  magic comment
+
+    outStream:writelnTail('// #######################')
+    outStream:writelnTail('// app run:' .. AppStamp)
+    outStream:writelnTail('// #######################')
+    outStream:writelnTail()
+
+    while not (err or inStream:eof()) do
+      line = string_trim(inStream:getLine())
+      -- 			TCore_Debug_show_var('inStream',inStream)
+      -- 			TCore_Debug_show_var('line',line )
+      if string.sub(line,1,2)=='--' then -- its a comment
+        if string.sub(line,3,3)=='@' then -- it's a magic comment
+          local magic = string.sub(line,4)
+          outStream:writeln('/// @' .. magic)
+          fn_magic = checkComment4fn(fn_magic,magic)
+        elseif string.sub(line,3,3)=='-' then -- it's a nonmagic doc comment
+          local comment = string.sub(line,4)
+          outStream:writeln('/// '.. comment)
+        elseif string.sub(line,3,4)=='[[' then -- it's a long comment
+          line = string.sub(line,5) -- nibble head
+          local comment = ''
+          local closeSquare,hitend,thisComment
+          while (not err) and (not hitend) and (not inStream:eof()) do
+            closeSquare = string.find(line,']]')
+            if not closeSquare then -- need to look on another line
+              thisComment = line .. '\n'
+              line = inStream:getLine()
+            else
+              thisComment = string.sub(line,1,closeSquare-1)
+              hitend = true
+
+              -- unget the tail of the line
+              -- in most cases it's empty. This may make us less efficient but
+              -- easier to program
+              inStream:ungetLine(string_trim(string.sub(line,closeSquare+2)))
+            end
+            comment = comment .. thisComment
+          end
+          if string.sub(comment,1,1)=='@' then -- it's a long magic comment
+            outStream:write('/*' .. comment .. '*/  ')
+            fn_magic = checkComment4fn(fn_magic,comment)
+          else -- discard
+            outStream:write('/* zz:' .. comment .. '*/  ')
+            fn_magic = nil
+          end
+        else
+          outStream:writeln('// zz:"' .. line .. '"')
+          fn_magic = nil
+        end
+      elseif string.find(line,'^function') or string.find(line,'^local%s+function') then
+        -- it's a function
+        local pos_fn = string.find(line,'function')
+        -- function
+        -- ....v...
+        if pos_fn then
+          -- we've got a function
+          local fn_type
+          if string.find(line,'^local%s+') then
+            fn_type = ''--'static ' -- static functions seem to be excluded
+          else
+            fn_type = ''
+          end
+          local fn = TString_removeCommentFromLine(string_trim(string.sub(line,pos_fn+8)))
+          if fn_magic then
+            fn = fn_magic
+            fn_magic = nil
+          end
+
+          if string.sub(fn,1,1)=='(' then
+            -- it's an anonymous function
+            outStream:writelnComment(line)
+          else
+            -- fn has a name, so is interesting
+
+            -- want to fix for iffy declarations
+            local open_paren = string.find(fn,'[%({]')
+            local fn0 = fn
+            if open_paren then
+              fn0 = string.sub(fn,1,open_paren-1)
+              -- we might have a missing close paren
+              if not string.find(fn,'%)') then
+                fn = fn .. ' ___MissingCloseParenHere___)'
+              end
+            end
+
+            local dot = string.find(fn0,'[%.:]')
+            if dot then -- it's a method
+              local klass = string.sub(fn,1,dot-1)
+              local method = string.sub(fn,dot+1)
+              --TCore_IO_writeln('function ' .. klass .. '::' .. method .. ftail .. '{}')
+              --TCore_IO_writeln(klass .. '::' .. method .. ftail .. '{}')
+              outStream:writeln(
+              '/*! \\memberof ' .. klass .. ' */ '
+              .. method .. '{}'
+              )
+            else
+              -- add vanilla function
+
+              outStream:writeln(fn_type .. 'function ' .. fn .. '{}')
+            end
+          end
+        else
+          this:warning(inStream:getLineNo(),'something weird here')
+        end
+        fn_magic = nil -- mustn't indavertently use it again
+      elseif string.find(line,'=%s*class%(') then
+        -- it's a class declaration
+        local tailComment
+        line,tailComment = TString_removeCommentFromLine(line)
+        local equals = string.find(line,'=')
+        local klass = string_trim(string.sub(line,1,equals-1))
+        local tail =  string_trim(string.sub(line,equals+1))
+        -- class(wibble wibble)
+        -- ....v.
+        local parent = string.sub(tail,7,-2)
+        if #parent>0 then
+          parent = ' :public ' .. parent
+        end
+        outStream:writeln('class ' .. klass .. parent .. '{};')
+      else
+        -- we don't know what this line means, so we can probably just comment it out
+        if #line>0 then
+          outStream:writeln('// zz: ' .. line)
+        else
+          outStream:writeln() -- keep this line blank
+        end
+      end
+    end
+
+    -- output the tail
+    outStream:write_tailLines()
+  else
+    outStream:writeln('!empty file')
+  end
+end
+
+--! \brief this application
+TApp = class()
+
+--! \brief constructor
+function TApp.init(this)
+  local t0 = TCore_Clock()
+  this.timestamp = t0:getTimeStamp()
+  this.name = 'Lua2DoX'
+  this.version = '0.2 20130128'
+  this.copyright = 'Copyright (c) Simon Dales 2012-13'
+end
+
+function TApp.getRunStamp(this)
+  return this.name .. ' (' .. this.version .. ') ' 
+  .. this.timestamp
+end
+
+function TApp.getVersion(this)
+  return this.name .. ' (' .. this.version .. ') ' 
+end
+
+function TApp.getCopyright(this)
+  return this.copyright 
+end
+
+local This_app = TApp()
+
+--main
+local cl = TCore_Commandline()
+
+local argv1 = cl:getRaw(2)
+if argv1 == '--help' then
+  TCore_IO_writeln(This_app:getVersion())
+  TCore_IO_writeln(This_app:getCopyright())
+  TCore_IO_writeln([[
+  run as:
+  lua2dox_filter <param>
+  --------------
+  Param:
+  <filename> : interprets filename
+  --version  : show version/copyright info
+  --help     : this help text]])
+elseif argv1 == '--version' then
+  TCore_IO_writeln(This_app:getVersion())
+  TCore_IO_writeln(This_app:getCopyright())
+else
+  -- it's a filter
+  local appStamp = This_app:getRunStamp()
+  local filename = argv1
+
+  local filter = TLua2DoX_filter()
+  filter:readfile(appStamp,filename)
+end
+
+
+--eof

scripts/lua2dox_filter

@@ -0,0 +1,87 @@
+#!/bin/bash
+
+###########################################################################
+#   Copyright (C) 2012 by Simon Dales   #
+#   simon@purrsoft.co.uk   #
+#                                                                         #
+#   This program is free software; you can redistribute it and/or modify  #
+#   it under the terms of the GNU General Public License as published by  #
+#   the Free Software Foundation; either version 2 of the License, or     #
+#   (at your option) any later version.                                   #
+#                                                                         #
+#   This program is distributed in the hope that it will be useful,       #
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of        #
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         #
+#   GNU General Public License for more details.                          #
+#                                                                         #
+#   You should have received a copy of the GNU General Public License     #
+#   along with this program; if not, write to the                         #
+#   Free Software Foundation, Inc.,                                       #
+#   59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.             #
+###########################################################################
+LANG=""
+
+##! \brief test executable to see if it exists
+test_executable(){
+	P_EXE="$1"
+	#########
+	WHICH=`which ${P_EXE}`
+	if test -z "${WHICH}"
+	then
+		echo "not found \"${P_EXE}\""
+	else
+		EXE="${P_EXE}"
+	fi
+	}
+
+##! \brief sets the lua interpreter
+set_lua(){
+	test_executable 'texlua'
+	if test -z "${EXE}"
+	then
+		test_executable 'lua'
+	fi
+	#echo "final EXE=\"${EXE}\""
+	}
+
+##! \brief makes canonical name of file
+##! 
+##! Note that "readlink -f" doesn't work in MacOSX
+##! 
+do_readlink(){
+	pushd . > /dev/null
+	TARGET_FILE=$1
+	
+	cd `dirname $TARGET_FILE`
+	TARGET_FILE=`basename $TARGET_FILE`
+	
+	# Iterate down a (possible) chain of symlinks
+	while [ -L "$TARGET_FILE" ]
+	do
+		TARGET_FILE=`readlink $TARGET_FILE`
+		cd `dirname $TARGET_FILE`
+		TARGET_FILE=`basename $TARGET_FILE`
+	done
+	
+	PHYS_DIR=`pwd -P`
+	RESULT=$PHYS_DIR
+	popd > /dev/null
+	}
+
+##main
+set_lua
+if test -z "${EXE}"
+then
+	echo "no lua interpreter available"
+else
+	BASENAME=`basename "$0"`
+	do_readlink "$0"
+	DIRNAME="${RESULT}"
+		
+	LUASCRIPT="${DIRNAME}/lua2dox.lua ${BASENAME}"
+	#echo "lua[${LUASCRIPT}]"
+
+	${EXE} ${LUASCRIPT} $@
+fi
+# 
+##eof

src/Doxyfile

@@ -243,7 +243,7 @@ OPTIMIZE_OUTPUT_VHDL   = NO
 # that for custom extensions you also need to set FILE_PATTERNS otherwise the
 # files are not read by doxygen.
 
-EXTENSION_MAPPING      =
+EXTENSION_MAPPING      = lua=C
 
 # If MARKDOWN_SUPPORT is enabled (the default) then doxygen pre-processes all
 # comments according to the Markdown format, which allows for more readable
@@ -672,7 +672,7 @@ INPUT_ENCODING         = UTF-8
 # *.hxx *.hpp *.h++ *.idl *.odl *.cs *.php *.php3 *.inc *.m *.mm *.dox *.py
 # *.f90 *.f *.for *.vhd *.vhdl
 
-FILE_PATTERNS          = *.h *.c
+FILE_PATTERNS          = *.h *.c *.lua
 
 # The RECURSIVE tag can be used to turn specify whether or not subdirectories
 # should be searched for input files as well. Possible values are YES and NO.
@@ -758,7 +758,7 @@ INPUT_FILTER           =
 # info on how filters are used. If FILTER_PATTERNS is empty or if
 # non of the patterns match the file name, INPUT_FILTER is applied.
 
-FILTER_PATTERNS        =
+FILTER_PATTERNS        = *.lua=scripts/lua2dox_filter
 
 # If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using
 # INPUT_FILTER) will be used to filter the input files when producing source