Add API version and have server reject incompatible clients.

This patch contains 2 parts.

The first part is a small utility to create and validate the current
API. To do this it needs to load ipalib which on a fresh system
introduces a few problems, namely that it relies on a python plugin
to set the default encoding to utf8. For our purposes we can skip that.
It is also important that any optional plugins be loadable so the
API can be examined.

The second part is a version exchange between the client and server.
The version has a major and a minor version. The major verion is
updated whenever existing API changes. The minor version is updated when
new API is added. A request will be rejected if either the major versions
don't match or if the client major version is higher than then server
major version (though by implication new API would return a command not
found if allowed to proceed).

To determine the API version of the server from a client use the ping
command.

ticket 584

BUILD.txt

@@ -67,6 +67,25 @@ Some tests may be skipped. For example, all the XML-RPC tests will be skipped
 if you haven't started the lite-server. The DNS tests will be skipped if
 the underlying IPA installation doesn't configure DNS, etc.
 
+API.txt
+-------
+The purpose of the file API.txt is to prevent accidental API changes. The
+program ./makeapi creates file and also validates it (with the --validate
+option). This validation is part of the build process.
+
+There are three solutions to changes to the API:
+
+1. Changes to existing API require a change to the MAJOR version.
+2. Addition of new API requires a change to the MINOR version.
+3. Or just back out your changes and don't make an API change.
+
+If the API changes you'll need to run ./makeapi to update API.txt and
+commit it along with VERSION with your API change.
+
+If a module is optionally loaded then you will need to be able to
+conditionally load it for API validation. The environment variable
+api.env.validate_api is True during validation.
+
 General Notes
 -------------
 IPA is not relocatable.

Makefile

@@ -49,7 +49,7 @@ client: client-autogen
 
 bootstrap-autogen: version-update client-autogen
 	@echo "Building IPA $(IPA_VERSION)"
-	cd daemons; if [ ! -e Makefile ]; then ../autogen.sh --prefix=/usr --sysconfdir=/etc --localstatedir=/var --libdir=$(LIBDIR); fi
+	cd daemons; if [ ! -e Makefile ]; then ../autogen.sh --prefix=/usr --sysconfdir=/etc --localstatedir=/var --libdir=$(LIBDIR) --with-openldap; fi
 	cd install; if [ ! -e Makefile ]; then ../autogen.sh --prefix=/usr --sysconfdir=/etc --localstatedir=/var --libdir=$(LIBDIR); fi
 
 client-autogen: version-update
@@ -90,6 +90,7 @@ version-update: release-update
 	sed -e s/__VERSION__/$(IPA_VERSION)/ ipapython/version.py.in \
 		> ipapython/version.py
 	perl -pi -e "s:__NUM_VERSION__:$(IPA_VERSION_MAJOR)$(IPA_VERSION_MINOR)$(IPA_VERSION_RELEASE):" ipapython/version.py
+	perl -pi -e "s:__API_VERSION__:$(IPA_API_VERSION_MAJOR).$(IPA_API_VERSION_MINOR):" ipapython/version.py
 	sed -e s/__VERSION__/$(IPA_VERSION)/ daemons/ipa-version.h.in \
 		> daemons/ipa-version.h
 	perl -pi -e "s:__NUM_VERSION__:$(IPA_VERSION_MAJOR)$(IPA_VERSION_MINOR)$(IPA_VERSION_RELEASE):" daemons/ipa-version.h
@@ -99,6 +100,9 @@ version-update: release-update
 		ipa-client/ipa-client.spec.in > ipa-client/ipa-client.spec
 	sed -e s/__VERSION__/$(IPA_VERSION)/ ipa-client/version.m4.in \
 		> ipa-client/version.m4
+	if [ "$(SKIP_API_VERSION_CHECK)" != "yes" ]; then \
+		./makeapi --validate; \
+	fi
 
 server:
 	python setup.py build

VERSION

@@ -66,3 +66,17 @@ IPA_VERSION_IS_GIT_SNAPSHOT="yes"
 #  ->  "20100614120000"                                #
 ########################################################
 IPA_DATA_VERSION=20100614120000
+
+########################################################
+# The version of the IPA API. This controls which      #
+# client versions can use the XML-RPC and json APIs    #
+#                                                      #
+# A change to existing API requires a MAJOR version    #
+# update.  The addition of new API bumps the MINOR     #
+# version.                                             #
+#                                                      #
+# The format is a whole number                         #
+#                                                      #
+########################################################
+IPA_API_VERSION_MAJOR=2
+IPA_API_VERSION_MINOR=0

ipa.spec.in

@@ -39,6 +39,11 @@ BuildRequires:  m4
 BuildRequires:  policycoreutils >= %{POLICYCOREUTILSVER}
 BuildRequires:  python-setuptools
 BuildRequires:  python-krbV
+BuildRequires:  python-nss
+BuildRequires:  python-netaddr
+BuildRequires:  python-kerberos
+BuildRequires:  python-ldap
+BuildRequires:  krb5-workstation
 BuildRequires:  xmlrpc-c-devel
 BuildRequires:  libcurl-devel
 BuildRequires:  gettext

ipalib/cli.py

@@ -32,7 +32,14 @@ import fcntl
 import termios
 import struct
 import base64
-import default_encoding_utf8
+try:
+    import default_encoding_utf8
+except ImportError:
+    # This is a chicken-and-egg problem. The api can't be imported unless
+    # this is already installed and since it is installed with IPA therein
+    # lies the problem. Skip it for now so ipalib can be imported in-tree
+    # even in cases that IPA isn't installed on the dev machine.
+    pass
 
 import frontend
 import backend
@@ -42,6 +49,7 @@ from errors import PublicError, CommandError, HelpError, InternalError, NoSuchNa
 from constants import CLI_TAB
 from parameters import Password, Bytes, File
 from text import _
+from ipapython.version import API_VERSION
 
 
 def to_cli(name):
@@ -884,6 +892,7 @@ class cli(backend.Executioner):
         if not isinstance(cmd, frontend.Local):
             self.create_context()
         kw = self.parse(cmd, argv)
+        kw['version'] = API_VERSION
         if self.env.interactive:
             self.prompt_interactively(cmd, kw)
         self.load_files(cmd, kw)
@@ -931,6 +940,8 @@ class cli(backend.Executioner):
                 dest=option.name,
                 help=unicode(option.doc),
             )
+            if 'no_option' in option.flags:
+                continue
             if option.password and self.env.interactive:
                 kw['action'] = 'store_true'
             elif option.type is bool and option.autofill:

ipalib/constants.py

@@ -140,6 +140,9 @@ DEFAULT_CONFIG = (
     ('enable_ra', False),
     ('ra_plugin', 'selfsign'),
 
+    # Used when verifying that the API hasn't changed. Not for production.
+    ('validate_api', False),
+
     # ********************************************************
     #  The remaining keys are never set from the values here!
     # ********************************************************

ipalib/crud.py

@@ -76,7 +76,7 @@ us:
 >>> list(api.Command.user_add.args)
 ['login']
 >>> list(api.Command.user_add.options)
-['first', 'last', 'all', 'raw']
+['first', 'last', 'all', 'raw', 'version']
 
 Notice that ``'ipauniqueid'`` isn't included in the options for our ``user_add``
 plugin.  This is because of the ``'no_create'`` flag we used when defining the
@@ -94,7 +94,7 @@ class created them for us:
 >>> list(api.Command.user_show.args)
 ['login']
 >>> list(api.Command.user_show.options)
-['all', 'raw']
+['all', 'raw', 'version']
 
 As you can see, `Retrieve` plugins take a single argument (the primary key) and
 no options.  If needed, you can still specify options for your `Retrieve` plugin

ipalib/frontend.py

@@ -30,9 +30,11 @@ from util import make_repr
 from output import Output, Entry, ListOfEntries
 from text import _, ngettext
 
-from errors import ZeroArgumentError, MaxArgumentError, OverlapError, RequiresRoot
+from errors import ZeroArgumentError, MaxArgumentError, OverlapError, RequiresRoot, VersionError, RequirementError
 from errors import InvocationError
 from constants import TYPE_ERROR
+from ipapython.version import API_VERSION
+from distutils import version
 
 
 RULE_FLAG = 'validation_rule'
@@ -412,6 +414,8 @@ class Command(HasParam):
         self.info(
             '%s(%s)', self.name, ', '.join(self._repr_iter(**params))
         )
+        if not self.api.env.in_server and 'version' not in params:
+            params['version'] = API_VERSION
         self.validate(**params)
         (args, options) = self.params_2_args_options(**params)
         ret = self.run(*args, **options)
@@ -680,6 +684,30 @@ class Command(HasParam):
             value = kw.get(param.name, None)
             param.validate(value, self.env.context)
 
+    def verify_client_version(self, client_version):
+        """
+        Compare the version the client provided to the version of the
+        server.
+
+        If the client major version does not match then return an error.
+        If the client minor version is less than or equal to the server
+        then let the request proceed.
+        """
+        ver = version.LooseVersion(client_version)
+        if len(ver.version) < 2:
+            raise VersionError(cver=ver.version, sver=server_ver.version, server= self.env.xmlrpc_uri)
+        client_major = ver.version[0]
+        client_minor = ver.version[1]
+
+        server_ver = version.LooseVersion(API_VERSION)
+        server_major = server_ver.version[0]
+        server_minor = server_ver.version[1]
+
+        if server_major != client_major:
+            raise VersionError(cver=client_version, sver=API_VERSION, server=self.env.xmlrpc_uri)
+        if client_minor > server_minor:
+            raise VersionError(cver=client_version, sver=API_VERSION, server=self.env.xmlrpc_uri)
+
     def run(self, *args, **options):
         """
         Dispatch to `Command.execute` or `Command.forward`.
@@ -693,6 +721,9 @@ class Command(HasParam):
         performs is executed remotely.
         """
         if self.api.env.in_server:
+            if 'version' in options:
+                self.verify_client_version(options['version'])
+                del options['version']
             return self.execute(*args, **options)
         return self.forward(*args, **options)
 
@@ -826,6 +857,11 @@ class Command(HasParam):
                     exclude='webui',
                     flags=['no_output'],
                 )
+                yield Str('version?',
+                    doc=_('Client version. Used to determine if server will accept request.'),
+                    exclude='webui',
+                    flags=['no_option', 'no_output'],
+                )
                 return
 
     def validate_output(self, output):

ipalib/plugins/batch.py

@@ -33,7 +33,7 @@ where the contenst of the file batch_request.json follow the below example
         {"method":"user_show","params":[["admin"],{"all":true}]}
         ],{}],"id":1}
 
-THe format of the response is nested the same way.  At the top you will see
+The format of the response is nested the same way.  At the top you will see
   "error": null,
     "id": 1,
     "result": {
@@ -51,6 +51,7 @@ from ipalib import Str, List
 from ipalib.output import Output
 from ipalib import output
 from ipalib.text import _
+from ipapython.version import API_VERSION
 
 class batch(Command):
     INTERNAL = True
@@ -61,6 +62,17 @@ class batch(Command):
              ),
         )
 
+    take_options = (
+        Str('version',
+            cli_name='version',
+            doc=_('Client version. Used to determine if server will accept request.'),
+            exclude='webui',
+            flags=['no_option', 'no_output'],
+            default=API_VERSION,
+            autofill=True,
+            )
+    )
+
     has_output = (
         Output('count', int, doc=_('')),
         Output('results', list, doc=_(''))

ipalib/plugins/ping.py

@@ -23,6 +23,7 @@ Ping the remote IPA server
 from ipalib import api
 from ipalib import Command
 from ipalib import output
+from ipapython.version import VERSION, API_VERSION
 
 class ping(Command):
     """
@@ -37,6 +38,6 @@ class ping(Command):
         A possible enhancement would be to take an argument and echo it
         back but a fixed value works for now.
         """
-        return dict(summary=u'pong')
+        return dict(summary=u'IPA server version %s. API version %s' % (VERSION, API_VERSION))
 
 api.register(ping)

ipapython/version.py.in

@@ -23,3 +23,6 @@ VERSION="__VERSION__"
 # Just the numeric portion of the version so one can do direct numeric
 # comparisons to see if the API is compatible.
 NUM_VERSION=__NUM_VERSION__
+
+# The version of the API.
+API_VERSION=u'__API_VERSION__'

makeapi

@@ -0,0 +1,238 @@
+#!/usr/bin/env python
+# Authors:
+#   Rob Crittenden <rcritten@redhat.com>
+#
+# Copyright (C) 2011  Red Hat
+# see file 'COPYING' for use and warranty information
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Test the API against a known-good API to ensure that changes aren't made
+# lightly.
+
+import sys
+import os
+import re
+from ipalib import *
+from ipalib.text import Gettext
+
+API_FILE='API.txt'
+
+API_FILE_DIFFERENCE = 1
+API_NEW_COMMAND = 2
+API_NO_FILE = 4
+
+def parse_options():
+    from optparse import OptionParser
+
+    parser = OptionParser()
+    parser.add_option("--validate", dest="validate", action="store_true",
+        default=False, help="Validate the API vs the stored API")
+
+    options, args = parser.parse_args()
+    return options, args
+
+def strip_doc(line):
+    """
+    Remove the doc= line from the repr() of a Paramter.
+    """
+    s = line.find(' doc=')
+    if s >= 0:
+        e = line.find('), ', s)
+        line = '%s%s' % (line[0:s], line[e+2:])
+
+    return line
+
+def make_api():
+    """
+    Write a new API file from the current tree.
+    """
+    fd = open(API_FILE, 'w')
+    for cmd in api.Command():
+        fd.write('command: %s\n' % cmd.name)
+        fd.write('args: %d,%d,%d\n' % (len(cmd.args), len(cmd.options), len(cmd.output)))
+        for a in cmd.args():
+            fd.write('arg: %s\n' % strip_doc(repr(a)))
+        for o in cmd.options():
+            fd.write('option: %s\n' % strip_doc(repr(o)))
+        for o in cmd.output():
+            fd.write('output: %s\n' % strip_doc(repr(o)))
+    fd.close()
+
+    return 0
+
+def find_name(line):
+    """
+    Break apart a Param line and pull out the name. It would be nice if we
+    could just eval() the line but we wouldn't have defined any validators
+    or normalizers it may be using.
+    """
+    m = re.match('^[a-zA-Z0-9]+\(\'([a-z][_a-z0-9?\*\+]*)\'.*', line)
+    if m:
+        name = m.group(1)
+    else:
+        print "Couldn't find name in: %s" % line
+        name = ''
+    return name
+
+def validate_api():
+    """
+    Compare the API in the file to the one in ipalib.
+
+    Return a bitwise return code to identify the types of errors found, if
+    any.
+    """
+    fd = open(API_FILE, 'r')
+    lines = fd.readlines()
+    fd.close()
+
+    rval = 0
+
+    # First run through the file and compare it to the API
+    existing_cmds = []
+    cmd = None
+    for line in lines:
+        line = line.strip()
+        if line.startswith('command:'):
+            if cmd:
+                # Check the args of the previous command.
+                if found_args != expected_args:
+                    print 'Argument count in %s of %d doesn\'t match expected: %d' % (
+                        name, found_args, expected_args)
+                    rval |= API_FILE_DIFFERENCE
+                if found_options != expected_options:
+                    print 'Options count in %s of %d doesn\'t match expected: %d' % (
+                        name, found_options, expected_options)
+                    rval |= API_FILE_DIFFERENCE
+                if found_output != expected_output:
+                    print 'Output count in %s of %d doesn\'t match expected: %d' % (
+                        name, found_output, expected_output)
+                    rval |= API_FILE_DIFFERENCE
+            (arg, name) = line.split(': ', 1)
+            if name not in api.Command:
+                print "Command %s in API file, not in ipalib" % name
+                rval |= API_FILE_DIFFERENCE
+                cmd = None
+            else:
+                existing_cmds.append(name)
+                cmd = api.Command[name]
+            found_args = 0
+            found_options = 0
+            found_output = 0
+        if line.startswith('args:') and cmd:
+            line = line.replace('args: ', '')
+            (expected_args, expected_options, expected_output) = line.split(',')
+            expected_args = int(expected_args)
+            expected_options = int(expected_options)
+            expected_output = int(expected_output)
+        if line.startswith('arg:') and cmd:
+            line = line.replace('arg: ', '')
+            found = False
+            for a in cmd.args():
+                if strip_doc(repr(a)) == line:
+                    found = True
+                else:
+                    arg = find_name(line)
+                    if a.name == arg:
+                        found = True
+                        print 'Arg in %s doesn\'t match.\nGot      %s\nExpected %s' % (
+                            name, strip_doc(repr(a)), line)
+                        rval |= API_FILE_DIFFERENCE
+            if found:
+                found_args += 1
+            else:
+                arg = find_name(line)
+                print "Argument '%s' in command '%s' in API file not found" % (arg, name)
+                rval |= API_FILE_DIFFERENCE
+        if line.startswith('option:') and cmd:
+            line = line.replace('option: ', '')
+            found = False
+            for o in cmd.options():
+                if strip_doc(repr(o)) == line:
+                    found = True
+                else:
+                    option = find_name(line)
+                    if o.name == option:
+                        found = True
+                        print 'Option in %s doesn\'t match. Got %s Expected %s' % (name, o, line)
+                        rval |= API_FILE_DIFFERENCE
+            if found:
+                found_options += 1
+            else:
+                option = find_name(line)
+                print "Option '%s' in command '%s' in API file not found" % (option, name)
+                rval |= API_FILE_DIFFERENCE
+        if line.startswith('output:') and cmd:
+            line = line.replace('output: ', '')
+            found = False
+            for o in cmd.output():
+                if strip_doc(repr(o)) == line:
+                    found = True
+                else:
+                    output = find_name(line)
+                    if o.name == output:
+                        found = True
+                        print 'Output in %s doesn\'t match. Got %s Expected %s' % (name, o, line)
+                        rval |= API_FILE_DIFFERENCE
+            if found:
+                found_output += 1
+            else:
+                output = find_name(line)
+                print "Option '%s' in command '%s' in API file not found" % (output, name)
+                rval |= API_FILE_DIFFERENCE
+
+    # Now look for new commands not in the current API
+    for cmd in api.Command():
+        if cmd.name not in existing_cmds:
+            print "Command %s in ipalib, not in API" % cmd.name
+            rval |= API_NEW_COMMAND
+
+    return rval
+
+def main():
+    options, args = parse_options()
+
+    cfg = dict(
+        context='cli',
+        in_server=False,
+        debug=False,
+        verbose=0,
+        validate_api=True,
+        enable_ra=True,
+    )
+
+    api.bootstrap(**cfg)
+    api.finalize()
+
+    if options.validate:
+        if not os.path.exists(API_FILE):
+            print 'No %s to validate' % API_FILE
+            rval = API_NO_FILE
+        else:
+            rval = validate_api()
+    else:
+        print "Writing API to API.txt"
+        rval = make_api()
+
+    if rval & API_FILE_DIFFERENCE:
+        print ''
+        print 'There are one or more changes to the API.\nEither undo the API changes or update API.txt and increment the major version in VERSION.'
+
+    if rval & API_NEW_COMMAND:
+        print ''
+        print 'There are one or more new commands defined.\nUpdate API.txt and increment the minor version in VERSION.'
+
+    return rval
+
+sys.exit(main())

tests/test_ipalib/test_crud.py

@@ -74,12 +74,13 @@ class test_Create(CrudChecker):
         """
         api = self.get_api()
         assert list(api.Method.user_verb.options) == \
-            ['givenname', 'sn', 'initials', 'all', 'raw']
+            ['givenname', 'sn', 'initials', 'all', 'raw', 'version']
         for param in api.Method.user_verb.options():
-            assert param.required is True
+            if param.name != 'version':
+                assert param.required is True
         api = self.get_api(options=('extra?',))
         assert list(api.Method.user_verb.options) == \
-            ['givenname', 'sn', 'initials', 'extra', 'all', 'raw']
+            ['givenname', 'sn', 'initials', 'extra', 'all', 'raw', 'version']
         assert api.Method.user_verb.options.extra.required is False
 
 
@@ -104,7 +105,7 @@ class test_Update(CrudChecker):
         """
         api = self.get_api()
         assert list(api.Method.user_verb.options) == \
-            ['givenname', 'initials', 'uidnumber', 'all', 'raw']
+            ['givenname', 'initials', 'uidnumber', 'all', 'raw', 'version']
         for param in api.Method.user_verb.options():
             if param.name in ['all', 'raw']:
                 assert param.required is True
@@ -132,7 +133,7 @@ class test_Retrieve(CrudChecker):
         Test the `ipalib.crud.Retrieve.get_options` method.
         """
         api = self.get_api()
-        assert list(api.Method.user_verb.options) == ['all', 'raw']
+        assert list(api.Method.user_verb.options) == ['all', 'raw', 'version']
 
 
 class test_Delete(CrudChecker):
@@ -180,7 +181,7 @@ class test_Search(CrudChecker):
         """
         api = self.get_api()
         assert list(api.Method.user_verb.options) == \
-            ['givenname', 'sn', 'uid', 'initials', 'all', 'raw']
+            ['givenname', 'sn', 'uid', 'initials', 'all', 'raw', 'version']
         for param in api.Method.user_verb.options():
             if param.name in ['all', 'raw']:
                 assert param.required is True

tests/test_ipalib/test_frontend.py

@@ -29,6 +29,7 @@ from ipalib.base import NameSpace
 from ipalib import frontend, backend, plugable, errors, parameters, config
 from ipalib import output
 from ipalib.parameters import Str
+from ipapython.version import API_VERSION
 
 def test_RULE_FLAG():
     assert frontend.RULE_FLAG == 'validation_rule'
@@ -431,6 +432,7 @@ class test_Command(ClassChecker):
             option0=u'option0',
             option1=u'option1',
             another_option='some value',
+            version=API_VERSION,
         )
         sub.validate(**okay)
 
@@ -561,7 +563,7 @@ class test_Command(ClassChecker):
                 return ('forward', args, kw)
 
         args = ('Hello,', 'world,')
-        kw = dict(how_are='you', on_this='fine day?')
+        kw = dict(how_are='you', on_this='fine day?', version=API_VERSION)
 
         # Test in server context:
         (api, home) = create_test_api(in_server=True)
@@ -569,7 +571,9 @@ class test_Command(ClassChecker):
         o = my_cmd()
         o.set_api(api)
         assert o.run.im_func is self.cls.run.im_func
-        assert ('execute', args, kw) == o.run(*args, **kw)
+        out = o.run(*args, **kw)
+        del kw['version']
+        assert ('execute', args, kw) == out
 
         # Test in non-server context
         (api, home) = create_test_api(in_server=False)