Add Custodia 0.6.0 to ipaserver package

Incorporate Custodia into IPA.

See: https://pagure.io/freeipa/issue/8882
Signed-off-by: Christian Heimes <cheimes@redhat.com>
Reviewed-By: Alexander Bokovoy <abokovoy@redhat.com>
Reviewed-By: Rob Crittenden <rcritten@redhat.com>

ipaserver/custodia/__about__.py

@@ -0,0 +1,19 @@
+# Copyright (C) 2017  Custodia Project Contributors - see LICENSE file
+
+__all__ = [
+    '__title__', '__summary__', '__uri__', '__version__', '__version_info__',
+    '__author__', '__email__', '__license__', '__copyright__',
+]
+
+__title__ = 'custodia'
+__summary__ = 'A service to manage, retrieve and store secrets.'
+__uri__ = 'https://github.com/latchset/custodia'
+
+__version_info__ = (0, 6, 0)
+__version__ = '.'.join(str(v) for v in __version_info__)
+
+__author__ = 'Custodia project Contributors'
+__email__ = 'simo@redhat.com'
+
+__license__ = 'GPLv3+'
+__copyright__ = 'Copyright 2015-2017 {0}'.format(__author__)

ipaserver/custodia/__init__.py

@@ -0,0 +1,3 @@
+# custodia namespace
+# You must NOT include any other code and data in __init__.py
+__import__('pkg_resources').declare_namespace(__name__)

ipaserver/custodia/cli/__init__.py

@@ -0,0 +1,373 @@
+# Copyright (C) 2016  Custodia Project Contributors - see LICENSE file
+from __future__ import absolute_import, print_function
+
+import argparse
+import operator
+import os
+import traceback
+
+import pkg_resources
+
+import requests.exceptions
+
+import six
+
+from custodia import log
+from custodia.client import CustodiaSimpleClient, requests_gssapi
+from custodia.compat import unquote, url_escape, urlparse
+
+if six.PY2:
+    from StringIO import StringIO  # pylint: disable=import-error
+else:
+    from io import StringIO
+
+try:
+    from json import JSONDecodeError
+except ImportError:
+    # Python 2.7 has no JSONDecodeError
+    JSONDecodeError = ValueError
+
+
+log.warn_provisional(__name__)
+
+# exit codes
+E_HTTP_ERROR = 1
+E_CONNECTION_ERROR = 2
+E_JSON_ERROR = 3
+E_OTHER = 100
+
+
+main_parser = argparse.ArgumentParser(
+    prog='custodia-cli',
+    description='Custodia command line interface'
+)
+
+
+def server_check(arg):
+    """Check and format --server arg
+    """
+    if arg.startswith(('http://', 'https://', 'http+unix://')):
+        return arg
+    if arg.startswith('./'):
+        arg = os.path.abspath(arg)
+    elif not arg.startswith('/'):
+        raise argparse.ArgumentTypeError(
+            'Unix socket path must start with / or ./')
+    # assume it is a unix socket
+    return 'http+unix://{}'.format(url_escape(arg, ''))
+
+
+def instance_check(arg):
+    if set(arg).intersection(':/@'):
+        raise argparse.ArgumentTypeError(
+            'Instance name contains invalid characters')
+    return arg
+
+
+def split_header(arg):
+    name, value = arg.split('=')
+    return name, value
+
+
+def timeout(arg):
+    try:
+        arg = float(arg)
+    except (TypeError, ValueError):
+        raise argparse.ArgumentTypeError('Argument is not a float')
+    if arg < 0.0:
+        raise argparse.ArgumentTypeError('Argument is negative')
+    if arg == 0.0:
+        # no timeout
+        return None
+    return arg
+
+
+group = main_parser.add_mutually_exclusive_group()
+group.add_argument(
+    '--server',
+    type=server_check,
+    help=('Custodia server location, supports http://, https://, '
+          'or path to a unix socket.')
+)
+group.add_argument(
+    '--instance',
+    default=os.getenv('CUSTODIA_INSTANCE', 'custodia'),
+    type=instance_check,
+    help="Instance name (default: CUSTODIA_INSTANCE or 'custodia')",
+)
+
+main_parser.add_argument(
+    '--uds-urlpath', type=str, default='/secrets/',
+    help='URL path for Unix Domain Socket'
+)
+
+main_parser.add_argument(
+    '--header', type=split_header, action='append',
+    help='Extra headers'
+)
+
+main_parser.add_argument(
+    '--verbose', action='store_true',
+)
+main_parser.add_argument(
+    '--debug', action='store_true',
+)
+
+main_parser.add_argument(
+    '--timeout', type=timeout, default=10.,
+    help='Connection timeout'
+)
+
+# TLS
+main_parser.add_argument(
+    '--cafile', type=str, default=None,
+    help='PEM encoded file with root CAs'
+)
+
+# authentication mechanisms
+# TLS client cert auth
+tlsclient_group = main_parser.add_argument_group(
+    title="TLS client cert auth"
+)
+tlsclient_group.add_argument(
+    '--certfile', type=str, default=None,
+    help='PEM encoded file with certs for TLS client authentication'
+)
+tlsclient_group.add_argument(
+    '--keyfile', type=str, default=None,
+    help='PEM encoded key file (if not given, key is read from certfile)'
+)
+
+# Use Negotiate / GSSAPI
+gssapi_group = main_parser.add_argument_group(
+    title="GSSAPI auth"
+)
+gssapi_group.add_argument(
+    '--gssapi', action='store_true',
+    help='Use Negotiate / GSSAPI auth'
+)
+
+
+# handlers
+def handle_name(args):
+    client = args.client_conn
+    func = getattr(client, args.command)
+    return func(args.name)
+
+
+def handle_name_value(args):
+    client = args.client_conn
+    func = getattr(client, args.command)
+    return func(args.name, args.value)
+
+
+# subparsers
+subparsers = main_parser.add_subparsers()
+subparsers.required = True
+
+parser_create_container = subparsers.add_parser(
+    'mkdir',
+    help='Create a container')
+parser_create_container.add_argument('name', type=str, help='key')
+parser_create_container.set_defaults(
+    func=handle_name,
+    command='create_container',
+    sub='mkdir',
+)
+
+parser_delete_container = subparsers.add_parser(
+    'rmdir',
+    help='Delete a container')
+parser_delete_container.add_argument('name', type=str, help='key')
+parser_delete_container.set_defaults(
+    func=handle_name,
+    command='delete_container',
+    sub='rmdir',
+)
+
+parser_list_container = subparsers.add_parser(
+    'ls', help='List content of a container')
+parser_list_container.add_argument('name', type=str, help='key')
+parser_list_container.set_defaults(
+    func=handle_name,
+    command='list_container',
+    sub='ls',
+)
+
+parser_get_secret = subparsers.add_parser(
+    'get', help='Get secret')
+parser_get_secret.add_argument('name', type=str, help='key')
+parser_get_secret.set_defaults(
+    func=handle_name,
+    command='get_secret',
+    sub='get',
+)
+
+parser_set_secret = subparsers.add_parser(
+    'set', help='Set secret')
+parser_set_secret.add_argument('name', type=str, help='key')
+parser_set_secret.add_argument('value', type=str, help='value')
+parser_set_secret.set_defaults(
+    command='set_secret',
+    func=handle_name_value,
+    sub='set'
+)
+
+parser_del_secret = subparsers.add_parser(
+    'del', help='Delete a secret')
+parser_del_secret.add_argument('name', type=str, help='key')
+parser_del_secret.set_defaults(
+    func=handle_name,
+    command='del_secret',
+    sub='del',
+)
+
+
+# plugins
+PLUGINS = [
+    'custodia.authenticators', 'custodia.authorizers', 'custodia.clients',
+    'custodia.consumers', 'custodia.stores'
+]
+
+
+def handle_plugins(args):
+    result = []
+    errmsg = "**ERR** {0} ({1.__class__.__name__}: {1})"
+    for plugin in PLUGINS:
+        result.append('[{}]'.format(plugin))
+        eps = pkg_resources.iter_entry_points(plugin)
+        eps = sorted(eps, key=operator.attrgetter('name'))
+        for ep in eps:
+            try:
+                if hasattr(ep, 'resolve'):
+                    ep.resolve()
+                else:
+                    ep.load(require=False)
+            except Exception as e:  # pylint: disable=broad-except
+                if args.verbose:
+                    result.append(errmsg.format(ep, e))
+            else:
+                result.append(str(ep))
+        result.append('')
+    return result[:-1]
+
+
+parser_plugins = subparsers.add_parser(
+    'plugins', help='List plugins')
+parser_plugins.set_defaults(
+    func=handle_plugins,
+    command='plugins',
+    sub='plugins',
+    name=None,
+)
+parser_plugins.add_argument(
+    '--verbose',
+    action='store_true',
+    help="Verbose mode, show failing plugins."
+)
+
+
+def error_message(args, exc):
+    out = StringIO()
+    parts = urlparse(args.server)
+
+    if args.debug:
+        traceback.print_exc(file=out)
+        out.write('\n')
+
+    out.write("ERROR: Custodia command '{args.sub} {args.name}' failed.\n")
+    if args.verbose:
+        out.write("Custodia server '{args.server}'.\n")
+
+    if isinstance(exc, requests.exceptions.HTTPError):
+        errcode = E_HTTP_ERROR
+        out.write("{exc.__class__.__name__}: {exc}\n")
+    elif isinstance(exc, requests.exceptions.ConnectionError):
+        errcode = E_CONNECTION_ERROR
+        if parts.scheme == 'http+unix':
+            out.write("Failed to connect to Unix socket '{unix_path}':\n")
+        else:
+            out.write("Failed to connect to '{parts.netloc}' "
+                      "({parts.scheme}):\n")
+        # ConnectionError always contains an inner exception
+        out.write("    {exc.args[0]}\n")
+    elif isinstance(exc, JSONDecodeError):
+        errcode = E_JSON_ERROR
+        out.write("Server returned invalid JSON response:\n")
+        out.write("    {exc}\n")
+    else:
+        errcode = E_OTHER
+        out.write("{exc.__class__.__name__}: {exc}\n")
+
+    msg = out.getvalue()
+    if not msg.endswith('\n'):
+        msg += '\n'
+    return errcode, msg.format(args=args, exc=exc, parts=parts,
+                               unix_path=unquote(parts.netloc))
+
+
+def parse_args(arglist=None):
+    args = main_parser.parse_args(arglist)
+
+    if args.keyfile and not args.certfile:
+        main_parser.error("keyfile without certfile is not supported\n")
+    # mutually exclusive groups don't supported nested subgroups
+    if args.gssapi and args.certfile:
+        main_parser.error("gssapi and certfile are mutually exclusive.\n")
+    if args.gssapi and requests_gssapi is None:
+        main_parser.error(
+            "'requests_gssapi' package is not available! You can install "
+            "it with: 'pip install custodia[gssapi]'.\n"
+        )
+
+    if args.debug:
+        args.verbose = True
+
+    if not args.server:
+        instance_socket = '/var/run/custodia/{}.sock'.format(args.instance)
+        args.server = 'http+unix://{}'.format(url_escape(instance_socket, ''))
+
+    if args.server.startswith('http+unix://'):
+        # append uds-path
+        if not args.server.endswith('/'):
+            udspath = args.uds_urlpath
+            if not udspath.startswith('/'):
+                udspath = '/' + udspath
+            args.server += udspath
+
+    args.client_conn = CustodiaSimpleClient(args.server)
+    args.client_conn.timeout = args.timeout
+    if args.header is not None:
+        args.client_conn.headers.update(args.header)
+    if args.cafile:
+        args.client_conn.set_ca_cert(args.cafile)
+    # authentication
+    if args.certfile:
+        args.client_conn.set_client_cert(args.certfile, args.keyfile)
+        args.client_conn.headers['CUSTODIA_CERT_AUTH'] = 'true'
+    elif args.gssapi:
+        args.client_conn.set_gssapi_auth()
+
+    return args
+
+
+def main():
+    args = parse_args()
+
+    log.setup_logging(debug=args.debug, auditfile=None)
+
+    try:
+        result = args.func(args)
+    except BaseException as e:
+        errcode, msg = error_message(args, e)
+        main_parser.exit(errcode, msg)
+    else:
+        if result is not None:
+            if isinstance(result, list):
+                print('\n'.join(result))
+            else:
+                print(result)
+
+
+if __name__ == '__main__':
+    main()

ipaserver/custodia/cli/__main__.py

@@ -0,0 +1,7 @@
+# Copyright (C) 2016  Custodia Project Contributors - see LICENSE file
+from __future__ import absolute_import
+
+from custodia.cli import main
+
+if __name__ == '__main__':
+    main()

ipaserver/custodia/client.py

@@ -0,0 +1,306 @@
+# Copyright (C) 2015  Custodia Project Contributors - see LICENSE file
+from __future__ import absolute_import
+
+
+import socket
+
+from jwcrypto.common import json_decode
+from jwcrypto.jwk import JWK
+
+import requests
+from requests.adapters import HTTPAdapter
+from requests.compat import unquote, urlparse
+# requests 2.18 no longer vendors urllib3
+# pylint: disable=import-error
+from requests.packages.urllib3.connection import HTTPConnection
+from requests.packages.urllib3.connectionpool import HTTPConnectionPool
+# pylint: enable=import-error
+
+try:
+    import requests_gssapi
+except ImportError:
+    requests_gssapi = None
+
+from custodia.log import getLogger
+from custodia.message.kem import (
+    check_kem_claims, decode_enc_kem, make_enc_kem
+)
+
+
+logger = getLogger(__name__)
+
+
+class HTTPUnixConnection(HTTPConnection):
+
+    def __init__(self, host, timeout=60, **kwargs):
+        # pylint: disable=bad-super-call
+        super(HTTPConnection, self).__init__('localhost')
+        self.unix_socket = host
+        self.timeout = timeout
+
+    def connect(self):
+        s = socket.socket(family=socket.AF_UNIX)
+        s.settimeout(self.timeout)
+        s.connect(self.unix_socket)
+        self.sock = s  # pylint: disable=attribute-defined-outside-init
+
+
+class HTTPUnixConnectionPool(HTTPConnectionPool):
+
+    scheme = 'http+unix'
+    ConnectionCls = HTTPUnixConnection
+
+
+class HTTPUnixAdapter(HTTPAdapter):
+
+    def get_connection(self, url, proxies=None):
+        # proxies, silently ignored
+        path = unquote(urlparse(url).netloc)
+        return HTTPUnixConnectionPool(path)
+
+
+DEFAULT_HEADERS = {'Content-Type': 'application/json'}
+
+
+class CustodiaHTTPClient(object):
+    timeout = None  # seconds (float)
+
+    def __init__(self, url):
+        self.session = requests.Session()
+        self.session.mount('http+unix://', HTTPUnixAdapter())
+        self.headers = dict(DEFAULT_HEADERS)
+        self.url = url
+        self._last_response = None
+
+    def set_simple_auth_keys(self, name, key,
+                             name_header='CUSTODIA_AUTH_ID',
+                             key_header='CUSTODIA_AUTH_KEY'):
+        self.headers[name_header] = name
+        self.headers[key_header] = key
+
+    def set_ca_cert(self, cafile):
+        self.session.verify = cafile
+
+    def set_client_cert(self, certfile, keyfile=None):
+        if keyfile is None:
+            self.session.cert = certfile
+        else:
+            self.session.cert = (certfile, keyfile)
+
+    def set_gssapi_auth(self, **kwargs):
+        if requests_gssapi is None:
+            raise ImportError('requests_gssapi')
+        self.session.auth = requests_gssapi.HTTPSPNEGOAuth(**kwargs)
+
+    def _join_url(self, path):
+        return self.url.rstrip('/') + '/' + path.lstrip('/')
+
+    def _add_headers(self, **kwargs):
+        headers = kwargs.get('headers', None)
+        if headers is None:
+            headers = dict()
+        headers.update(self.headers)
+        return headers
+
+    def _request(self, cmd, path, **kwargs):
+        self._last_response = None
+        url = self._join_url(path)
+        kwargs.setdefault('timeout', self.timeout)
+        kwargs['headers'] = self._add_headers(**kwargs)
+        logger.debug("%s %s", cmd.__name__.upper(), url)
+        self._last_response = cmd(url, **kwargs)
+        logger.debug("Response: %s", self._last_response)
+        return self._last_response
+
+    @property
+    def last_response(self):
+        return self._last_response
+
+    def delete(self, path, **kwargs):
+        return self._request(self.session.delete, path, **kwargs)
+
+    def get(self, path, **kwargs):
+        return self._request(self.session.get, path, **kwargs)
+
+    def head(self, path, **kwargs):
+        return self._request(self.session.head, path, **kwargs)
+
+    def patch(self, path, **kwargs):
+        return self._request(self.session.patch, path, **kwargs)
+
+    def post(self, path, **kwargs):
+        return self._request(self.session.post, path, **kwargs)
+
+    def put(self, path, **kwargs):
+        return self._request(self.session.put, path, **kwargs)
+
+    def container_name(self, name):
+        return name if name.endswith('/') else name + '/'
+
+    def create_container(self, name):
+        raise NotImplementedError
+
+    def list_container(self, name):
+        raise NotImplementedError
+
+    def delete_container(self, name):
+        raise NotImplementedError
+
+    def get_secret(self, name):
+        raise NotImplementedError
+
+    def set_secret(self, name, value):
+        raise NotImplementedError
+
+    def del_secret(self, name):
+        raise NotImplementedError
+
+
+class CustodiaSimpleClient(CustodiaHTTPClient):
+
+    def create_container(self, name):
+        r = self.post(self.container_name(name))
+        r.raise_for_status()
+
+    def delete_container(self, name):
+        r = self.delete(self.container_name(name))
+        r.raise_for_status()
+
+    def list_container(self, name):
+        r = self.get(self.container_name(name))
+        r.raise_for_status()
+        return r.json()
+
+    def get_secret(self, name):
+        r = self.get(name)
+        r.raise_for_status()
+        simple = r.json()
+        ktype = simple.get("type", None)
+        if ktype != "simple":
+            raise TypeError("Invalid key type: %s" % ktype)
+        return simple["value"]
+
+    def set_secret(self, name, value):
+        r = self.put(name, json={"type": "simple", "value": value})
+        r.raise_for_status()
+
+    def del_secret(self, name):
+        r = self.delete(name)
+        r.raise_for_status()
+
+
+class CustodiaKEMClient(CustodiaHTTPClient):
+    def __init__(self, url):
+        super(CustodiaKEMClient, self).__init__(url)
+        self._cli_signing_key = None
+        self._cli_decryption_key = None
+        self._srv_verifying_key = None
+        self._srv_encryption_key = None
+        self._sig_alg = None
+        self._enc_alg = None
+
+    def _decode_key(self, key):
+        if key is None:
+            return None
+        elif isinstance(key, JWK):
+            return key
+        elif isinstance(key, dict):
+            return JWK(**key)
+        elif isinstance(key, str):
+            return JWK(**(json_decode(key)))
+        else:
+            raise TypeError("Invalid key type")
+
+    def set_server_public_keys(self, sig, enc):
+        self._srv_verifying_key = self._decode_key(sig)
+        self._srv_encryption_key = self._decode_key(enc)
+
+    def set_client_keys(self, sig, enc):
+        self._cli_signing_key = self._decode_key(sig)
+        self._cli_decryption_key = self._decode_key(enc)
+
+    def set_algorithms(self, sig, enc):
+        self._sig_alg = sig
+        self._enc_alg = enc
+
+    def _signing_algorithm(self, key):
+        if self._sig_alg is not None:
+            return self._sig_alg
+        elif key.key_type == 'RSA':
+            return 'RS256'
+        elif key.key_type == 'EC':
+            return 'ES256'
+        else:
+            raise ValueError('Unsupported key type')
+
+    def _encryption_algorithm(self, key):
+        if self._enc_alg is not None:
+            return self._enc_alg
+        elif key.key_type == 'RSA':
+            return ('RSA-OAEP', 'A256CBC-HS512')
+        elif key.key_type == 'EC':
+            return ('ECDH-ES+A256KW', 'A256CBC-HS512')
+        else:
+            raise ValueError('Unsupported key type')
+
+    def _kem_wrap(self, name, value):
+        if self._cli_signing_key is None:
+            raise KeyError("Client Signing key is not available")
+        if self._srv_encryption_key is None:
+            raise KeyError("Server Encryption key is not available")
+        sig_alg = self._signing_algorithm(self._cli_signing_key)
+        enc_alg = self._encryption_algorithm(self._srv_encryption_key)
+        return make_enc_kem(name, value,
+                            self._cli_signing_key, sig_alg,
+                            self._srv_encryption_key, enc_alg)
+
+    def _kem_unwrap(self, name, message):
+        if message.get("type", None) != "kem":
+            raise TypeError("Invalid token type, expected 'kem', got %s" % (
+                            message.get("type", None),))
+
+        if self._cli_decryption_key is None:
+            raise KeyError("Client Decryption key is not available")
+        if self._srv_verifying_key is None:
+            raise KeyError("Server Verifying key is not available")
+        claims = decode_enc_kem(message["value"],
+                                self._cli_decryption_key,
+                                self._srv_verifying_key)
+        check_kem_claims(claims, name)
+        return claims
+
+    def create_container(self, name):
+        cname = self.container_name(name)
+        message = self._kem_wrap(cname, None)
+        r = self.post(cname, json={"type": "kem", "value": message})
+        r.raise_for_status()
+        self._kem_unwrap(cname, r.json())
+
+    def delete_container(self, name):
+        cname = self.container_name(name)
+        message = self._kem_wrap(cname, None)
+        r = self.delete(cname, json={"type": "kem", "value": message})
+        r.raise_for_status()
+        self._kem_unwrap(cname, r.json())
+
+    def list_container(self, name):
+        return self.get_secret(self.container_name(name))
+
+    def get_secret(self, name):
+        message = self._kem_wrap(name, None)
+        r = self.get(name, params={"type": "kem", "value": message})
+        r.raise_for_status()
+        claims = self._kem_unwrap(name, r.json())
+        return claims['value']
+
+    def set_secret(self, name, value):
+        message = self._kem_wrap(name, value)
+        r = self.put(name, json={"type": "kem", "value": message})
+        r.raise_for_status()
+        self._kem_unwrap(name, r.json())
+
+    def del_secret(self, name):
+        message = self._kem_wrap(name, None)
+        r = self.delete(name, json={"type": "kem", "value": message})
+        r.raise_for_status()
+        self._kem_unwrap(name, r.json())

ipaserver/custodia/compat.py

@@ -0,0 +1,25 @@
+# Copyright (C) 2015  Custodia Project Contributors - see LICENSE file
+"""Python 2/3 compatibility
+"""
+# pylint: disable=no-name-in-module,import-error
+from __future__ import absolute_import
+
+import six
+
+
+if six.PY2:
+    # use https://pypi.python.org/pypi/configparser/ on Python 2
+    from backports import configparser
+    from urllib import quote as url_escape
+    from urllib import quote_plus, unquote
+    from urlparse import parse_qs, urlparse
+else:
+    import configparser
+    from urllib.parse import quote as url_escape
+    from urllib.parse import parse_qs, quote_plus, unquote, urlparse
+
+
+__all__ = (
+    'configparser',
+    'parse_qs', 'quote_plus', 'unquote', 'url_escape', 'urlparse'
+)

ipaserver/custodia/forwarder.py

@@ -0,0 +1,88 @@
+# Copyright (C) 2015  Custodia Project Contributors - see LICENSE file
+from __future__ import absolute_import
+
+import uuid
+
+from custodia.client import CustodiaHTTPClient
+from custodia.plugin import HTTPConsumer, HTTPError
+from custodia.plugin import INHERIT_GLOBAL, PluginOption, REQUIRED
+
+
+class Forwarder(HTTPConsumer):
+    forward_uri = PluginOption(str, REQUIRED, None)
+    tls_cafile = PluginOption(str, INHERIT_GLOBAL(None), 'Path to CA file')
+    tls_certfile = PluginOption(
+        str, None, 'Path to cert file for client cert auth')
+    tls_keyfile = PluginOption(
+        str, None, 'Path to key file for client cert auth')
+    forward_headers = PluginOption('json', '{}', None)
+    prefix_remote_user = PluginOption(bool, True, None)
+    timeout = PluginOption(float, 10.0, 'Connection timeout in seconds')
+
+    def __init__(self, config, section):
+        super(Forwarder, self).__init__(config, section)
+        self.client = CustodiaHTTPClient(self.forward_uri)
+        if self.tls_certfile is not None:
+            self.client.set_client_cert(self.tls_certfile, self.tls_keyfile)
+        if self.tls_cafile is not None:
+            self.client.set_ca_cert(self.tls_cafile)
+        self.client.timeout = self.timeout
+        self.uuid = str(uuid.uuid4())
+        # pylint: disable=unsubscriptable-object
+        # pylint: disable=unsupported-assignment-operation
+        self.forward_headers['X-LOOP-CUSTODIA'] = self.uuid
+
+    def _path(self, request):
+        trail = request.get('trail', [])
+        if self.prefix_remote_user:
+            prefix = [request.get('remote_user', 'guest').rstrip('/')]
+        else:
+            prefix = []
+        return '/'.join(prefix + trail)
+
+    def _headers(self, request):
+        headers = {}
+        headers.update(self.forward_headers)
+        loop = request['headers'].get('X-LOOP-CUSTODIA', None)
+        if loop is not None:
+            headers['X-LOOP-CUSTODIA'] += ',' + loop
+        return headers
+
+    def _response(self, reply, response):
+        if reply.status_code < 200 or reply.status_code > 299:
+            raise HTTPError(reply.status_code)
+        response['code'] = reply.status_code
+        if reply.content:
+            response['output'] = reply.content
+
+    def _request(self, cmd, request, response, path, **kwargs):
+        if self.uuid in request['headers'].get('X-LOOP-CUSTODIA', ''):
+            raise HTTPError(502, "Loop detected")
+        reply = cmd(path, **kwargs)
+        self._response(reply, response)
+
+    def GET(self, request, response):
+        self._request(self.client.get, request, response,
+                      self._path(request),
+                      params=request.get('query', None),
+                      headers=self._headers(request))
+
+    def PUT(self, request, response):
+        self._request(self.client.put, request, response,
+                      self._path(request),
+                      data=request.get('body', None),
+                      params=request.get('query', None),
+                      headers=self._headers(request))
+
+    def DELETE(self, request, response):
+        self._request(self.client.delete, request, response,
+                      self._path(request),
+                      params=request.get('query', None),
+                      headers=self._headers(request))
+
+    def POST(self, request, response):
+        self._request(self.client.post, request, response,
+                      self._path(request),
+                      data=request.get('body', None),
+                      params=request.get('query', None),
+                      headers=self._headers(request))

ipaserver/custodia/httpd/authenticators.py

@@ -0,0 +1,134 @@
+# Copyright (C) 2015  Custodia Project Contributors - see LICENSE file
+from __future__ import absolute_import
+
+import os
+
+from cryptography.hazmat.primitives import constant_time
+
+from custodia import log
+from custodia.plugin import HTTPAuthenticator, PluginOption
+
+
+class SimpleCredsAuth(HTTPAuthenticator):
+    uid = PluginOption('pwd_uid', -1, "User id or name, -1 ignores user")
+    gid = PluginOption('grp_gid', -1, "Group id or name, -1 ignores group")
+
+    def handle(self, request):
+        creds = request.get('creds')
+        if creds is None:
+            self.logger.debug('SCA: Missing "creds" from request')
+            return False
+        uid = int(creds['uid'])
+        gid = int(creds['gid'])
+        uid_match = self.uid != -1 and self.uid == uid
+        gid_match = self.gid != -1 and self.gid == gid
+        if uid_match or gid_match:
+            self.audit_svc_access(log.AUDIT_SVC_AUTH_PASS,
+                                  request['client_id'],
+                                  "%d, %d" % (uid, gid))
+            return True
+        else:
+            self.audit_svc_access(log.AUDIT_SVC_AUTH_FAIL,
+                                  request['client_id'],
+                                  "%d, %d" % (uid, gid))
+            return False
+
+
+class SimpleHeaderAuth(HTTPAuthenticator):
+    header = PluginOption(str, 'REMOTE_USER', "header name")
+    value = PluginOption('str_set', None,
+                         "Comma-separated list of required values")
+
+    def handle(self, request):
+        if self.header not in request['headers']:
+            self.logger.debug('SHA: No "headers" in request')
+            return None
+        value = request['headers'][self.header]
+        if self.value is not None:
+            # pylint: disable=unsupported-membership-test
+            if value not in self.value:
+                self.audit_svc_access(log.AUDIT_SVC_AUTH_FAIL,
+                                      request['client_id'], value)
+                return False
+
+        self.audit_svc_access(log.AUDIT_SVC_AUTH_PASS,
+                              request['client_id'], value)
+        request['remote_user'] = value
+        return True
+
+
+class SimpleAuthKeys(HTTPAuthenticator):
+    id_header = PluginOption(str, 'CUSTODIA_AUTH_ID', "auth id header name")
+    key_header = PluginOption(str, 'CUSTODIA_AUTH_KEY', "auth key header name")
+    store = PluginOption('store', None, None)
+    store_namespace = PluginOption(str, 'custodiaSAK', "")
+
+    def _db_key(self, name):
+        return os.path.join(self.store_namespace, name)
+
+    def handle(self, request):
+        name = request['headers'].get(self.id_header, None)
+        key = request['headers'].get(self.key_header, None)
+        if name is None and key is None:
+            self.logger.debug('Ignoring request no relevant headers provided')
+            return None
+
+        validated = False
+        try:
+            val = self.store.get(self._db_key(name))
+            if val is None:
+                raise ValueError("No such ID")
+            if constant_time.bytes_eq(val.encode('utf-8'),
+                                      key.encode('utf-8')):
+                validated = True
+        except Exception:  # pylint: disable=broad-except
+            self.audit_svc_access(log.AUDIT_SVC_AUTH_FAIL,
+                                  request['client_id'], name)
+            return False
+
+        if validated:
+            self.audit_svc_access(log.AUDIT_SVC_AUTH_PASS,
+                                  request['client_id'], name)
+            request['remote_user'] = name
+            return True
+
+        self.audit_svc_access(log.AUDIT_SVC_AUTH_FAIL,
+                              request['client_id'], name)
+        return False
+
+
+class SimpleClientCertAuth(HTTPAuthenticator):
+    header = PluginOption(str, 'CUSTODIA_CERT_AUTH', "header name")
+
+    def handle(self, request):
+        cert_auth = request['headers'].get(self.header, "false").lower()
+        client_cert = request['client_cert']  # {} or None
+        if not client_cert or cert_auth not in {'1', 'yes', 'true', 'on'}:
+            self.logger.debug('Ignoring request no relevant header or cert'
+                              ' provided')
+            return None
+
+        subject = client_cert.get('subject', {})
+        dn = []
+        name = None
+        # TODO: check SAN first
+        for rdn in subject:
+            for key, value in rdn:
+                dn.append('{}="{}"'.format(key, value.replace('"', r'\"')))
+                if key == 'commonName':
+                    name = value
+                    break
+
+        dn = ', '.join(dn)
+        self.logger.debug('Client cert subject: {}, serial: {}'.format(
+            dn, client_cert.get('serialNumber')))
+
+        if name:
+            self.audit_svc_access(log.AUDIT_SVC_AUTH_PASS,
+                                  request['client_id'], name)
+            request['remote_user'] = name
+            return True
+
+        self.audit_svc_access(log.AUDIT_SVC_AUTH_FAIL,
+                              request['client_id'], dn)
+        return False

ipaserver/custodia/httpd/authorizers.py

@@ -0,0 +1,80 @@
+# Copyright (C) 2015  Custodia Project Contributors - see LICENSE file
+from __future__ import absolute_import
+
+import os
+
+from custodia import log
+from custodia.plugin import HTTPAuthorizer, PluginOption
+
+
+class SimplePathAuthz(HTTPAuthorizer):
+    # keep SimplePathAuthz an old-style plugin for now.
+    # KEMKeysStore and IPAKEMKeys haven't been ported.
+
+    def __init__(self, config):
+        super(SimplePathAuthz, self).__init__(config)
+        self.paths = []
+        if 'paths' in self.config:
+            self.paths = self.config['paths'].split()
+
+    def handle(self, request):
+        reqpath = path = request.get('path', '')
+
+        # if an authorized path does not end in /
+        # check if it matches fullpath for strict match
+        for authz in self.paths:  # pylint: disable=not-an-iterable
+            if authz.endswith('/'):
+                continue
+            if authz.endswith('.'):
+                # special case to match a path ending in /
+                authz = authz[:-1]
+            if authz == path:
+                self.audit_svc_access(log.AUDIT_SVC_AUTHZ_PASS,
+                                      request['client_id'], path)
+                return True
+
+        while path != '':
+            # pylint: disable=unsupported-membership-test
+            if path in self.paths:
+                self.audit_svc_access(log.AUDIT_SVC_AUTHZ_PASS,
+                                      request['client_id'], path)
+                return True
+            if path == '/':
+                path = ''
+            else:
+                path, _ = os.path.split(path)
+
+        self.logger.debug('No path in %s matched %s', self.paths, reqpath)
+        return None
+
+
+class UserNameSpace(HTTPAuthorizer):
+    path = PluginOption(str, '/', 'User namespace path')
+    store = PluginOption('store', None, None)
+
+    def handle(self, request):
+        # Only check if we are in the right (sub)path
+        path = request.get('path', '/')
+        if not path.startswith(self.path):
+            self.logger.debug('%s is not contained in %s', path, self.path)
+            return None
+
+        name = request.get('remote_user', None)
+        if name is None:
+            # UserNameSpace requires a user ...
+            self.audit_svc_access(log.AUDIT_SVC_AUTHZ_FAIL,
+                                  request['client_id'], path)
+            return False
+
+        # pylint: disable=no-member
+        namespace = self.path.rstrip('/') + '/' + name + '/'
+        if not path.startswith(namespace):
+            # Not in the namespace
+            self.audit_svc_access(log.AUDIT_SVC_AUTHZ_FAIL,
+                                  request['client_id'], path)
+            return False
+
+        request['default_namespace'] = name
+        self.audit_svc_access(log.AUDIT_SVC_AUTHZ_PASS,
+                              request['client_id'], path)
+        return True

ipaserver/custodia/httpd/consumer.py

@@ -0,0 +1,13 @@
+# Copyright (C) 2015  Custodia Project Contributors - see LICENSE file
+from __future__ import absolute_import
+
+import warnings
+
+from custodia.plugin import DEFAULT_CTYPE, HTTPConsumer, SUPPORTED_COMMANDS
+
+
+__all__ = ('DEFAULT_CTYPE', 'SUPPORTED_COMMANDS', 'HTTPConsumer')
+
+
+warnings.warn('custodia.httpd.consumer is deprecated, import from '
+              'custodia.plugin instead.', DeprecationWarning)

ipaserver/custodia/httpd/server.py

@@ -0,0 +1,569 @@
+# Copyright (C) 2015  Custodia Project Contributors - see LICENSE file
+from __future__ import absolute_import
+
+import atexit
+import errno
+import os
+import shutil
+import socket
+import ssl
+import struct
+import sys
+import warnings
+
+import six
+
+from custodia import log
+from custodia.compat import parse_qs, unquote, urlparse
+from custodia.plugin import HTTPError
+
+# pylint: disable=import-error,no-name-in-module
+if six.PY2:
+    from BaseHTTPServer import BaseHTTPRequestHandler
+    from SocketServer import ForkingTCPServer, BaseServer
+else:
+    from http.server import BaseHTTPRequestHandler
+    from socketserver import ForkingTCPServer, BaseServer
+# pylint: enable=import-error,no-name-in-module
+
+try:
+    from systemd import daemon as sd  # pylint: disable=import-error
+except ImportError:
+    sd = None
+    if 'NOTIFY_SOCKET' in os.environ:
+        warnings.warn(
+            "NOTIFY_SOCKET env var is set but python-systemd bindings are "
+            "not available!",
+            category=RuntimeWarning
+        )
+    if 'LISTEN_FDS' in os.environ:
+        warnings.warn(
+            "LISTEN_FDS env var is set, but python-systemd bindings are"
+            "not available!",
+            category=RuntimeWarning
+        )
+
+
+logger = log.getLogger(__name__)
+
+SO_PEERCRED = getattr(socket, 'SO_PEERCRED', 17)
+SO_PEERSEC = getattr(socket, 'SO_PEERSEC', 31)
+SELINUX_CONTEXT_LEN = 256
+MAX_REQUEST_SIZE = 10 * 1024 * 1024  # For now limit body to 10MiB
+
+
+class ForkingHTTPServer(ForkingTCPServer):
+    """
+    A forking HTTP Server.
+    Each request runs into a forked server so that the whole environment
+    is clean and isolated, and parallel requests cannot unintentionally
+    influence one another.
+
+    When a request is received it is parsed by the handler_class provided
+    at server initialization.
+    """
+    server_string = "Custodia/0.1"
+    allow_reuse_address = True
+    socket_file = None
+
+    def __init__(self, server_address, handler_class, config,
+                 bind_and_activate=True):
+        # pylint: disable=super-init-not-called, non-parent-init-called
+        # Init just BaseServer, TCPServer creates a socket.
+        BaseServer.__init__(self, server_address, handler_class)
+
+        if isinstance(server_address, socket.socket):
+            # It's a bound and activates socket from systemd.
+            self.socket = server_address
+            bind_and_activate = False
+        else:
+            self.socket = socket.socket(self.address_family,
+                                        self.socket_type)
+
+        # copied from TCPServer
+        if bind_and_activate:
+            try:
+                self.server_bind()
+                self.server_activate()
+            except BaseException:
+                self.server_close()
+                raise
+
+        if self.socket.family == socket.AF_UNIX:
+            self.socket_file = self.socket.getsockname()
+
+        if 'consumers' not in config:
+            raise ValueError('Configuration does not provide any consumer')
+        self.config = config
+        if 'server_string' in self.config:
+            self.server_string = self.config['server_string']
+        self.auditlog = log.auditlog
+
+
+class ForkingUnixHTTPServer(ForkingHTTPServer):
+    address_family = socket.AF_UNIX
+
+    def server_bind(self):
+        self.unlink()
+        # Remove on exit
+        atexit.register(self.unlink)
+        basedir = os.path.dirname(self.server_address)
+        if not os.path.isdir(basedir):
+            os.makedirs(basedir, mode=0o755)
+        ForkingHTTPServer.server_bind(self)
+        os.chmod(self.server_address, 0o666)
+
+    def unlink(self):
+        try:
+            os.unlink(self.server_address)
+        except OSError:
+            pass
+
+
+class ForkingTLSServer(ForkingHTTPServer):
+    def __init__(self, server_address, handler_class, config, context=None,
+                 bind_and_activate=True):
+        ForkingHTTPServer.__init__(self, server_address, handler_class, config,
+                                   bind_and_activate=bind_and_activate)
+        if context is None:
+            try:
+                self._context = self._mkcontext()
+            except Exception as e:
+                logger.error(
+                    "Failed to create a SSLContext for TLS server: %s", e
+                )
+                raise
+        else:
+            self._context = context
+
+    def _mkcontext(self):
+        certfile = self.config.get('tls_certfile')
+        keyfile = self.config.get('tls_keyfile')
+        cafile = self.config.get('tls_cafile')
+        capath = self.config.get('tls_capath')
+        if self.config.get('tls_verify_client', False):
+            verifymode = ssl.CERT_REQUIRED
+        else:
+            verifymode = ssl.CERT_NONE
+
+        if not certfile:
+            raise ValueError('tls_certfile is not set.')
+
+        logger.info(
+            "Creating SSLContext for TLS server (cafile: '%s', capath: '%s', "
+            "verify client: %s).",
+            cafile, capath, verifymode == ssl.CERT_REQUIRED
+        )
+        context = ssl.create_default_context(
+            ssl.Purpose.CLIENT_AUTH,
+            cafile=cafile,
+            capath=capath)
+        context.verify_mode = verifymode
+        logger.info(
+            "Loading cert chain '%s' (keyfile: '%s')", certfile, keyfile)
+        context.load_cert_chain(certfile, keyfile)
+        return context
+
+    def get_request(self):
+        conn, client_addr = self.socket.accept()
+        sslconn = self._context.wrap_socket(conn, server_side=True)
+        return sslconn, client_addr
+
+
+class HTTPRequestHandler(BaseHTTPRequestHandler):
+
+    """
+    This request handler is a slight modification of BaseHTTPRequestHandler
+    where the per-request handler is replaced.
+
+    When a request comes in it is parsed and the 'request' dictionary is
+    populated accordingly. Additionally a 'creds' structure is added to the
+    request.
+
+    The 'creds' structure contains the data retrieved via a call to
+    getsockopt with the SO_PEERCRED option. This retrieves via kernel assist
+    the uid,gid and pid of the process on the other side of the unix socket
+    on which the request has been made. This can be used for authentication
+    and/or authorization purposes.
+    The 'creds' structure is further augmented with a 'context' option
+    containing the Selinux Context string for the calling process, if
+    available.
+
+    after the request is parsed the server's pipeline() function is invoked
+    in order to handle it. The pipeline() should return a response object,
+    where te return 'code', the 'output' and 'headers' may be found.
+
+    If no 'code' is present the request is assumed to be successful and a
+    '200 OK' status code will be sent back to the client.
+
+    The 'output' parameter can be a string or a file like object.
+
+    The 'headers' objct must be a dictionary where keys are headers names.
+
+    By default we assume HTTP1.0
+    """
+
+    protocol_version = "HTTP/1.0"
+
+    def __init__(self, request, client_address, server):
+        self.requestline = ''
+        self.request_version = ''
+        self.command = ''
+        self.raw_requestline = None
+        self.close_connection = 0
+        self.path = None  # quoted, raw path
+        self.path_chain = None  # tuple of unquoted segments
+        self.query = None
+        self.url = None
+        self.body = None
+        self.loginuid = None
+        self._creds = False
+        BaseHTTPRequestHandler.__init__(self, request, client_address, server)
+
+    def version_string(self):
+        return self.server.server_string
+
+    def _get_loginuid(self, pid):
+        loginuid = None
+        # NOTE: Using proc to find the login uid is not reliable
+        # this is why login uid is fetched separately and not stored
+        # into 'creds', to avoid giving the false impression it can be
+        # used to perform access control decisions
+        try:
+            with open("/proc/%i/loginuid" % pid, "r") as f:
+                loginuid = int(f.read())
+        except IOError as e:
+            if e.errno != errno.ENOENT:
+                raise
+        if loginuid == -1:
+            loginuid = None
+        return loginuid
+
+    @property
+    def peer_creds(self):
+        if self._creds is not False:
+            return self._creds
+        # works only for unix sockets
+        if self.request.family != socket.AF_UNIX:
+            self._creds = None
+            return self._creds
+        # pid_t: signed int32, uid_t/gid_t: unsigned int32
+        fmt = 'iII'
+        creds = self.request.getsockopt(socket.SOL_SOCKET, SO_PEERCRED,
+                                        struct.calcsize(fmt))
+        pid, uid, gid = struct.unpack(fmt, creds)
+        try:
+            creds = self.request.getsockopt(socket.SOL_SOCKET, SO_PEERSEC,
+                                            SELINUX_CONTEXT_LEN)
+            context = creds.rstrip(b'\x00').decode('utf-8')
+        except Exception:  # pylint: disable=broad-except
+            logger.debug("Couldn't retrieve SELinux Context", exc_info=True)
+            context = None
+
+        self._creds = {'pid': pid, 'uid': uid, 'gid': gid, 'context': context}
+        return self._creds
+
+    @property
+    def peer_info(self):
+        if self.peer_creds is not None:
+            return self._creds['pid']
+        elif self.request.family in {socket.AF_INET, socket.AF_INET6}:
+            return self.request.getpeername()
+        return None
+
+    @property
+    def peer_cert(self):
+        if not hasattr(self.request, 'getpeercert'):
+            return None
+        return self.request.getpeercert()
+
+    def parse_request(self):
+        if not BaseHTTPRequestHandler.parse_request(self):
+            return False
+
+        # grab the loginuid from `/proc` as soon as possible
+        creds = self.peer_creds
+        if creds is not None:
+            self.loginuid = self._get_loginuid(creds['pid'])
+
+        # after basic parsing also use urlparse to retrieve individual
+        # elements of a request.
+        url = urlparse(self.path)
+
+        # Yes, override path with the path part only
+        self.path = url.path
+        self.path_chain = self._parse_path(url)
+
+        # Create dict out of query
+        self.query = parse_qs(url.query)
+
+        # keep the rest into the 'url' element in case someone needs it
+        self.url = url
+
+        return True
+
+    def _parse_path(self, url):
+        path_chain = []
+        for segment in url.path.split('/'):
+            # unquote URL path encoding
+            segment = unquote(segment)
+            path_chain.append(segment)
+        return tuple(path_chain)
+
+    def parse_body(self):
+        length = int(self.headers.get('content-length', 0))
+        if length > MAX_REQUEST_SIZE:
+            raise HTTPError(413)
+        if length == 0:
+            self.body = None
+        else:
+            self.body = self.rfile.read(length)
+
+    def handle_one_request(self):
+        if self.request.family == socket.AF_UNIX:
+            # Set a fake client address to make log functions happy
+            self.client_address = ['127.0.0.1', 0]
+        try:
+            if not self.server.config:
+                self.close_connection = 1
+                return
+            self.raw_requestline = self.rfile.readline(65537)
+            if not self.raw_requestline:
+                self.close_connection = 1
+                return
+            if len(self.raw_requestline) > 65536:
+                self.requestline = ''
+                self.request_version = ''
+                self.command = ''
+                self.send_error(414)
+                self.wfile.flush()
+                return
+            if not self.parse_request():
+                self.close_connection = 1
+                return
+            try:
+                self.parse_body()
+            except HTTPError as e:
+                self.send_error(e.code, e.mesg)
+                self.wfile.flush()
+                return
+            request = {'creds': self.peer_creds,
+                       'client_cert': self.peer_cert,
+                       'client_id': self.peer_info,
+                       'command': self.command,
+                       'path': self.path,
+                       'path_chain': self.path_chain,
+                       'query': self.query,
+                       'url': self.url,
+                       'version': self.request_version,
+                       'headers': self.headers,
+                       'body': self.body}
+            logger.debug(
+                "REQUEST: %s %s, query: %r, cred: %r, client_id: %s, "
+                "headers: %r, body: %r",
+                request['command'], request['path_chain'], request['query'],
+                request['creds'], request['client_id'],
+                dict(request['headers']), request['body']
+            )
+            try:
+                response = self.pipeline(self.server.config, request)
+                if response is None:
+                    raise HTTPError(500)
+            except HTTPError as e:
+                self.send_error(e.code, e.mesg)
+                self.wfile.flush()
+                return
+            except socket.timeout as e:
+                self.log_error("Request timed out: %r", e)
+                self.close_connection = 1
+                return
+            except Exception as e:  # pylint: disable=broad-except
+                self.log_error("Handler failed: %r", e, exc_info=True)
+                self.send_error(500)
+                self.wfile.flush()
+                return
+
+            self.send_response(response.get('code', 200))
+            for header, value in six.iteritems(response.get('headers', {})):
+                self.send_header(header, value)
+            self.end_headers()
+
+            output = response.get('output', None)
+            if hasattr(output, 'read'):
+                shutil.copyfileobj(output, self.wfile)
+                output.close()
+            elif output is not None:
+                self.wfile.write(output)
+            else:
+                self.close_connection = 1
+            self.wfile.flush()
+            return
+        except socket.timeout as e:
+            self.log_error("Request timed out: %r", e)
+            self.close_connection = 1
+            return
+
+    # pylint: disable=arguments-differ
+    def log_error(self, fmtstr, *args, **kwargs):
+        logger.error(fmtstr, *args, **kwargs)
+
+    def pipeline(self, config, request):
+        """
+        The pipeline() function handles authentication and invocation of
+        the correct consumer based on the server configuration, that is
+        provided at initialization time.
+
+        When authentication is performed all the authenticators are
+        executed. If any returns False, authentication fails and a 403
+        error is raised. If none of them positively succeeds and they all
+        return None then also authentication fails and a 403 error is
+        raised. Authentication plugins can add attributes to the request
+        object for use of authorization or other plugins.
+
+        When authorization is performed and positive result will cause the
+        operation to be accepted and any negative result will cause it to
+        fail. If no authorization plugin returns a positive result a 403
+        error is returned.
+
+        Once authentication and authorization are successful the pipeline
+        will parse the path component and find the consumer plugin that
+        handles the provided path walking up the path component by
+        component until a consumer is found.
+
+        Paths are walked up from the leaf to the root, so if two consumers
+        hang on the same tree, the one closer to the leaf will be used. If
+        there is a trailing path when the conumer is selected then it will
+        be stored in the request dicstionary named 'trail'. The 'trail' is
+        an ordered list of the path components below the consumer entry
+        point.
+        """
+        path_chain = request['path_chain']
+        if not path_chain or path_chain[0] != '':
+            # no path or not an absolute path
+            raise HTTPError(400)
+
+        # auth framework here
+        authers = config.get('authenticators')
+        if authers is None:
+            raise HTTPError(403)
+        valid_once = False
+        for auth in authers:
+            valid = authers[auth].handle(request)
+            if valid is False:
+                raise HTTPError(403)
+            elif valid is True:
+                valid_once = True
+        if valid_once is not True:
+            self.server.auditlog.svc_access(self.__class__.__name__,
+                                            log.AUDIT_SVC_AUTH_FAIL,
+                                            request['client_id'], 'No auth')
+            raise HTTPError(403)
+
+        # auhz framework here
+        authzers = config.get('authorizers')
+        if authzers is None:
+            raise HTTPError(403)
+        authz_ok = None
+        for authz in authzers:
+            valid = authzers[authz].handle(request)
+            if valid is True:
+                authz_ok = True
+            elif valid is False:
+                authz_ok = False
+                break
+        if authz_ok is not True:
+            self.server.auditlog.svc_access(self.__class__.__name__,
+                                            log.AUDIT_SVC_AUTHZ_FAIL,
+                                            request['client_id'],
+                                            path_chain)
+            raise HTTPError(403)
+
+        # Select consumer
+        trail = []
+        while path_chain:
+            if path_chain in config['consumers']:
+                con = config['consumers'][path_chain]
+                if len(trail) != 0:
+                    request['trail'] = trail
+                return con.handle(request)
+            trail.insert(0, path_chain[-1])
+            path_chain = path_chain[:-1]
+
+        raise HTTPError(404)
+
+
+class HTTPServer(object):
+    handler = HTTPRequestHandler
+
+    def __init__(self, srvurl, config):
+        url = urlparse(srvurl)
+        serverclass, address = self._get_serverclass(url)
+        if sd is not None:
+            address = self._get_systemd_socket(address)
+        self.httpd = serverclass(address, self.handler, config)
+
+    def _get_serverclass(self, url):
+        if url.scheme == 'http+unix':
+            # Unix socket
+            address = unquote(url.netloc)
+            if not address:
+                raise ValueError('Empty address {}'.format(url))
+            logger.info('Serving on Unix socket %s', address)
+            serverclass = ForkingUnixHTTPServer
+        elif url.scheme == 'http':
+            host, port = url.netloc.split(":")
+            address = (host, int(port))
+            logger.info('Serving on %s (HTTP)', url.netloc)
+            serverclass = ForkingHTTPServer
+        elif url.scheme == 'https':
+            host, port = url.netloc.split(":")
+            address = (host, int(port))
+            logger.info('Serving on %s (HTTPS)', url.netloc)
+            serverclass = ForkingTLSServer
+        else:
+            raise ValueError('Unknown URL Scheme: %s' % url.scheme)
+        return serverclass, address
+
+    def _get_systemd_socket(self, address):
+        fds = sd.listen_fds()
+        if not fds:
+            return address
+        elif len(fds) > 1:
+            raise ValueError('Too many listening sockets', fds)
+
+        if isinstance(address, tuple):
+            port = address[1]
+            # systemd uses IPv6
+            if not sd.is_socket_inet(fds[0], family=socket.AF_INET6,
+                                     type=socket.SOCK_STREAM,
+                                     listening=True, port=port):
+                raise ValueError(
+                    "FD {} is not TCP IPv6 socket on port {}".format(
+                        fds[0], port
+                    )
+                )
+            logger.info('Using systemd socket activation on port %i', port)
+            sock = socket.fromfd(fds[0], socket.AF_INET6, socket.SOCK_STREAM)
+        else:
+            if not sd.is_socket_unix(fds[0], socket.SOCK_STREAM,
+                                     listening=True, path=address):
+                raise ValueError(
+                    "FD {} is not Unix stream socket on path {}".format(
+                        fds[0], address
+                    )
+                )
+            logger.info('Using systemd socket activation on path %s', address)
+            sock = socket.fromfd(fds[0], socket.AF_UNIX, socket.SOCK_STREAM)
+
+        if sys.version_info[0] < 3:
+            # Python 2.7's socket.fromfd() returns _socket.socket
+            sock = socket.socket(_sock=sock)
+        return sock
+
+    def get_socket(self):
+        return (self.httpd.socket, self.httpd.socket_file)
+
+    def serve(self):
+        if sd is not None and sd.booted():
+            sd.notify("READY=1")
+        return self.httpd.serve_forever()

ipaserver/custodia/ipa/__init__.py

@@ -0,0 +1 @@
+# Copyright (C) 2016  Custodia Project Contributors - see LICENSE file

ipaserver/custodia/ipa/certrequest.py

@@ -0,0 +1,349 @@
+# Copyright (C) 2017  Custodia Project Contributors - see LICENSE file
+"""FreeIPA cert request store
+"""
+from __future__ import absolute_import
+
+import abc
+import base64
+import datetime
+import textwrap
+
+from cryptography import x509
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives import hashes
+from cryptography.hazmat.primitives import serialization
+from cryptography.hazmat.primitives.asymmetric import rsa
+from cryptography.x509 import oid
+
+from ipalib.errors import AuthorizationError, NotFound
+from ipalib.krb_utils import krb5_format_service_principal_name
+
+
+import six
+
+from custodia.plugin import CSStore, PluginOption, REQUIRED
+from custodia.plugin import CSStoreDenied, CSStoreError
+
+from .interface import IPAInterface
+
+
+TLS_SERVERAUTH = oid.ObjectIdentifier('2.5.29.37.1')
+
+
+@six.add_metaclass(abc.ABCMeta)
+class _CSRGenerator(object):
+    """Build and sign certificate signing request
+    """
+    TEMPLATE = textwrap.dedent("""\
+        Issuer: {issuer}
+        Subject: {subject}
+        Serial Number: {cert.serial_number}
+        Validity:
+            Not Before: {cert.not_valid_before}
+            Not After: {cert.not_valid_after}
+        {pem}\
+    """)
+
+    def __init__(self, plugin, backend=None):
+        if backend is None:
+            self.backend = default_backend()
+        else:
+            self.backend = backend
+        self.plugin = plugin
+        self._privkey = self._gen_private()
+
+    def _gen_private(self):
+        """Generate private key
+        """
+        return rsa.generate_private_key(
+            public_exponent=65537,
+            key_size=self.plugin.key_size,
+            backend=self.backend
+        )
+
+    @abc.abstractmethod
+    def build_csr(self, **kwargs):
+        """Generate a certificate signing request builder
+        """
+
+    def _sign_csr(self, builder):
+        return builder.sign(self._privkey, hashes.SHA256(), self.backend)
+
+    @abc.abstractmethod
+    def _cert_request(self, csr_pem, **kwargs):
+        """Request certificate from IPA
+        """
+
+    def request_cert(self, builder, **kwargs):
+        """Send CSR and request certificate
+        """
+        signed = self._sign_csr(builder)
+        csr_pem = signed.public_bytes(serialization.Encoding.PEM)
+        if not isinstance(csr_pem, six.text_type):
+            csr_pem = csr_pem.decode('ascii')
+
+        response = self._cert_request(csr_pem, **kwargs)
+
+        if self.plugin.chain:
+            certs = tuple(
+                x509.load_der_x509_certificate(cert, self.backend)
+                for cert in response[u'result'][u'certificate_chain']
+            )
+        else:
+            # certificate is just base64 without BEGIN/END certificate
+            cert = base64.b64decode(response[u'result'][u'certificate'])
+            certs = (x509.load_der_x509_certificate(cert, self.backend), )
+
+        pem = [self._dump_privkey(self._privkey)]
+        pem.extend(self._dump_cert(cert) for cert in certs)
+        return response, '\n'.join(pem)
+
+    def _dump_cert(self, cert):
+        pem = cert.public_bytes(encoding=serialization.Encoding.PEM)
+        if six.PY3:
+            pem = pem.decode('ascii')
+        return self.TEMPLATE.format(
+            issuer=self._dump_x509name(cert.issuer),
+            subject=self._dump_x509name(cert.subject),
+            cert=cert,
+            pem=pem
+        )
+
+    def _dump_x509name(self, name):
+        # no quoting, just for debugging
+        out = []
+        # pylint: disable=protected-access
+        for nameattr in list(name):
+            out.append("{}={}".format(nameattr.oid._name, nameattr.value))
+        # pylint: enable=protected-access
+        return ', '.join(out)
+
+    def _dump_privkey(self, privkey):
+        privkey = privkey.private_bytes(
+            encoding=serialization.Encoding.PEM,
+            format=serialization.PrivateFormat.TraditionalOpenSSL,
+            encryption_algorithm=serialization.NoEncryption()
+        )
+        if six.PY3:
+            privkey = privkey.decode('ascii')
+        return privkey
+
+
+class _ServerCSRGenerator(_CSRGenerator):
+    # pylint: disable=arguments-differ
+    def build_csr(self, hostname, **kwargs):
+        realm = self.plugin.ipa.env.realm
+        builder = x509.CertificateSigningRequestBuilder()
+        builder = builder.subject_name(
+            x509.Name([
+                x509.NameAttribute(oid.NameOID.COMMON_NAME, hostname),
+                x509.NameAttribute(oid.NameOID.ORGANIZATION_NAME, realm),
+            ])
+        )
+        build = builder.add_extension(
+            x509.BasicConstraints(ca=False, path_length=None), critical=True,
+        )
+        build = builder.add_extension(
+            x509.ExtendedKeyUsage([TLS_SERVERAUTH]), critical=True
+        )
+        builder = build.add_extension(
+            x509.SubjectAlternativeName([x509.DNSName(hostname)]),
+            critical=False
+        )
+        return builder
+
+    # pylint: disable=arguments-differ
+    def _cert_request(self, pem_req, principal, **kwargs):
+        # FreeIPA 4.4 has no chain option, only pass kwarg when enabled
+        if self.plugin.chain:
+            kwargs['chain'] = True
+        with self.plugin.ipa as ipa:
+            return ipa.Command.cert_request(
+                pem_req,
+                profile_id=self.plugin.cert_profile,
+                add=self.plugin.add_principal,
+                principal=principal,
+                **kwargs
+            )
+
+
+class IPACertRequest(CSStore):
+    """IPA cert request store
+
+    The IPACertRequest store plugin generates or revokes certificates on the
+    fly. It uses a backing store to cache certs and private keys.
+
+    The request ```GET /secrets/certs/HTTP/client1.ipa.example``` generates a
+    private key and CSR for the service ```HTTP/client1.ipa.example``` with
+    DNS subject alternative name ```client1.ipa.example```.
+
+    A DELETE request removes the cert/key pair from the backing store and
+    revokes the cert at the same time.
+    """
+    backing_store = PluginOption(str, REQUIRED, None)
+
+    key_size = PluginOption(int, 2048, 'RSA key size')
+    cert_profile = PluginOption(str, 'caIPAserviceCert', 'IPA cert profile')
+    add_principal = PluginOption(bool, True, 'Add missing principal')
+    chain = PluginOption(bool, True, 'Return full cert chain')
+    allowed_services = PluginOption('str_set', {'HTTP'}, 'Service prefixes')
+    revocation_reason = PluginOption(
+        int, 4, 'Cert revocation reason (4: superseded)')
+
+    def __init__(self, config, section=None):
+        super(IPACertRequest, self).__init__(config, section)
+        self.store_name = self.backing_store
+        self.store = None
+        self.ipa = None
+        if not isinstance(self.cert_profile, six.text_type):
+            self.cert_profile = self.cert_profile.decode('utf-8')
+
+    def finalize_init(self, config, cfgparser, context=None):
+        super(IPACertRequest, self).finalize_init(config, cfgparser, context)
+        if self.ipa is not None:
+            return
+        self.ipa = IPAInterface.from_config(config)
+        self.ipa.finalize_init(config, cfgparser, context=self)
+
+    def _parse_key(self, key):
+        if not isinstance(key, six.text_type):
+            key = key.decode('utf-8')
+        parts = key.split(u'/')
+        # XXX why is 'keys' added in in Secrets._db_key()?
+        if len(parts) != 3 or parts[0] != 'keys':
+            raise CSStoreDenied("Invalid cert request key '{}'".format(key))
+        service, hostname = parts[1:3]
+        # pylint: disable=unsupported-membership-test
+        if service not in self.allowed_services:
+            raise CSStoreDenied("Invalid service '{}'".format(key))
+        principal = krb5_format_service_principal_name(
+            service, hostname, self.ipa.env.realm
+        )
+        # use cert prefix in storage key
+        key = u"cert/{}/{}".format(service, hostname)
+        return key, hostname, principal
+
+    def get(self, key):
+        # check key first
+        key, hostname, principal = self._parse_key(key)
+        value = self.store.get(key)
+        if value is not None:
+            # TODO: validate certificate
+            self.logger.info("Found cached certificate for %s", principal)
+            return value
+        # found no cached key/cert pair, generate one
+        try:
+            data = self._request_cert(hostname, principal)
+        except AuthorizationError:
+            msg = "Unauthorized request for '{}' ({})".format(
+                hostname, principal
+            )
+            self.logger.exception(msg)
+            raise CSStoreDenied(msg)
+        except NotFound:
+            msg = "Host '{}' or principal '{}' not found".format(
+                hostname, principal
+            )
+            self.logger.exception(msg)
+            raise CSStoreDenied(msg)
+        except Exception:
+            msg = "Failed to request cert '{}' ({})".format(
+                hostname, principal
+            )
+            self.logger.exception(msg)
+            raise CSStoreError(msg)
+        self.store.set(key, data, replace=True)
+        return data
+
+    def set(self, key, value, replace=False):
+        key, hostname, principal = self._parse_key(key)
+        del hostname, principal
+        return self.store.set(key, value, replace)
+
+    def span(self, key):
+        key, hostname, principal = self._parse_key(key)
+        del hostname, principal
+        return self.store.span(key)
+
+    def list(self, keyfilter=''):
+        return self.store.list(keyfilter)
+
+    def cut(self, key):
+        key, hostname, principal = self._parse_key(key)
+        certs = self._revoke_certs(hostname, principal)
+        return self.store.cut(key) or certs
+
+    def _request_cert(self, hostname, principal):
+        self.logger.info("Requesting certificate for %s", hostname)
+        csrgen = _ServerCSRGenerator(plugin=self)
+        builder = csrgen.build_csr(hostname=hostname)
+        response, pem = csrgen.request_cert(builder, principal=principal)
+        self.logger.info(
+            "Got certificate for '%s', request id %s, serial number %s",
+            response[u'result'][u'subject'],
+            response[u'result'][u'request_id'],
+            response[u'result'][u'serial_number'],
+        )
+        return pem
+
+    def _revoke_certs(self, hostname, principal):
+        with self.ipa as ipa:
+            response = ipa.Command.cert_find(
+                service=principal,
+                validnotafter_from=datetime.datetime.utcnow(),
+            )
+            # XXX cert_find has no filter for valid cert
+            certs = list(
+                cert for cert in response['result']
+                if not cert[u'revoked']
+            )
+            for cert in certs:
+                self.logger.info(
+                    'Revoking cert %i (subject: %s, issuer: %s)',
+                    cert[u'serial_number'], cert[u'subject'],
+                    cert[u'issuer']
+                )
+                ipa.Command.cert_revoke(
+                    cert[u'serial_number'],
+                    revocation_reason=self.revocation_reason,
+                )
+            return certs
+
+
+def test():
+    from custodia.compat import configparser
+    from custodia.log import setup_logging
+    from .interface import IPA_SECTIONNAME
+    from .vault import IPAVault
+
+    parser = configparser.ConfigParser(
+        interpolation=configparser.ExtendedInterpolation()
+    )
+    parser.read_string(u"""
+    [auth:ipa]
+    handler = IPAInterface
+    [store:ipa_vault]
+    handler = IPAVault
+    [store:ipa_certreq]
+    handler = IPAVault
+    backing_store = ipa_vault
+    """)
+
+    setup_logging(debug=True, auditfile=None)
+    config = {
+        'authenticators': {
+            'ipa': IPAInterface(parser, IPA_SECTIONNAME)
+        }
+    }
+    vault = IPAVault(parser, 'store:ipa_vault')
+    vault.finalize_init(config, parser, None)
+    s = IPACertRequest(parser, 'store:ipa_certreq')
+    s.store = vault
+    s.finalize_init(config, parser, None)
+    print(s.get('keys/HTTP/client1.ipa.example'))
+    print(s.get('keys/HTTP/client1.ipa.example'))
+    print(s.cut('keys/HTTP/client1.ipa.example'))
+
+
+if __name__ == '__main__':
+    test()

ipaserver/custodia/ipa/interface.py

@@ -0,0 +1,165 @@
+# Copyright (C) 2017  Custodia Project Contributors - see LICENSE file
+"""IPA API wrapper and interface
+"""
+from __future__ import absolute_import
+
+import os
+import sys
+
+import ipalib
+import ipalib.constants
+from ipalib.krb_utils import get_principal
+
+import six
+
+from custodia.plugin import HTTPAuthenticator, PluginOption
+
+
+IPA_SECTIONNAME = 'auth:ipa'
+
+
+class IPAInterface(HTTPAuthenticator):
+    """IPA interface authenticator
+
+    Custodia uses a forking server model. We can bootstrap FreeIPA API in
+    the main process. Connections must be created in the client process.
+    """
+    # Kerberos flags
+    krb5config = PluginOption(str, None, "Kerberos krb5.conf override")
+    keytab = PluginOption(str, None, "Kerberos keytab for auth")
+    ccache = PluginOption(
+        str, None, "Kerberos ccache, e,g. FILE:/path/to/ccache")
+
+    # ipalib.api arguments
+    ipa_confdir = PluginOption(str, None, "IPA confdir override")
+    ipa_context = PluginOption(str, "cli", "IPA bootstrap context")
+    ipa_debug = PluginOption(bool, False, "debug mode for ipalib")
+
+    # filled by gssapi()
+    principal = False
+
+    def __init__(self, config, section=None, api=None):
+        super(IPAInterface, self).__init__(config, section)
+        # only one instance of this plugin is supported
+        if section != IPA_SECTIONNAME:
+            raise ValueError(section)
+
+        if api is None:
+            self._api = ipalib.api
+        else:
+            self._api = api
+
+        if self._api.isdone('bootstrap'):
+            raise RuntimeError("IPA API already initialized")
+
+        self._ipa_config = dict(
+            context=self.ipa_context,
+            debug=self.ipa_debug,
+            log=None,  # disable logging to file
+        )
+        if self.ipa_confdir is not None:
+            self._ipa_config['confdir'] = self.ipa_confdir
+
+    @classmethod
+    def from_config(cls, config):
+        return config['authenticators']['ipa']
+
+    def finalize_init(self, config, cfgparser, context=None):
+        super(IPAInterface, self).finalize_init(config, cfgparser, context)
+
+        if self.principal:
+            # already initialized
+            return
+
+        # get rundir from own section or DEFAULT
+        rundir = cfgparser.get(self.section, 'rundir', fallback=None)
+        if rundir:
+            self._ipa_config['dot_ipa'] = rundir
+            self._ipa_config['home'] = rundir
+            # workaround https://pagure.io/freeipa/issue/6761#comment-440329
+            # monkey-patch ipalib.constants and all loaded ipa modules
+            ipalib.constants.USER_CACHE_PATH = rundir
+            for name, mod in six.iteritems(sys.modules):
+                if (name.startswith(('ipalib.', 'ipaclient.')) and
+                        hasattr(mod, 'USER_CACHE_PATH')):
+                    mod.USER_CACHE_PATH = rundir
+
+        self._gssapi_config()
+        self._bootstrap()
+        with self:
+            self.logger.info("IPA server '%s': %s",
+                             self.env.server,
+                             self.Command.ping()[u'summary'])
+
+    def handle(self, request):
+        request[IPA_SECTIONNAME] = self
+        return None
+
+    # rest is interface and initialization
+
+    def _gssapi_config(self):
+        # set client keytab env var for authentication
+        if self.keytab is not None:
+            os.environ['KRB5_CLIENT_KTNAME'] = self.keytab
+        if self.ccache is not None:
+            os.environ['KRB5CCNAME'] = self.ccache
+        if self.krb5config is not None:
+            os.environ['KRB5_CONFIG'] = self.krb5config
+
+        self.principal = self._gssapi_cred()
+        self.logger.info(u"Kerberos principal '%s'", self.principal)
+
+    def _gssapi_cred(self):
+        try:
+            return get_principal()
+        except Exception:
+            self.logger.exception(
+                "Unable to get principal from GSSAPI. Are you missing a "
+                "TGT or valid Kerberos keytab?"
+            )
+            raise
+
+    def _bootstrap(self):
+        # TODO: bandaid for "A PKCS #11 module returned CKR_DEVICE_ERROR"
+        # https://github.com/avocado-framework/avocado/issues/1112#issuecomment-206999400
+        os.environ['NSS_STRICT_NOFORK'] = 'DISABLED'
+        self._api.bootstrap(**self._ipa_config)
+        self._api.finalize()
+
+    @property
+    def Command(self):
+        return self._api.Command  # pylint: disable=no-member
+
+    @property
+    def env(self):
+        return self._api.env  # pylint: disable=no-member
+
+    def __enter__(self):
+        # pylint: disable=no-member
+        self._gssapi_cred()
+        if not self._api.Backend.rpcclient.isconnected():
+            self._api.Backend.rpcclient.connect()
+        # pylint: enable=no-member
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        # pylint: disable=no-member
+        if self._api.Backend.rpcclient.isconnected():
+            self._api.Backend.rpcclient.disconnect()
+        # pylint: enable=no-member
+
+
+if __name__ == '__main__':
+    from custodia.compat import configparser
+    from custodia.log import setup_logging
+
+    parser = configparser.ConfigParser(
+        interpolation=configparser.ExtendedInterpolation()
+    )
+    parser.read_string(u"""
+    [auth:ipa]
+    handler = IPAInterface
+    """)
+
+    setup_logging(debug=True, auditfile=None)
+    IPAInterface(parser, "auth:ipa")

ipaserver/custodia/ipa/vault.py

@@ -0,0 +1,246 @@
+# Copyright (C) 2016  Custodia Project Contributors - see LICENSE file
+"""FreeIPA vault store (PoC)
+"""
+from __future__ import absolute_import
+
+from ipalib.errors import AuthorizationError, DuplicateEntry, NotFound
+
+import six
+
+from custodia.plugin import CSStore, PluginOption
+from custodia.plugin import (
+    CSStoreDenied, CSStoreError, CSStoreExists, CSStoreUnsupported
+)
+
+from .interface import IPAInterface
+
+
+def krb5_unparse_principal_name(name):
+    """Split a Kerberos principal name into parts
+
+    Returns:
+       * ('host', hostname, realm) for a host principal
+       * (servicename, hostname, realm) for a service principal
+       * (None, username, realm) for a user principal
+
+    :param text name: Kerberos principal name
+    :return: (service, host, realm) or (None, username, realm)
+    """
+    prefix, realm = name.split(u'@')
+    if u'/' in prefix:
+        service, host = prefix.rsplit(u'/', 1)
+        return service, host, realm
+    else:
+        return None, prefix, realm
+
+
+class IPAVault(CSStore):
+    # vault arguments
+    principal = PluginOption(
+        str, None,
+        "Service principal for service vault (auto-discovered from GSSAPI)"
+    )
+    user = PluginOption(
+        str, None,
+        "User name for user vault (auto-discovered from GSSAPI)"
+    )
+    vault_type = PluginOption(
+        str, None,
+        "vault type, one of 'user', 'service', 'shared', or "
+        "auto-discovered from GSSAPI"
+    )
+
+    def __init__(self, config, section=None, api=None):
+        super(IPAVault, self).__init__(config, section)
+        self._vault_args = None
+        self.ipa = None
+
+    def finalize_init(self, config, cfgparser, context=None):
+        super(IPAVault, self).finalize_init(config, cfgparser, context)
+
+        if self.ipa is not None:
+            return
+        self.ipa = IPAInterface.from_config(config)
+        self.ipa.finalize_init(config, cfgparser, context=self)
+
+        # connect
+        with self.ipa:
+            # retrieve and cache KRA transport cert
+            response = self.ipa.Command.vaultconfig_show()
+            servers = response[u'result'].get(u'kra_server_server', ())
+            if servers:
+                self.logger.info("KRA server(s) %s", ', '.join(servers))
+
+        service, user_host, realm = krb5_unparse_principal_name(
+            self.ipa.principal)
+        self._init_vault_args(service, user_host, realm)
+
+    def _init_vault_args(self, service, user_host, realm):
+        if self.vault_type is None:
+            self.vault_type = 'user' if service is None else 'service'
+            self.logger.info("Setting vault type to '%s' from Kerberos",
+                             self.vault_type)
+
+        if self.vault_type == 'shared':
+            self._vault_args = {'shared': True}
+        elif self.vault_type == 'user':
+            if self.user is None:
+                if service is not None:
+                    msg = "{!r}: User vault requires 'user' parameter"
+                    raise ValueError(msg.format(self))
+                else:
+                    self.user = user_host
+                    self.logger.info(u"Setting username '%s' from Kerberos",
+                                     self.user)
+            if six.PY2 and isinstance(self.user, str):
+                self.user = self.user.decode('utf-8')
+            self._vault_args = {'username': self.user}
+        elif self.vault_type == 'service':
+            if self.principal is None:
+                if service is None:
+                    msg = "{!r}: Service vault requires 'principal' parameter"
+                    raise ValueError(msg.format(self))
+                else:
+                    self.principal = u'/'.join((service, user_host))
+                    self.logger.info(u"Setting principal '%s' from Kerberos",
+                                     self.principal)
+            if six.PY2 and isinstance(self.principal, str):
+                self.principal = self.principal.decode('utf-8')
+            self._vault_args = {'service': self.principal}
+        else:
+            msg = '{!r}: Invalid vault type {}'
+            raise ValueError(msg.format(self, self.vault_type))
+
+    def _mangle_key(self, key):
+        if '__' in key:
+            raise ValueError
+        key = key.replace('/', '__')
+        if isinstance(key, bytes):
+            key = key.decode('utf-8')
+        return key
+
+    def get(self, key):
+        key = self._mangle_key(key)
+        with self.ipa as ipa:
+            try:
+                result = ipa.Command.vault_retrieve(
+                    key, **self._vault_args)
+            except NotFound as e:
+                self.logger.info("Key '%s' not found: %s", key, e)
+                return None
+            except Exception:
+                msg = "Failed to retrieve entry {}".format(key)
+                self.logger.exception(msg)
+                raise CSStoreError(msg)
+            else:
+                return result[u'result'][u'data']
+
+    def set(self, key, value, replace=False):
+        key = self._mangle_key(key)
+        if not isinstance(value, bytes):
+            value = value.encode('utf-8')
+        with self.ipa as ipa:
+            try:
+                ipa.Command.vault_add(
+                    key, ipavaulttype=u"standard", **self._vault_args)
+            except DuplicateEntry as e:
+                self.logger.info("Vault '%s' already exists: %s", key, e)
+                if not replace:
+                    raise CSStoreExists(key)
+            except AuthorizationError:
+                msg = "vault_add denied for entry {}".format(key)
+                self.logger.exception(msg)
+                raise CSStoreDenied(msg)
+            except Exception:
+                msg = "Failed to add entry {}".format(key)
+                self.logger.exception(msg)
+                raise CSStoreError(msg)
+            try:
+                ipa.Command.vault_archive(
+                    key, data=value, **self._vault_args)
+            except AuthorizationError:
+                msg = "vault_archive denied for entry {}".format(key)
+                self.logger.exception(msg)
+                raise CSStoreDenied(msg)
+            except Exception:
+                msg = "Failed to archive entry {}".format(key)
+                self.logger.exception(msg)
+                raise CSStoreError(msg)
+
+    def span(self, key):
+        raise CSStoreUnsupported("span is not implemented")
+
+    def list(self, keyfilter=None):
+        with self.ipa as ipa:
+            try:
+                result = ipa.Command.vault_find(
+                    ipavaulttype=u"standard", **self._vault_args)
+            except AuthorizationError:
+                msg = "vault_find denied"
+                self.logger.exception(msg)
+                raise CSStoreDenied(msg)
+            except Exception:
+                msg = "Failed to list entries"
+                self.logger.exception(msg)
+                raise CSStoreError(msg)
+
+        names = []
+        for entry in result[u'result']:
+            cn = entry[u'cn'][0]
+            key = cn.replace('__', '/')
+            if keyfilter is not None and not key.startswith(keyfilter):
+                continue
+            names.append(key.rsplit('/', 1)[-1])
+        return names
+
+    def cut(self, key):
+        key = self._mangle_key(key)
+        with self.ipa as ipa:
+            try:
+                ipa.Command.vault_del(key, **self._vault_args)
+            except NotFound:
+                return False
+            except AuthorizationError:
+                msg = "vault_del denied for entry {}".format(key)
+                self.logger.exception(msg)
+                raise CSStoreDenied(msg)
+            except Exception:
+                msg = "Failed to delete entry {}".format(key)
+                self.logger.exception(msg)
+                raise CSStoreError(msg)
+            else:
+                return True
+
+
+def test():
+    from custodia.compat import configparser
+    from custodia.log import setup_logging
+    from .interface import IPA_SECTIONNAME
+
+    parser = configparser.ConfigParser(
+        interpolation=configparser.ExtendedInterpolation()
+    )
+    parser.read_string(u"""
+    [auth:ipa]
+    handler = IPAInterface
+    [store:ipa_vault]
+    handler = IPAVault
+    """)
+
+    setup_logging(debug=True, auditfile=None)
+    config = {
+        'authenticators': {
+            'ipa': IPAInterface(parser, IPA_SECTIONNAME)
+        }
+    }
+    v = IPAVault(parser, 'store:ipa_vault')
+    v.finalize_init(config, parser, None)
+    v.set('foo', 'bar', replace=True)
+    print(v.get('foo'))
+    print(v.list())
+    v.cut('foo')
+    print(v.list())
+
+
+if __name__ == '__main__':
+    test()

ipaserver/custodia/log.py

@@ -0,0 +1,191 @@
+# Copyright (C) 2015  Custodia Project Contributors - see LICENSE file
+from __future__ import absolute_import
+
+import logging
+import sys
+import warnings
+
+
+import six
+
+
+LOGGING_FORMAT = "%(asctime)s - %(origin)-32s - %(message)s"
+LOGGING_DATEFORMAT = "%Y-%m-%d %H:%M:%S"
+
+
+class OriginContextFilter(logging.Filter):
+    """Context filter to include 'origin' attribute in record
+    """
+    def filter(self, record):
+        if not hasattr(record, 'origin'):
+            record.origin = record.name.split('.')[-1]
+
+        return True
+
+
+class CustodiaFormatter(logging.Formatter):
+    def format(self, record):
+        # customize record.exc_text, Formatter.format() does not modify
+        # exc_text when it has been set before.
+        short_exc = False
+        if record.exc_info and not record.exc_text:
+            if getattr(record, "exc_fullstack", True):
+                record.exc_text = self.formatException(record.exc_info)
+            else:
+                short_exc = True
+                record.exc_text = u"{0.__name__}: {1}".format(
+                    record.exc_info[0], record.exc_info[1]
+                )
+
+        result = super(CustodiaFormatter, self).format(record)
+        if short_exc:
+            # format() adds \n between message and exc_text
+            text, exc = result.rsplit(u'\n', 1)
+            return u"{0} ({1})".format(text, exc)
+        else:
+            return result
+
+
+class CustodiaLoggingAdapter(logging.LoggerAdapter):
+    def __init__(self, plugin, debug):
+        logger = logging.getLogger(
+            '{0.__class__.__module__}.{0.__class__.__name__}'.format(plugin)
+        )
+        logger.setLevel(logging.DEBUG if debug else logging.INFO)
+        extra = {'origin': plugin.origin}
+        super(CustodiaLoggingAdapter, self).__init__(logger, extra=extra)
+
+    # pylint: disable=arguments-differ
+    def exception(self, msg, *args, **kwargs):
+        """Like standard exception() logger but only print stack in debug mode
+        """
+        extra = kwargs.setdefault('extra', {})
+        extra['exc_fullstack'] = self.isEnabledFor(logging.DEBUG)
+        kwargs['exc_info'] = True
+        self.log(logging.ERROR, msg, *args, **kwargs)
+
+
+def getLogger(name):
+    """Create logger with custom exception() method
+    """
+    def exception(self, msg, *args, **kwargs):
+        extra = kwargs.setdefault('extra', {})
+        extra['exc_fullstack'] = self.isEnabledFor(logging.DEBUG)
+        kwargs['exc_info'] = True
+        self.log(logging.ERROR, msg, *args, **kwargs)
+
+    logger = logging.getLogger(name)
+    logger.exception = six.create_bound_method(exception, logger)
+    return logger
+
+
+def setup_logging(debug=False, auditfile=None, handler=None):
+    root_logger = logging.getLogger()
+    # default is stream handler to stderr
+    if handler is None:
+        handler = logging.StreamHandler(sys.stderr)
+
+    # remove handler instance from root handler to prevent multiple
+    # output handlers.
+    handler_cls = type(handler)
+    root_logger.handlers[:] = list(
+        h for h in root_logger.handlers if not isinstance(h, handler_cls)
+    )
+
+    # configure handler
+    handler.setFormatter(CustodiaFormatter(
+        fmt=LOGGING_FORMAT, datefmt=LOGGING_DATEFORMAT
+    ))
+    handler.addFilter(OriginContextFilter())
+    root_logger.addHandler(handler)
+
+    # set logging level
+    custodia_logger = getLogger('custodia')
+    if debug:
+        custodia_logger.setLevel(logging.DEBUG)
+        custodia_logger.debug('Custodia debug logger enabled')
+        # If the global debug is enabled, turn debug on in all 'custodia.*'
+        # loggers
+        logdict = logging.Logger.manager.loggerDict
+        for name, obj in logdict.items():
+            if not isinstance(obj, logging.Logger):
+                continue
+            if name.startswith('custodia.'):
+                obj.setLevel(logging.DEBUG)
+    else:
+        custodia_logger.setLevel(logging.INFO)
+
+    # setup file handler for audit log
+    audit_logger = logging.getLogger('custodia.audit')
+    if auditfile is not None and len(audit_logger.handlers) == 0:
+        audit_fmt = logging.Formatter(LOGGING_FORMAT, LOGGING_DATEFORMAT)
+        audit_hdrl = logging.FileHandler(auditfile)
+        audit_hdrl.setFormatter(audit_fmt)
+        audit_logger.addHandler(audit_hdrl)
+
+        custodia_logger.debug('Custodia audit log: %s', auditfile)
+
+
+AUDIT_NONE = 0
+AUDIT_GET_ALLOWED = 1
+AUDIT_GET_DENIED = 2
+AUDIT_SET_ALLOWED = 3
+AUDIT_SET_DENIED = 4
+AUDIT_DEL_ALLOWED = 5
+AUDIT_DEL_DENIED = 6
+AUDIT_LAST = 7
+AUDIT_SVC_NONE = 8
+AUDIT_SVC_AUTH_PASS = 9
+AUDIT_SVC_AUTH_FAIL = 10
+AUDIT_SVC_AUTHZ_PASS = 11
+AUDIT_SVC_AUTHZ_FAIL = 12
+AUDIT_SVC_LAST = 13
+AUDIT_MESSAGES = [
+    "AUDIT FAILURE",
+    "ALLOWED: '%(client)s' requested key '%(key)s'",  # AUDIT_GET_ALLOWED
+    "DENIED: '%(client)s' requested key '%(key)s'",   # AUDIT_GET_DENIED
+    "ALLOWED: '%(client)s' stored key '%(key)s'",     # AUDIT_SET_ALLOWED
+    "DENIED: '%(client)s' stored key '%(key)s'",      # AUDIT_SET_DENIED
+    "ALLOWED: '%(client)s' deleted key '%(key)s'",    # AUDIT_DEL_ALLOWED
+    "DENIED: '%(client)s' deleted key '%(key)s'",     # AUDIT_DEL_DENIED
+    "AUDIT FAILURE 7",
+    "AUDIT FAILURE 8",
+    "PASS: '%(cli)s' authenticated as '%(name)s'",  # SVC_AUTH_PASS
+    "FAIL: '%(cli)s' authenticated as '%(name)s'",  # SVC_AUTH_FAIL
+    "PASS: '%(cli)s' authorized for '%(name)s'",    # SVC_AUTHZ_PASS
+    "FAIL: '%(cli)s' authorized for '%(name)s'",    # SVC_AUTHZ_FAIL
+    "AUDIT FAILURE 13",
+]
+
+
+class AuditLog(object):
+    def __init__(self, logger):
+        self.logger = logger
+
+    def key_access(self, origin, action, client, keyname):
+        if action <= AUDIT_NONE or action >= AUDIT_LAST:
+            action = AUDIT_NONE
+        msg = AUDIT_MESSAGES[action]
+        args = {'client': client, 'key': keyname}
+        self.logger.info(msg, args, extra={'origin': origin})
+
+    def svc_access(self, origin, action, client, name):
+        if action <= AUDIT_SVC_NONE or action >= AUDIT_SVC_LAST:
+            action = AUDIT_NONE
+        msg = AUDIT_MESSAGES[action]
+        args = {'cli': client, 'name': name}
+        self.logger.info(msg, args, extra={'origin': origin})
+
+
+auditlog = AuditLog(logging.getLogger('custodia.audit'))
+
+
+class ProvisionalWarning(FutureWarning):
+    pass
+
+
+def warn_provisional(modulename, stacklevel=3):
+    msg = ("Module '{}' is a provisional API. It may changed or get "
+           "removed in future releases.")
+    return warnings.warn(msg.format(modulename), ProvisionalWarning,
+                         stacklevel=stacklevel)

ipaserver/custodia/message/common.py

@@ -0,0 +1,68 @@
+# Copyright (C) 2015  Custodia Project Contributors - see LICENSE file
+from __future__ import absolute_import
+
+from custodia.log import getLogger
+
+logger = getLogger(__name__)
+
+
+class InvalidMessage(Exception):
+    """Invalid Message.
+
+    This exception is raised when a message cannot be parsed
+    or validated.
+    """
+    def __init__(self, message=None):
+        logger.debug(message)
+        super(InvalidMessage, self).__init__(message)
+
+
+class UnknownMessageType(Exception):
+    """Unknown Message Type.
+
+    This exception is raised when a message is of an unknown
+    type.
+    """
+    def __init__(self, message=None):
+        logger.debug(message)
+        super(UnknownMessageType, self).__init__(message)
+
+
+class UnallowedMessage(Exception):
+    """Unallowed Message.
+
+    This exception is raise when the message type is know but
+    is not allowed.
+    """
+    def __init__(self, message=None):
+        logger.debug(message)
+        super(UnallowedMessage, self).__init__(message)
+
+
+class MessageHandler(object):
+
+    def __init__(self, request):
+        self.req = request
+        self.name = None
+        self.payload = None
+        self.msg_type = None
+
+    def parse(self, msg, name):
+        """Parses the message.
+
+        :param req: the original request
+        :param msg: a decoded json string with the incoming message
+
+        :raises InvalidMessage: if the message cannot be parsed or validated
+        """
+
+        raise NotImplementedError
+
+    def reply(self, output):
+        """Generates a reply.
+
+        :param req: the original request
+        :param output: a Python object that can be converted to JSON
+        """
+
+        raise NotImplementedError

ipaserver/custodia/message/formats.py

@@ -0,0 +1,64 @@
+# Copyright (C) 2015  Custodia Project Contributors - see LICENSE file
+from __future__ import absolute_import
+
+from custodia.message.common import InvalidMessage
+from custodia.message.common import UnallowedMessage
+from custodia.message.common import UnknownMessageType
+from custodia.message.kem import KEMHandler
+from custodia.message.simple import SimpleKey
+
+
+default_types = ['simple', 'kem']
+
+key_types = {'simple': SimpleKey,
+             'kem': KEMHandler}
+
+
+class Validator(object):
+    """Validates incoming messages."""
+
+    def __init__(self, allowed=None):
+        """Creates a Validator object.
+
+        :param allowed: list of allowed message types (optional)
+        """
+        self.allowed = allowed or default_types
+        self.types = key_types.copy()
+
+    def add_types(self, types):
+        self.types.update(types)
+
+    def parse(self, request, msg, name):
+        if not isinstance(msg, dict):
+            raise InvalidMessage('The message must be a dict')
+
+        if 'type' not in msg:
+            raise InvalidMessage('The type is missing')
+
+        if isinstance(msg['type'], list):
+            if len(msg['type']) != 1:
+                raise InvalidMessage('Type is multivalued: %s' % msg['type'])
+            msg_type = msg['type'][0]
+        else:
+            msg_type = msg['type']
+
+        if 'value' not in msg:
+            raise InvalidMessage('The value is missing')
+
+        if isinstance(msg['value'], list):
+            if len(msg['value']) != 1:
+                raise InvalidMessage('Value is multivalued: %s' % msg['value'])
+            msg_value = msg['value'][0]
+        else:
+            msg_value = msg['value']
+
+        if msg_type not in list(self.types.keys()):
+            raise UnknownMessageType("Type '%s' is unknown" % msg_type)
+
+        if msg_type not in self.allowed:
+            raise UnallowedMessage("Message type '%s' not allowed" % (
+                                   msg_type,))
+
+        handler = self.types[msg_type](request)
+        handler.parse(msg_value, name)
+        return handler

ipaserver/custodia/message/kem.py

@@ -0,0 +1,247 @@
+# Copyright (C) 2015  Custodia Project Contributors - see LICENSE file
+from __future__ import absolute_import
+
+import os
+import time
+
+from jwcrypto.common import json_decode
+from jwcrypto.common import json_encode
+from jwcrypto.jwe import JWE
+from jwcrypto.jwk import JWK
+from jwcrypto.jws import JWS
+from jwcrypto.jwt import JWT
+
+from custodia.httpd.authorizers import SimplePathAuthz
+from custodia.log import getLogger
+from custodia.message.common import InvalidMessage
+from custodia.message.common import MessageHandler
+
+logger = getLogger(__name__)
+
+KEY_USAGE_SIG = 0
+KEY_USAGE_ENC = 1
+KEY_USAGE_MAP = {KEY_USAGE_SIG: 'sig', KEY_USAGE_ENC: 'enc'}
+
+
+class UnknownPublicKey(Exception):
+    def __init__(self, message=None):
+        logger.debug(message)
+        super(UnknownPublicKey, self).__init__(message)
+
+
+class KEMKeysStore(SimplePathAuthz):
+    """A KEM Keys Store.
+
+    This is a store that holds public keys of registered
+    clients allowed to use KEM messages. It takes the form
+    of an authorizer merely for the purpose of attaching
+    itself to a 'request' so that later on the KEM Parser
+    can fetch the appropriate key to verify/decrypt an
+    incoming request and make the payload available.
+
+    The KEM Parser will actually perform additional
+    authorization checks in this case.
+
+    SimplePathAuthz is extended here as we ant to attach the
+    store only to requests on paths we are configured to
+    manage.
+    """
+
+    def __init__(self, config):
+        super(KEMKeysStore, self).__init__(config)
+        self._server_keys = None
+        self._alg = None
+        self._enc = None
+
+    def _db_key(self, kid):
+        return os.path.join('kemkeys', kid)
+
+    def handle(self, request):
+        inpath = super(KEMKeysStore, self).handle(request)
+        if inpath:
+            request['KEMKeysStore'] = self
+        return inpath
+
+    def find_key(self, kid, usage):
+        dbkey = self._db_key('%s/%s' % (KEY_USAGE_MAP[usage], kid))
+        pubkey = self.store.get(dbkey)
+        if pubkey is None:
+            raise UnknownPublicKey(kid)
+        return pubkey
+
+    @property
+    def server_keys(self):
+        if self._server_keys is None:
+            if 'server_keys' not in self.config:
+                raise UnknownPublicKey("Server Keys not defined")
+            skey = self.find_key(self.config['server_keys'], KEY_USAGE_SIG)
+            ekey = self.find_key(self.config['server_keys'], KEY_USAGE_ENC)
+            self._server_keys = [JWK(**(json_decode(skey))),
+                                 JWK(**(json_decode(ekey)))]
+        return self._server_keys
+
+    @property
+    def alg(self):
+        if self._alg is None:
+            alg = self.config.get('signing_algorithm', None)
+            if alg is None:
+                ktype = self.server_keys[KEY_USAGE_SIG].key_type
+                if ktype == 'RSA':
+                    alg = 'RS256'
+                elif ktype == 'EC':
+                    alg = 'ES256'
+                else:
+                    raise ValueError('Key type unsupported for signing')
+            self._alg = alg
+        return self._alg
+
+
+def check_kem_claims(claims, name):
+    if 'sub' not in claims:
+        raise InvalidMessage('Missing subject in payload')
+    if claims['sub'] != name:
+        raise InvalidMessage('Key name %s does not match subject %s' % (
+            name, claims['sub']))
+    if 'exp' not in claims:
+        raise InvalidMessage('Missing expiration time in payload')
+    if claims['exp'] - (10 * 60) > int(time.time()):
+        raise InvalidMessage('Message expiration too far in the future')
+    if claims['exp'] < int(time.time()):
+        raise InvalidMessage('Message Expired')
+
+
+class KEMHandler(MessageHandler):
+    """Handles 'kem' messages"""
+
+    def __init__(self, request):
+        super(KEMHandler, self).__init__(request)
+        self.kkstore = self.req.get('KEMKeysStore', None)
+        if self.kkstore is None:
+            raise Exception('KEM KeyStore not configured')
+        self.client_keys = None
+        self.name = None
+
+    def _get_key(self, header, usage):
+        if 'kid' not in header:
+            raise InvalidMessage("Missing key identifier")
+
+        key = self.kkstore.find_key(header['kid'], usage)
+        if key is None:
+            raise UnknownPublicKey('Key found [kid:%s]' % header['kid'])
+        return json_decode(key)
+
+    def parse(self, msg, name):
+        """Parses the message.
+
+        We check that the message is properly formatted.
+
+        :param msg: a json-encoded value containing a JWS or JWE+JWS token
+
+        :raises InvalidMessage: if the message cannot be parsed or validated
+
+        :returns: A verified payload
+        """
+
+        try:
+            jtok = JWT(jwt=msg)
+        except Exception as e:
+            raise InvalidMessage('Failed to parse message: %s' % str(e))
+
+        try:
+            token = jtok.token
+            if isinstance(token, JWE):
+                token.decrypt(self.kkstore.server_keys[KEY_USAGE_ENC])
+                # If an encrypted payload is received then there must be
+                # a nested signed payload to verify the provenance.
+                payload = token.payload.decode('utf-8')
+                token = JWS()
+                token.deserialize(payload)
+            elif isinstance(token, JWS):
+                pass
+            else:
+                raise TypeError("Invalid Token type: %s" % type(jtok))
+
+            # Retrieve client keys for later use
+            self.client_keys = [
+                JWK(**self._get_key(token.jose_header, KEY_USAGE_SIG)),
+                JWK(**self._get_key(token.jose_header, KEY_USAGE_ENC))]
+
+            # verify token and get payload
+            token.verify(self.client_keys[KEY_USAGE_SIG])
+            claims = json_decode(token.payload)
+        except Exception as e:
+            logger.debug('Failed to validate message', exc_info=True)
+            raise InvalidMessage('Failed to validate message: %s' % str(e))
+
+        check_kem_claims(claims, name)
+        self.name = name
+        self.payload = claims.get('value')
+        self.msg_type = 'kem'
+
+        return {'type': self.msg_type,
+                'value': {'kid': self.client_keys[KEY_USAGE_ENC].key_id,
+                          'claims': claims}}
+
+    def reply(self, output):
+        if self.client_keys is None:
+            raise UnknownPublicKey("Peer key not defined")
+
+        ktype = self.client_keys[KEY_USAGE_ENC].key_type
+        if ktype == 'RSA':
+            enc = ('RSA-OAEP', 'A256CBC-HS512')
+        else:
+            raise ValueError("'%s' type not supported yet" % ktype)
+
+        value = make_enc_kem(self.name, output,
+                             self.kkstore.server_keys[KEY_USAGE_SIG],
+                             self.kkstore.alg,
+                             self.client_keys[1], enc)
+
+        return {'type': 'kem', 'value': value}
+
+
+class KEMClient(object):
+
+    def __init__(self, server_keys, client_keys):
+        self.server_keys = server_keys
+        self.client_keys = client_keys
+
+    def make_request(self, name, value=None, alg="RS256", encalg=None):
+        if encalg is None:
+            return make_sig_kem(name, value,
+                                self.client_keys[KEY_USAGE_SIG], alg)
+        else:
+            return make_enc_kem(name, value,
+                                self.client_keys[KEY_USAGE_SIG], alg,
+                                self.server_keys[KEY_USAGE_ENC], encalg)
+
+    def parse_reply(self, name, message):
+        claims = decode_enc_kem(message,
+                                self.client_keys[KEY_USAGE_ENC],
+                                self.server_keys[KEY_USAGE_SIG])
+        check_kem_claims(claims, name)
+        return claims['value']
+
+
+def make_sig_kem(name, value, key, alg):
+    header = {'kid': key.key_id, 'alg': alg}
+    claims = {'sub': name, 'exp': int(time.time() + (5 * 60))}
+    if value is not None:
+        claims['value'] = value
+    jwt = JWT(header, claims)
+    jwt.make_signed_token(key)
+    return jwt.serialize(compact=True)
+
+
+def make_enc_kem(name, value, sig_key, alg, enc_key, enc):
+    plaintext = make_sig_kem(name, value, sig_key, alg)
+    eprot = {'kid': enc_key.key_id, 'alg': enc[0], 'enc': enc[1]}
+    jwe = JWE(plaintext, json_encode(eprot))
+    jwe.add_recipient(enc_key)
+    return jwe.serialize(compact=True)
+
+
+def decode_enc_kem(message, enc_key, sig_key):
+    jwe = JWT(jwt=message, key=enc_key)
+    jws = JWT(jwt=jwe.claims, key=sig_key)
+    return json_decode(jws.claims)

ipaserver/custodia/message/simple.py

@@ -0,0 +1,42 @@
+# Copyright (C) 2015  Custodia Project Contributors - see LICENSE file
+from __future__ import absolute_import
+
+from six import string_types
+
+from custodia.message.common import InvalidMessage
+from custodia.message.common import MessageHandler
+
+
+class SimpleKey(MessageHandler):
+    """Handles 'simple' messages"""
+
+    def parse(self, msg, name):
+        """Parses a simple message
+
+        :param msg: the json-decoded value
+        :param name: the requested name
+
+        :raises UnknownMessageType: if the type is not 'simple'
+        :raises InvalidMessage: if the message cannot be parsed or validated
+        """
+
+        # On requests we imply 'simple' if there is no input message
+        if msg is None:
+            return
+
+        if not isinstance(msg, string_types):
+            raise InvalidMessage("The 'value' attribute is not a string")
+
+        self.name = name
+        self.payload = msg
+        self.msg_type = 'simple'
+
+    def reply(self, output):
+        if output is None:
+            return None
+
+        if self.name.endswith('/'):
+            # directory listings are pass-through with simple messages
+            return output
+
+        return {'type': self.msg_type, 'value': output}

ipaserver/custodia/plugin.py

@@ -0,0 +1,490 @@
+# Copyright (C) 2016  Custodia Project Contributors - see LICENSE file
+from __future__ import absolute_import
+
+import abc
+import grp
+import inspect
+import json
+import pwd
+import re
+import sys
+
+from jwcrypto.common import json_encode
+
+import six
+
+from .compat import configparser
+from .log import CustodiaLoggingAdapter, auditlog, getLogger
+
+
+logger = getLogger(__name__)
+
+
+class _Required(object):
+    __slots__ = ()
+
+    def __repr__(self):
+        return 'REQUIRED'
+
+
+class INHERIT_GLOBAL(object):  # noqa: N801
+    __slots__ = ('default',)
+
+    def __init__(self, default):
+        self.default = default
+
+    def __repr__(self):
+        return 'INHERIT_GLOBAL({})'.format(self.default)
+
+
+REQUIRED = _Required()
+
+
+class CustodiaException(Exception):
+    pass
+
+
+class HTTPError(CustodiaException):
+    def __init__(self, code=None, message=None):
+        self.code = code if code is not None else 500
+        self.mesg = message
+        errstring = '%d: %s' % (self.code, self.mesg)
+        super(HTTPError, self).__init__(errstring)
+
+
+class CSStoreError(CustodiaException):
+    pass
+
+
+class CSStoreExists(CustodiaException):
+    pass
+
+
+class CSStoreUnsupported(CustodiaException):
+    pass
+
+
+class CSStoreDenied(CustodiaException):
+    pass
+
+
+class OptionHandler(object):
+    """Handler and parser for plugin options
+    """
+    def __init__(self, parser, section):
+        self.parser = parser
+        self.section = section
+        # handler is reserved to look up the plugin class
+        self.seen = {'handler'}
+
+    def get(self, po):
+        """Lookup value for a PluginOption instance
+
+        Args:
+            po: PluginOption
+
+        Returns: converted value
+        """
+        name = po.name
+        typ = po.typ
+        default = po.default
+
+        handler = getattr(self, '_get_{}'.format(typ), None)
+        if handler is None:
+            raise ValueError(typ)
+        self.seen.add(name)
+
+        # pylint: disable=not-callable
+        if not self.parser.has_option(self.section, name):
+            if default is REQUIRED:
+                raise NameError(self.section, name)
+            if isinstance(default, INHERIT_GLOBAL):
+                return handler('global', name, default.default)
+            # don't return default here, give the handler a chance to modify
+            # the default, e.g. pw_uid with default='root' returns 0.
+
+        return handler(self.section, name, default)
+        # pylint: enable=not-callable
+
+    def check_surplus(self):
+        surplus = []
+        for name, _ in self.parser.items(self.section):
+            if (name not in self.seen and not
+                    self.parser.has_option(configparser.DEFAULTSECT, name)):
+                surplus.append(name)
+        return surplus
+
+    def _get_int(self, section, name, default):
+        return self.parser.getint(section, name, fallback=default)
+
+    def _get_oct(self, section, name, default):
+        value = self.parser.get(section, name, fallback=default)
+        return int(value, 8)
+
+    def _get_hex(self, section, name, default):
+        value = self.parser.get(section, name, fallback=default)
+        return int(value, 16)
+
+    def _get_float(self, section, name, default):
+        return self.parser.getfloat(section, name, fallback=default)
+
+    def _get_bool(self, section, name, default):
+        return self.parser.getboolean(section, name, fallback=default)
+
+    def _get_regex(self, section, name, default):
+        value = self.parser.get(section, name, fallback=default)
+        if not value:
+            return None
+        else:
+            return re.compile(value)
+
+    def _get_str(self, section, name, default):
+        return self.parser.get(section, name, fallback=default)
+
+    def _split_string(self, value):
+        if ',' in value:
+            values = value.split(',')
+        else:
+            values = value.split(' ')
+        return list(v.strip() for v in values if v.strip())
+
+    def _get_str_set(self, section, name, default):
+        try:
+            value = self.parser.get(section, name)
+        except configparser.NoOptionError:
+            return default
+        if not value or not value.strip():
+            return None
+        else:
+            return set(self._split_string(value))
+
+    def _get_str_list(self, section, name, default):
+        try:
+            value = self.parser.get(section, name)
+        except configparser.NoOptionError:
+            return default
+        if not value or not value.strip():
+            return None
+        else:
+            return self._split_string(value)
+
+    def _get_store(self, section, name, default):
+        return self.parser.get(section, name, fallback=default)
+
+    def _get_pwd_uid(self, section, name, default):
+        value = self.parser.get(section, name, fallback=default)
+        try:
+            return int(value)
+        except ValueError:
+            return pwd.getpwnam(value).pw_uid
+
+    def _get_grp_gid(self, section, name, default):
+        value = self.parser.get(section, name, fallback=default)
+        try:
+            return int(value)
+        except ValueError:
+            return grp.getgrnam(value).gr_gid
+
+    def _get_json(self, section, name, default):
+        value = self.parser.get(section, name, fallback=default)
+        return json.loads(value)
+
+
+class PluginOption(object):
+    """Plugin option
+
+    code::
+
+        class MyPlugin(CustodiaPlugin):
+            number = PluginOption(int, REQUIRED, 'my value')
+            values = PluginOption('str_list', 'foo bar', 'a list of strings')
+
+
+    config::
+
+        [myplugin]
+        handler = MyPlugin
+        number = 1
+        values = egg spam python
+
+
+    **Supported value types**
+
+    *str*
+      plain string
+    *str_set*
+      set of comma-separated or space-separated strings
+    *str_list*
+      ordered list of comma-separated or space-separated strings
+    *int*
+      number (converted from base 10)
+    *hex*
+      number (converted from base 16)
+    *oct*
+      number (converted from base 8)
+    *float*
+      floating point number
+    *bool*
+      boolean (true: on, true, yes, 1; false: off, false, no, 0)
+    *regex*
+      regular expression string
+    *store*
+      special value for refer to a store plugin
+    *pwd_uid*
+      numeric user id or user name
+    *grp_gid*
+      numeric group id or group name
+    *json*
+      JSON string
+    """
+    __slots__ = ('name', 'typ', 'default', 'doc')
+
+    def __init__(self, typ, default, doc):
+        self.name = None
+        if typ in {str, int, float, bool, oct, hex}:
+            self.typ = typ.__name__
+        else:
+            self.typ = typ
+        self.default = default
+        self.doc = doc
+
+    def __repr__(self):
+        if self.default is REQUIRED:
+            msg = "<Required option {0.name} ({0.typ}): {0.doc}>"
+        else:
+            msg = ("<Option {0.name} ({0.typ}, default: '{0.default}'): "
+                   "{0.doc}>")
+        return msg.format(self)
+
+
+class CustodiaPluginMeta(abc.ABCMeta):
+    def __new__(cls, name, bases, namespace, **kwargs):
+        ncls = super(CustodiaPluginMeta, cls).__new__(
+            cls, name, bases, namespace, **kwargs)
+
+        if sys.version_info < (3, 0):
+            # pylint: disable=deprecated-method
+            args = inspect.getargspec(ncls.__init__).args
+            # pylint: enable=deprecated-method
+        else:
+            sig = inspect.signature(ncls.__init__)  # pylint: disable=no-member
+            args = list(sig.parameters)
+
+        if args[1:3] != ['config', 'section']:
+            # old-style plugin class
+            ncls._options = None  # pylint: disable=protected-access
+            return ncls
+
+        # new-style plugin class
+        # every plugin has a debug option. In case it is not set, the debug
+        # flag from [global] is inherited.
+        if not hasattr(ncls, 'debug'):
+            ncls.debug = PluginOption(bool, INHERIT_GLOBAL(False), '')
+        # get options
+        options = []
+        for name, value in inspect.getmembers(ncls):
+            if not isinstance(value, PluginOption):
+                continue
+            value.name = name
+            options.append(value)
+
+        ncls._options = tuple(options)  # pylint: disable=protected-access
+        return ncls
+
+
+@six.add_metaclass(CustodiaPluginMeta)
+class CustodiaPlugin(object):
+    """Abstract base class for all Custodia plugins
+    """
+    _options = ()
+
+    def __init__(self, config, section=None):
+        origin, debug = self._configure(config, section)
+        self._auditlog = auditlog
+        self.section = section  # plugin loader sets section for old plugins
+        self.origin = origin
+        self.logger = CustodiaLoggingAdapter(self, debug)
+
+    def audit_key_access(self, *args, **kwargs):
+        self._auditlog.key_access(self.origin, *args, **kwargs)
+
+    def audit_svc_access(self, *args, **kwargs):
+        self._auditlog.svc_access(self.origin, *args, **kwargs)
+
+    def _configure(self, config, section):
+        if section is not None and self._options is not None:
+            # new style configuration
+            opt = OptionHandler(config, section)
+            # pylint: disable=not-an-iterable
+            for option in self._options:
+                value = opt.get(option)
+                # special case for store
+                if option.typ == 'store':
+                    if option.name != 'store':
+                        raise ValueError(option.name)
+                    self.store_name = value
+                    self.store = None
+                else:
+                    setattr(self, option.name, value)
+
+            surplus = opt.check_surplus()
+            if surplus:
+                raise ValueError('Surplus options in {}: {}'.format(
+                    section, surplus))
+
+            origin = '%s-[%s]' % (type(self).__name__, section)
+            debug = self.debug  # pylint: disable=no-member
+        else:
+            # old style configuration
+            if config is None:
+                config = {}
+            self.config = config
+            # special case for store
+            if 'store' in config:
+                self.store_name = self.config.get('store')
+                self.store = None
+            origin = config.get('facility_name', self.__class__.__name__)
+            debug = config.get('debug', 'false').lower() == 'true'
+
+        return origin, debug
+
+    def _attach_store(self, config, cfgparser, context):
+        """Attach nested store
+        """
+        if getattr(self, 'store', None) is not None:
+            # already attached
+            return
+        store_plugin = config['stores'].get(self.store_name)
+        if store_plugin is None:
+            raise ValueError(
+                "'{}' references non-existing store '{}'".format(
+                    self.section, self.store_name))
+        # pylint: disable=attribute-defined-outside-init
+        self.store = store_plugin
+        # pylint: enable=attribute-defined-outside-init
+        store_plugin.finalize_init(config, cfgparser, context=self)
+
+    def finalize_init(self, config, cfgparser, context=None):
+        """Two-phase initialization
+
+        Args:
+            config: server config dictionary
+            cfgparser: configparser instance
+            context: initialization context (None for global)
+        """
+        if getattr(self, 'store_name', None) is not None:
+            self._attach_store(config, cfgparser, context)
+
+
+class CSStore(CustodiaPlugin):
+    """Base class for stores
+    """
+    @abc.abstractmethod
+    def get(self, key):
+        pass
+
+    @abc.abstractmethod
+    def set(self, key, value, replace=False):
+        pass
+
+    # relax ABC for now, see https://github.com/latchset/custodia/issues/84
+
+    # @abc.abstractmethod
+    def span(self, key):
+        raise NotImplementedError
+
+    # @abc.abstractmethod
+    def list(self, keyfilter=None):
+        raise NotImplementedError
+
+    # @abc.abstractmethod
+    def cut(self, key):
+        raise NotImplementedError
+
+
+class HTTPAuthorizer(CustodiaPlugin):
+    """Base class for authorizers
+    """
+    @abc.abstractmethod
+    def handle(self, request):
+        pass
+
+
+class HTTPAuthenticator(CustodiaPlugin):
+    """Base class for authenticators
+    """
+    @abc.abstractmethod
+    def handle(self, request):
+        pass
+
+
+DEFAULT_CTYPE = 'text/html; charset=utf-8'
+SUPPORTED_COMMANDS = ['GET', 'PUT', 'POST', 'DELETE']
+
+
+class HTTPConsumer(CustodiaPlugin):
+    """Base class for consumers
+    """
+    def __init__(self, config, section=None):
+        super(HTTPConsumer, self).__init__(config, section)
+        self.subs = dict()
+        self.root = self
+
+    def add_sub(self, name, sub):
+        self.subs[name] = sub
+        if hasattr(sub, 'root'):
+            sub.root = self.root
+
+    def _find_handler(self, request):
+        base = self
+        command = request.get('command', 'GET')
+        if command not in SUPPORTED_COMMANDS:
+            raise HTTPError(501)
+        trail = request.get('trail', None)
+        if trail is not None:
+            for comp in trail:
+                subs = getattr(base, 'subs', {})
+                if comp in subs:
+                    base = subs[comp]
+                    trail.pop(0)
+                else:
+                    break
+
+        handler = getattr(base, command)
+        if handler is None:
+            raise HTTPError(400)
+
+        return handler
+
+    def handle(self, request):
+        handler = self._find_handler(request)
+        response = {'headers': dict()}
+
+        # Handle request
+        output = handler(request, response)
+        if output is None:
+            output = response.get('output')
+
+        ct = response['headers'].get('Content-Type')
+        if ct is None:
+            ct = response['headers']['Content-Type'] = DEFAULT_CTYPE
+
+        if 'application/json' in ct and isinstance(output, (dict, list)):
+            output = json_encode(output).encode('utf-8')
+            response['headers']['Content-Length'] = str(len(output))
+
+        response['output'] = output
+
+        if output is not None and not hasattr(output, 'read') \
+                and not isinstance(output, six.binary_type):
+            msg = "Handler {} returned unsupported type {} ({}):\n{!r}"
+            raise TypeError(msg.format(handler, type(output), ct, output))
+
+        if output is not None and 'Content-Length' not in response['headers']:
+            if hasattr(output, 'read'):
+                # LOG: warning file-type objects should set Content-Length
+                pass
+            else:
+                response['headers']['Content-Length'] = str(len(output))
+
+        return response

ipaserver/custodia/root.py

@@ -0,0 +1,20 @@
+# Copyright (C) 2015  Custodia Project Contributors - see LICENSE file
+from __future__ import absolute_import
+
+import json
+
+from custodia.plugin import HTTPConsumer, PluginOption
+from custodia.secrets import Secrets
+
+
+class Root(HTTPConsumer):
+    store = PluginOption('store', None, None)
+
+    def __init__(self, config, section):
+        super(Root, self).__init__(config, section)
+        if self.store_name is not None:
+            self.add_sub('secrets', Secrets(config, section))
+
+    def GET(self, request, response):
+        msg = json.dumps({'message': "Quis custodiet ipsos custodes?"})
+        return msg.encode('utf-8')

ipaserver/custodia/secrets.py

@@ -0,0 +1,400 @@
+# Copyright (C) 2015  Custodia Project Contributors - see LICENSE file
+from __future__ import absolute_import
+
+import json
+import os
+from base64 import b64decode, b64encode
+
+from custodia import log
+from custodia.message.common import UnallowedMessage
+from custodia.message.common import UnknownMessageType
+from custodia.message.formats import Validator
+from custodia.plugin import (
+    CSStoreDenied, CSStoreError, CSStoreExists, CSStoreUnsupported
+)
+from custodia.plugin import HTTPConsumer, HTTPError, PluginOption
+
+
+class Secrets(HTTPConsumer):
+    allowed_keytypes = PluginOption('str_set', 'simple', None)
+    store = PluginOption('store', None, None)
+
+    def __init__(self, config, section):
+        super(Secrets, self).__init__(config, section)
+        self._validator = Validator(self.allowed_keytypes)
+
+    def _db_key(self, trail):
+        if len(trail) < 2:
+            self.logger.debug(
+                "Forbidden action: Operation only permitted within a "
+                "container")
+            raise HTTPError(403)
+        return os.path.join('keys', *trail)
+
+    def _db_container_key(self, default, trail):
+        f = None
+        if len(trail) > 1:
+            f = self._db_key(trail)
+        elif len(trail) == 1 and trail[0] != '':
+            self.logger.debug(
+                "Forbidden action: Wrong container path. Container names must "
+                "end with '/'")
+            raise HTTPError(403)
+        elif default is None:
+            self.logger.debug("Forbidden action: No default namespace")
+            raise HTTPError(403)
+        else:
+            # Use the default namespace
+            f = self._db_key([default, ''])
+        return f
+
+    def _parse(self, request, query, name):
+        return self._validator.parse(request, query, name)
+
+    def _parse_query(self, request, name):
+        # default to simple
+        query = request.get('query', '')
+        if len(query) == 0:
+            query = {'type': 'simple', 'value': ''}
+        return self._parse(request, query, name)
+
+    def _parse_bin_body(self, request, name):
+        body = request.get('body')
+        if body is None:
+            raise HTTPError(400)
+        value = b64encode(bytes(body)).decode('utf-8')
+        payload = {'type': 'simple', 'value': value}
+        return self._parse(request, payload, name)
+
+    def _parse_body(self, request, name):
+        body = request.get('body')
+        if body is None:
+            raise HTTPError(400)
+        value = json.loads(bytes(body).decode('utf-8'))
+        return self._parse(request, value, name)
+
+    def _parse_maybe_body(self, request, name):
+        body = request.get('body')
+        if body is None:
+            value = {'type': 'simple', 'value': ''}
+        else:
+            value = json.loads(bytes(body).decode('utf-8'))
+        return self._parse(request, value, name)
+
+    def _parent_exists(self, default, trail):
+        # check that the containers exist
+        basename = self._db_container_key(trail[0], trail[:-1] + [''])
+        try:
+            keylist = self.root.store.list(basename)
+        except CSStoreError:
+            raise HTTPError(500)
+
+        self.logger.debug('parent_exists: %s (%s, %r) -> %r',
+                          basename, default, trail, keylist)
+
+        if keylist is not None:
+            return True
+
+        # create default namespace if it is the only missing piece
+        if len(trail) == 2 and default == trail[0]:
+            container = self._db_container_key(default, '')
+            self.root.store.span(container)
+            return True
+
+        return False
+
+    def _format_reply(self, request, response, handler, output):
+        reply = handler.reply(output)
+        # special case to allow *very* simple clients
+        if handler.msg_type == 'simple':
+            binary = False
+            accept = request.get('headers', {}).get('Accept', None)
+            if accept is not None:
+                types = accept.split(',')
+                for t in types:
+                    if t.strip() == 'application/json':
+                        binary = False
+                        break
+                    elif t.strip() == 'application/octet-stream':
+                        binary = True
+            if binary is True:
+                response['headers'][
+                    'Content-Type'] = 'application/octet-stream'
+                response['output'] = b64decode(reply['value'])
+                return
+
+        if reply is not None:
+            response['headers'][
+                'Content-Type'] = 'application/json; charset=utf-8'
+            response['output'] = reply
+
+    def GET(self, request, response):
+        trail = request.get('trail', [])
+        if len(trail) == 0 or trail[-1] == '':
+            self._list(trail, request, response)
+        else:
+            self._get_key(trail, request, response)
+
+    def PUT(self, request, response):
+        trail = request.get('trail', [])
+        if len(trail) == 0 or trail[-1] == '':
+            raise HTTPError(405)
+        else:
+            self._set_key(trail, request, response)
+
+    def DELETE(self, request, response):
+        trail = request.get('trail', [])
+        if len(trail) == 0:
+            raise HTTPError(405)
+        if trail[-1] == '':
+            self._destroy(trail, request, response)
+        else:
+            self._del_key(trail, request, response)
+
+    def POST(self, request, response):
+        trail = request.get('trail', [])
+        if len(trail) > 0 and trail[-1] == '':
+            self._create(trail, request, response)
+        else:
+            raise HTTPError(405)
+
+    def _list(self, trail, request, response):
+        try:
+            name = '/'.join(trail)
+            msg = self._parse_query(request, name)
+        except Exception as e:
+            raise HTTPError(406, str(e))
+        default = request.get('default_namespace', None)
+        basename = self._db_container_key(default, trail)
+        try:
+            keylist = self.root.store.list(basename)
+            self.logger.debug('list %s returned %r', basename, keylist)
+            if keylist is None:
+                raise HTTPError(404)
+            response['headers'][
+                'Content-Type'] = 'application/json; charset=utf-8'
+            response['output'] = msg.reply(keylist)
+        except CSStoreDenied:
+            self.logger.exception(
+                "List: Permission to perform this operation was denied")
+            raise HTTPError(403)
+        except CSStoreError:
+            self.logger.exception('List: Internal server error')
+            raise HTTPError(500)
+        except CSStoreUnsupported:
+            self.logger.exception('List: Unsupported operation')
+            raise HTTPError(501)
+
+    def _create(self, trail, request, response):
+        try:
+            name = '/'.join(trail)
+            msg = self._parse_maybe_body(request, name)
+        except Exception as e:
+            raise HTTPError(406, str(e))
+        default = request.get('default_namespace', None)
+        basename = self._db_container_key(None, trail)
+        try:
+            if len(trail) > 2:
+                ok = self._parent_exists(default, trail[:-1])
+                if not ok:
+                    raise HTTPError(404)
+
+            self.root.store.span(basename)
+        except CSStoreDenied:
+            self.logger.exception(
+                "Create: Permission to perform this operation was denied")
+            raise HTTPError(403)
+        except CSStoreExists:
+            self.logger.debug('Create: Key already exists')
+            response['code'] = 200
+            return
+        except CSStoreError:
+            self.logger.exception('Create: Internal server error')
+            raise HTTPError(500)
+        except CSStoreUnsupported:
+            self.logger.exception('Create: Unsupported operation')
+            raise HTTPError(501)
+
+        output = msg.reply(None)
+        if output is not None:
+            response['headers'][
+                'Content-Type'] = 'application/json; charset=utf-8'
+            response['output'] = output
+        response['code'] = 201
+
+    def _destroy(self, trail, request, response):
+        try:
+            name = '/'.join(trail)
+            msg = self._parse_maybe_body(request, name)
+        except Exception as e:
+            raise HTTPError(406, str(e))
+        basename = self._db_container_key(None, trail)
+        try:
+            keylist = self.root.store.list(basename)
+            if keylist is None:
+                raise HTTPError(404)
+            if len(keylist) != 0:
+                raise HTTPError(409)
+            ret = self.root.store.cut(basename.rstrip('/'))
+        except CSStoreDenied:
+            self.logger.exception(
+                "Delete: Permission to perform this operation was denied")
+            raise HTTPError(403)
+        except CSStoreError:
+            self.logger.exception('Delete: Internal server error')
+            raise HTTPError(500)
+        except CSStoreUnsupported:
+            self.logger.exception('Delete: Unsupported operation')
+            raise HTTPError(501)
+
+        if ret is False:
+            raise HTTPError(404)
+
+        output = msg.reply(None)
+        if output is None:
+            response['code'] = 204
+        else:
+            response['headers'][
+                'Content-Type'] = 'application/json; charset=utf-8'
+            response['output'] = output
+            response['code'] = 200
+
+    def _client_name(self, request):
+        if 'remote_user' in request:
+            return request['remote_user']
+        elif 'creds' in request:
+            creds = request['creds']
+            return '<pid={pid:d} uid={uid:d} gid={gid:d}>'.format(**creds)
+        else:
+            return 'Unknown'
+
+    def _audit(self, ok, fail, fn, trail, request, response):
+        action = fail
+        client = self._client_name(request)
+        key = '/'.join(trail)
+        try:
+            fn(trail, request, response)
+            action = ok
+        finally:
+            self.audit_key_access(action, client, key)
+
+    def _get_key(self, trail, request, response):
+        self._audit(log.AUDIT_GET_ALLOWED, log.AUDIT_GET_DENIED,
+                    self._int_get_key, trail, request, response)
+
+    def _int_get_key(self, trail, request, response):
+        try:
+            name = '/'.join(trail)
+            handler = self._parse_query(request, name)
+        except Exception as e:
+            raise HTTPError(406, str(e))
+        key = self._db_key(trail)
+        try:
+            output = self.root.store.get(key)
+            if output is None:
+                raise HTTPError(404)
+            elif len(output) == 0:
+                raise HTTPError(406)
+            self._format_reply(request, response, handler, output)
+        except CSStoreDenied:
+            self.logger.exception(
+                "Get: Permission to perform this operation was denied")
+            raise HTTPError(403)
+        except CSStoreError:
+            self.logger.exception('Get: Internal server error')
+            raise HTTPError(500)
+        except CSStoreUnsupported:
+            self.logger.exception('Get: Unsupported operation')
+            raise HTTPError(501)
+
+    def _set_key(self, trail, request, response):
+        self._audit(log.AUDIT_SET_ALLOWED, log.AUDIT_SET_DENIED,
+                    self._int_set_key, trail, request, response)
+
+    def _int_set_key(self, trail, request, response):
+        try:
+            name = '/'.join(trail)
+
+            content_type = request.get('headers', {}).get('Content-Type', '')
+            content_type_value = content_type.split(';')[0].strip()
+            if content_type_value == 'application/octet-stream':
+                msg = self._parse_bin_body(request, name)
+            elif content_type_value == 'application/json':
+                msg = self._parse_body(request, name)
+            else:
+                raise ValueError('Invalid Content-Type')
+        except UnknownMessageType as e:
+            raise HTTPError(406, str(e))
+        except UnallowedMessage as e:
+            raise HTTPError(406, str(e))
+        except Exception as e:
+            raise HTTPError(400, str(e))
+
+        # must _db_key first as access control is done here for now
+        # otherwise users would e able to probe containers in namespaces
+        # they do not have access to.
+        key = self._db_key(trail)
+
+        try:
+            default = request.get('default_namespace', None)
+            ok = self._parent_exists(default, trail)
+            if not ok:
+                raise HTTPError(404)
+
+            ok = self.root.store.set(key, msg.payload)
+        except CSStoreDenied:
+            self.logger.exception(
+                "Set: Permission to perform this operation was denied")
+            raise HTTPError(403)
+        except CSStoreExists:
+            self.logger.exception('Set: Key already exist')
+            raise HTTPError(409)
+        except CSStoreError:
+            self.logger.exception('Set: Internal Server Error')
+            raise HTTPError(500)
+        except CSStoreUnsupported:
+            self.logger.exception('Set: Unsupported operation')
+            raise HTTPError(501)
+
+        output = msg.reply(None)
+        if output is not None:
+            response['headers'][
+                'Content-Type'] = 'application/json; charset=utf-8'
+            response['output'] = output
+        response['code'] = 201
+
+    def _del_key(self, trail, request, response):
+        self._audit(log.AUDIT_DEL_ALLOWED, log.AUDIT_DEL_DENIED,
+                    self._int_del_key, trail, request, response)
+
+    def _int_del_key(self, trail, request, response):
+        try:
+            name = '/'.join(trail)
+            msg = self._parse_maybe_body(request, name)
+        except Exception as e:
+            raise HTTPError(406, str(e))
+        key = self._db_key(trail)
+        try:
+            ret = self.root.store.cut(key)
+        except CSStoreDenied:
+            self.logger.exception(
+                "Delete: Permission to perform this operation was denied")
+            raise HTTPError(403)
+        except CSStoreError:
+            self.logger.exception('Delete: Internal Server Error')
+            raise HTTPError(500)
+        except CSStoreUnsupported:
+            self.logger.exception('Delete: Unsupported operation')
+            raise HTTPError(501)
+
+        if ret is False:
+            raise HTTPError(404)
+
+        output = msg.reply(None)
+        if output is None:
+            response['code'] = 204
+        else:
+            response['headers'][
+                'Content-Type'] = 'application/json; charset=utf-8'
+            response['output'] = output
+            response['code'] = 200

ipaserver/custodia/server/__init__.py

@@ -0,0 +1,142 @@
+# Copyright (C) 2015  Custodia Project Contributors - see LICENSE file
+from __future__ import absolute_import
+
+import importlib
+import os
+
+import pkg_resources
+
+import six
+
+from custodia import log
+from custodia.httpd.server import HTTPServer
+
+from .args import default_argparser
+from .args import parse_args as _parse_args
+from .config import parse_config as _parse_config
+
+logger = log.getLogger('custodia')
+
+__all__ = ['default_argparser', 'main']
+
+
+def attach_store(typename, plugins, stores):
+    for name, c in six.iteritems(plugins):
+        if getattr(c, 'store_name', None) is None:
+            continue
+        try:
+            c.store = stores[c.store_name]
+        except KeyError:
+            raise ValueError('[%s%s] references unexisting store '
+                             '"%s"' % (typename, name, c.store_name))
+
+
+def _load_plugin_class(menu, name):
+    """Load Custodia plugin
+
+    Entry points are preferred over dotted import path.
+    """
+    group = 'custodia.{}'.format(menu)
+    eps = list(pkg_resources.iter_entry_points(group, name))
+    if len(eps) > 1:
+        raise ValueError(
+            "Multiple entry points for {} {}: {}".format(menu, name, eps))
+    elif len(eps) == 1:
+        # backwards compatibility with old setuptools
+        ep = eps[0]
+        if hasattr(ep, 'resolve'):
+            return ep.resolve()
+        else:
+            return ep.load(require=False)
+    elif '.' in name:
+        # fall back to old style dotted name
+        module, classname = name.rsplit('.', 1)
+        m = importlib.import_module(module)
+        return getattr(m, classname)
+    else:
+        raise ValueError("{}: {} not found".format(menu, name))
+
+
+def _create_plugin(cfgparser, section, menu):
+    if not cfgparser.has_option(section, 'handler'):
+        raise ValueError('Invalid section, missing "handler"')
+
+    handler_name = cfgparser.get(section, 'handler')
+    hconf = {'facility_name': section}
+    try:
+        handler = _load_plugin_class(menu, handler_name)
+        classname = handler.__name__
+        hconf['facility_name'] = '%s-[%s]' % (classname, section)
+    except Exception as e:  # pylint: disable=broad-except
+        raise ValueError('Invalid format for "handler" option '
+                         '[%r]: %s' % (e, handler_name))
+
+    if handler._options is not None:  # pylint: disable=protected-access
+        # new-style plugin with parser and section
+        plugin = handler(cfgparser, section)
+    else:
+        # old-style plugin with config dict
+        hconf.update(cfgparser.items(section))
+        hconf.pop('handler')
+        plugin = handler(hconf)
+        plugin.section = section
+    return plugin
+
+
+def _load_plugins(config, cfgparser):
+    """Load and initialize plugins
+    """
+    # set umask before any plugin gets a chance to create a file
+    os.umask(config['umask'])
+
+    for s in cfgparser.sections():
+        if s in {'ENV', 'global'}:
+            # ENV section is only used for interpolation
+            continue
+
+        if s.startswith('/'):
+            menu = 'consumers'
+            path_chain = s.split('/')
+            if path_chain[-1] == '':
+                path_chain = path_chain[:-1]
+            name = tuple(path_chain)
+        else:
+            if s.startswith('auth:'):
+                menu = 'authenticators'
+                name = s[5:]
+            elif s.startswith('authz:'):
+                menu = 'authorizers'
+                name = s[6:]
+            elif s.startswith('store:'):
+                menu = 'stores'
+                name = s[6:]
+            else:
+                raise ValueError('Invalid section name [%s].\n' % s)
+
+        try:
+            config[menu][name] = _create_plugin(cfgparser, s, menu)
+        except Exception as e:
+            logger.debug("Plugin '%s' failed to load.", name, exc_info=True)
+            raise RuntimeError(menu, name, e)
+
+    # 2nd initialization stage
+    for menu in ['authenticators', 'authorizers', 'consumers', 'stores']:
+        plugins = config[menu]
+        for name in sorted(plugins):
+            plugin = plugins[name]
+            plugin.finalize_init(config, cfgparser, context=None)
+
+
+def main(argparser=None):
+    args = _parse_args(argparser=argparser)
+    # parse arguments and populate config with basic settings
+    cfgparser, config = _parse_config(args)
+    # initialize logging
+    log.setup_logging(config['debug'], config['auditlog'])
+    logger.info('Custodia instance %s', args.instance or '<main>')
+    logger.debug('Config file(s) %s loaded', config['configfiles'])
+    # load plugins after logging
+    _load_plugins(config, cfgparser)
+    # create and run server
+    httpd = HTTPServer(config['server_url'], config)
+    httpd.serve()

ipaserver/custodia/server/__main__.py

@@ -0,0 +1,7 @@
+# Copyright (C) 2015  Custodia Project Contributors - see LICENSE file
+from __future__ import absolute_import
+
+from custodia.server import main
+
+if __name__ == '__main__':
+    main()

ipaserver/custodia/server/args.py

@@ -0,0 +1,80 @@
+# Copyright (C) 2015-2017  Custodia Project Contributors - see LICENSE file
+from __future__ import absolute_import
+
+import argparse
+import os
+
+
+class AbsFileType(argparse.FileType):
+    """argparse file type with absolute path
+    """
+    def __call__(self, string):
+        if string != '-':
+            string = os.path.abspath(string)
+        return super(AbsFileType, self).__call__(string)
+
+
+class ConfigfileAction(argparse.Action):
+    """Default action handler for configfile
+    """
+    default_path = '/etc/custodia/custodia.conf'
+    default_instance = '/etc/custodia/{instance}.conf'
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        if values is None:
+            if namespace.instance is not None:
+                values = self.default_instance.format(
+                    instance=namespace.instance
+                )
+            else:
+                values = self.default_path
+            values = self.type(values)
+        setattr(namespace, self.dest, values)
+
+
+def instance_name(string):
+    """Check for valid instance name
+    """
+    invalid = ':/@'
+    if set(string).intersection(invalid):
+        msg = 'Invalid instance name {}'.format(string)
+        raise argparse.ArgumentTypeError(msg)
+    return string
+
+
+default_argparser = argparse.ArgumentParser(
+    prog='custodia',
+    description='Custodia server'
+)
+default_argparser.add_argument(
+    '--debug',
+    action='store_true',
+    help='Debug mode'
+)
+default_argparser.add_argument(
+    '--instance',
+    type=instance_name,
+    help='Instance name',
+    default=None
+)
+default_argparser.add_argument(
+    'configfile',
+    nargs='?',
+    action=ConfigfileAction,
+    type=AbsFileType('r'),
+    help=('Path to custodia server config (default: '
+          '/etc/custodia/{instance}/custodia.conf)'),
+)
+
+
+def parse_args(args=None, argparser=None):
+    if argparser is None:
+        argparser = default_argparser
+
+    # namespace with default values
+    namespace = argparse.Namespace(
+        debug=False,
+        instance=None,
+    )
+
+    return argparser.parse_args(args, namespace)

ipaserver/custodia/server/config.py

@@ -0,0 +1,166 @@
+# Copyright (C) 2015-2017  Custodia Project Contributors - see LICENSE file
+from __future__ import absolute_import
+
+import glob
+import os
+import socket
+
+import six
+
+from custodia.compat import configparser
+from custodia.compat import url_escape
+
+
+class CustodiaConfig(object):
+    CONFIG_SPECIALS = ['authenticators', 'authorizers', 'consumers', 'stores']
+
+    DEFAULT_PATHS = [
+        ('libdir', '/var/lib/custodia/{instance}'),
+        ('logdir', '/var/log/custodia/{instance}'),
+        ('rundir', '/var/run/custodia/{instance}'),
+        ('socketdir', '/var/run/custodia'),
+    ]
+
+    def __init__(self, args):
+        self.args = args
+        self.config = {}
+        self.defaults = None
+        self.parser = None
+
+    def get_defaults(self):
+        configpath = self.args.configfile.name
+        instance = self.args.instance
+        defaults = {
+            # Do not use getfqdn(). Internaly it calls gethostbyaddr which
+            # might perform a DNS query.
+            'hostname': socket.gethostname(),
+            'configdir': os.path.dirname(configpath),
+            'confdpattern': os.path.join(configpath + '.d', '*.conf'),
+            'instance': instance if instance else '',
+        }
+        for name, path in self.DEFAULT_PATHS:
+            defaults[name] = os.path.abspath(path.format(**defaults))
+        return defaults
+
+    def create_parser(self):
+        parser = configparser.ConfigParser(
+            interpolation=configparser.ExtendedInterpolation(),
+            defaults=self.defaults
+        )
+        parser.optionxform = str
+
+        # add env
+        parser.add_section(u'ENV')
+        for k, v in os.environ.items():
+            if set(v).intersection('\r\n\x00'):
+                continue
+            if six.PY2:
+                k = k.decode('utf-8', 'replace')
+                v = v.decode('utf-8', 'replace')
+            parser.set(u'ENV', k, v.replace(u'$', u'$$'))
+
+        # default globals
+        parser.add_section(u'global')
+        parser.set(u'global', u'auditlog', u'${logdir}/audit.log')
+        parser.set(u'global', u'debug', u'false')
+        parser.set(u'global', u'umask', u'027')
+        parser.set(u'global', u'makedirs', u'false')
+
+        return parser
+
+    def read_configs(self):
+        with self.args.configfile as f:
+            self.parser.read_file(f)
+
+        configfiles = [self.args.configfile.name]
+
+        pattern = self.parser.get(u'DEFAULT', u'confdpattern')
+        if pattern:
+            confdfiles = glob.glob(pattern)
+            confdfiles.sort()
+            for confdfile in confdfiles:
+                with open(confdfile) as f:
+                    self.parser.read_file(f)
+                configfiles.append(confdfile)
+
+        return configfiles
+
+    def makedirs(self):
+        for name, _ in self.DEFAULT_PATHS:
+            path = self.parser.get(u'DEFAULT', name)
+            parent = os.path.dirname(path)
+            # create parents according to umask
+            if not os.path.isdir(parent):
+                os.makedirs(parent)
+            # create final directory with restricted permissions
+            if not os.path.isdir(path):
+                os.mkdir(path, 0o700)
+
+    def populate_config(self):
+        config = self.config
+
+        for s in self.CONFIG_SPECIALS:
+            config[s] = {}
+
+        for opt, val in self.parser.items(u'global'):
+            if opt in self.CONFIG_SPECIALS:
+                raise ValueError('"%s" is an invalid '
+                                 '[global] option' % opt)
+            config[opt] = val
+
+        config['tls_verify_client'] = self.parser.getboolean(
+            'global', 'tls_verify_client', fallback=False)
+        config['debug'] = self.parser.getboolean(
+            'global', 'debug', fallback=False)
+        config['makedirs'] = self.parser.getboolean(
+            'global', 'makedirs', fallback=False)
+        if self.args.debug:
+            config['debug'] = self.args.debug
+
+        config['auditlog'] = os.path.abspath(config.get('auditlog'))
+        config['umask'] = int(config.get('umask', '027'), 8)
+
+        url = config.get('server_url')
+        sock = config.get('server_socket')
+
+        if url and sock:
+            raise ValueError(
+                "'server_url' and 'server_socket' are mutually exclusive.")
+
+        if not url and not sock:
+            # no option but, use default socket path
+            socketdir = self.parser.get(u'DEFAULT', u'socketdir')
+            name = self.args.instance if self.args.instance else 'custodia'
+            sock = os.path.join(socketdir, name + '.sock')
+
+        if sock:
+            server_socket = os.path.abspath(sock)
+            config['server_url'] = 'http+unix://{}/'.format(
+                url_escape(server_socket, ''))
+
+    def __call__(self):
+        self.defaults = self.get_defaults()
+        self.parser = self.create_parser()
+        self.config['configfiles'] = self.read_configs()
+        self.populate_config()
+        if self.config[u'makedirs']:
+            self.makedirs()
+        return self.parser, self.config
+
+
+def parse_config(args):
+    ccfg = CustodiaConfig(args)
+    return ccfg()
+
+
+def test(arglist):
+    from pprint import pprint
+    from .args import parse_args
+    args = parse_args(arglist)
+    parser, config = parse_config(args)
+    pprint(parser.items("DEFAULT"))
+    pprint(config)
+
+
+if __name__ == '__main__':
+    test(['--instance=demo', './tests/empty.conf'])

ipaserver/custodia/store/encgen.py

@@ -0,0 +1,107 @@
+# Copyright (C) 2015  Custodia Project Contributors - see LICENSE file
+from __future__ import absolute_import
+
+import os
+
+from jwcrypto.common import json_decode, json_encode
+from jwcrypto.jwe import JWE
+from jwcrypto.jwk import JWK
+
+from custodia.plugin import CSStore, CSStoreError
+from custodia.plugin import PluginOption, REQUIRED
+
+
+class EncryptedOverlay(CSStore):
+    """Encrypted overlay for storage backends
+
+    Arguments:
+        backing_store (required):
+            name of backing storage
+        master_key (required)
+            path to master key (JWK JSON)
+        autogen_master_key (default: false)
+            auto-generate key file if missing?
+        master_enctype (default: A256CBC_HS512)
+            JWE algorithm name
+        secret_protection (default: 'encrypt'):
+            Determine the kind of protection used to save keys:
+            - 'encrypt': this is the classic method (backwards compatible)
+            - 'pinning': this adds a protected header with the key name as
+            add data, to prevent key swapping in the db
+            - 'migrate': as pinning, but on missing key information the
+            secret is updated instead of throwing an exception.
+    """
+    key_sizes = {
+        'A128CBC-HS256': 256,
+        'A256CBC-HS512': 512,
+    }
+
+    backing_store = PluginOption(str, REQUIRED, None)
+    master_enctype = PluginOption(str, 'A256CBC-HS512', None)
+    master_key = PluginOption(str, REQUIRED, None)
+    autogen_master_key = PluginOption(bool, False, None)
+    secret_protection = PluginOption(str, False, 'encrypt')
+
+    def __init__(self, config, section):
+        super(EncryptedOverlay, self).__init__(config, section)
+        self.store_name = self.backing_store
+        self.store = None
+        self.protected_header = None
+
+        if (not os.path.isfile(self.master_key) and
+                self.autogen_master_key):
+            # XXX https://github.com/latchset/jwcrypto/issues/50
+            size = self.key_sizes.get(self.master_enctype, 512)
+            key = JWK(generate='oct', size=size)
+            with open(self.master_key, 'w') as f:
+                os.fchmod(f.fileno(), 0o600)
+                f.write(key.export())
+
+        with open(self.master_key) as f:
+            data = f.read()
+            key = json_decode(data)
+            self.mkey = JWK(**key)
+
+    def get(self, key):
+        value = self.store.get(key)
+        if value is None:
+            return None
+        try:
+            jwe = JWE()
+            jwe.deserialize(value, self.mkey)
+            value = jwe.payload.decode('utf-8')
+        except Exception as err:
+            self.logger.error("Error parsing key %s: [%r]" % (key, repr(err)))
+            raise CSStoreError('Error occurred while trying to parse key')
+        if self.secret_protection == 'encrypt':
+            return value
+        if 'custodia.key' not in jwe.jose_header:
+            if self.secret_protection == 'migrate':
+                self.set(key, value, replace=True)
+            else:
+                raise CSStoreError('Secret Pinning check failed!' +
+                                   'Missing custodia.key element')
+        elif jwe.jose_header['custodia.key'] != key:
+            raise CSStoreError(
+                'Secret Pinning check failed! Expected {} got {}'.format(
+                    key, jwe.jose_header['custodia.key']))
+        return value
+
+    def set(self, key, value, replace=False):
+        self.protected_header = {'alg': 'dir', 'enc': self.master_enctype}
+        if self.secret_protection != 'encrypt':
+            self.protected_header['custodia.key'] = key
+        protected = json_encode(self.protected_header)
+        jwe = JWE(value, protected)
+        jwe.add_recipient(self.mkey)
+        cvalue = jwe.serialize(compact=True)
+        return self.store.set(key, cvalue, replace)
+
+    def span(self, key):
+        return self.store.span(key)
+
+    def list(self, keyfilter=''):
+        return self.store.list(keyfilter)
+
+    def cut(self, key):
+        return self.store.cut(key)

ipaserver/custodia/store/enclite.py

@@ -0,0 +1,40 @@
+# Copyright (C) 2015  Custodia Project Contributors - see LICENSE file
+from __future__ import absolute_import
+
+from jwcrypto.common import json_decode, json_encode
+from jwcrypto.jwe import JWE
+from jwcrypto.jwk import JWK
+
+from custodia.plugin import CSStoreError, PluginOption, REQUIRED
+from custodia.store.sqlite import SqliteStore
+
+
+class EncryptedStore(SqliteStore):
+    master_key = PluginOption(str, REQUIRED, None)
+    master_enctype = PluginOption(str, 'A256CBC-HS512', None)
+
+    def __init__(self, config, section):
+        super(EncryptedStore, self).__init__(config, section)
+        with open(self.master_key) as f:
+            data = f.read()
+            key = json_decode(data)
+            self.mkey = JWK(**key)
+
+    def get(self, key):
+        value = super(EncryptedStore, self).get(key)
+        if value is None:
+            return None
+        try:
+            jwe = JWE()
+            jwe.deserialize(value, self.mkey)
+            return jwe.payload.decode('utf-8')
+        except Exception:
+            self.logger.exception("Error parsing key %s", key)
+            raise CSStoreError('Error occurred while trying to parse key')
+
+    def set(self, key, value, replace=False):
+        protected = json_encode({'alg': 'dir', 'enc': self.master_enctype})
+        jwe = JWE(value, protected)
+        jwe.add_recipient(self.mkey)
+        cvalue = jwe.serialize(compact=True)
+        return super(EncryptedStore, self).set(key, cvalue, replace)

ipaserver/custodia/store/interface.py

@@ -0,0 +1,12 @@
+# Copyright (C) 2015  Custodia Project Contributors - see LICENSE file
+from __future__ import absolute_import
+
+import warnings
+
+from custodia.plugin import CSStore, CSStoreError, CSStoreExists
+
+__all__ = ('CSStore', 'CSStoreError', 'CSStoreExists')
+
+
+warnings.warn('custodia.store.interface is deprecated, import from '
+              'custodia.plugin instead.', DeprecationWarning,)

ipaserver/custodia/store/sqlite.py

@@ -0,0 +1,145 @@
+# Copyright (C) 2015  Custodia Project Contributors - see LICENSE file
+from __future__ import absolute_import, print_function
+
+import os
+import sqlite3
+
+from custodia.plugin import CSStore, CSStoreError, CSStoreExists
+from custodia.plugin import PluginOption, REQUIRED
+
+
+class SqliteStore(CSStore):
+    dburi = PluginOption(str, REQUIRED, None)
+    table = PluginOption(str, "CustodiaSecrets", None)
+    filemode = PluginOption(oct, '600', None)
+
+    def __init__(self, config, section):
+        super(SqliteStore, self).__init__(config, section)
+        # Initialize the DB by trying to create the default table
+        try:
+            conn = sqlite3.connect(self.dburi)
+            os.chmod(self.dburi, self.filemode)
+            with conn:
+                c = conn.cursor()
+                self._create(c)
+        except sqlite3.Error:
+            self.logger.exception("Error creating table %s", self.table)
+            raise CSStoreError('Error occurred while trying to init db')
+
+    def get(self, key):
+        self.logger.debug("Fetching key %s", key)
+        query = "SELECT value from %s WHERE key=?" % self.table
+        try:
+            conn = sqlite3.connect(self.dburi)
+            c = conn.cursor()
+            r = c.execute(query, (key,))
+            value = r.fetchall()
+        except sqlite3.Error:
+            self.logger.exception("Error fetching key %s", key)
+            raise CSStoreError('Error occurred while trying to get key')
+        self.logger.debug("Fetched key %s got result: %r", key, value)
+        if len(value) > 0:
+            return value[0][0]
+        else:
+            return None
+
+    def _create(self, cur):
+        create = "CREATE TABLE IF NOT EXISTS %s " \
+                 "(key PRIMARY KEY UNIQUE, value)" % self.table
+        cur.execute(create)
+
+    def set(self, key, value, replace=False):
+        self.logger.debug("Setting key %s to value %s (replace=%s)",
+                          key, value, replace)
+        if key.endswith('/'):
+            raise ValueError('Invalid Key name, cannot end in "/"')
+        if replace:
+            query = "INSERT OR REPLACE into %s VALUES (?, ?)"
+        else:
+            query = "INSERT into %s VALUES (?, ?)"
+        setdata = query % (self.table,)
+        try:
+            conn = sqlite3.connect(self.dburi)
+            with conn:
+                c = conn.cursor()
+                self._create(c)
+                c.execute(setdata, (key, value))
+        except sqlite3.IntegrityError as err:
+            raise CSStoreExists(str(err))
+        except sqlite3.Error as err:
+            self.logger.exception("Error storing key %s", key)
+            raise CSStoreError('Error occurred while trying to store key')
+
+    def span(self, key):
+        name = key.rstrip('/')
+        self.logger.debug("Creating container %s", name)
+        query = "INSERT into %s VALUES (?, '')"
+        setdata = query % (self.table,)
+        try:
+            conn = sqlite3.connect(self.dburi)
+            with conn:
+                c = conn.cursor()
+                self._create(c)
+                c.execute(setdata, (name,))
+        except sqlite3.IntegrityError as err:
+            raise CSStoreExists(str(err))
+        except sqlite3.Error:
+            self.logger.exception("Error creating key %s", name)
+            raise CSStoreError('Error occurred while trying to span container')
+
+    def list(self, keyfilter=''):
+        path = keyfilter.rstrip('/')
+        self.logger.debug("Listing keys matching %s", path)
+        child_prefix = path if path == '' else path + '/'
+        search = "SELECT key, value FROM %s WHERE key LIKE ?" % self.table
+        key = "%s%%" % (path,)
+        try:
+            conn = sqlite3.connect(self.dburi)
+            r = conn.execute(search, (key,))
+            rows = r.fetchall()
+        except sqlite3.Error:
+            self.logger.exception("Error listing %s: [%r]", keyfilter)
+            raise CSStoreError('Error occurred while trying to list keys')
+        self.logger.debug("Searched for %s got result: %r", path, rows)
+        if len(rows) > 0:
+            parent_exists = False
+            result = list()
+            for key, value in rows:
+                if key == path or key == child_prefix:
+                    parent_exists = True
+                    continue
+                if not key.startswith(child_prefix):
+                    continue
+                result_value = key[len(child_prefix):].lstrip('/')
+                if not value:
+                    result.append(result_value + '/')
+                else:
+                    result.append(result_value)
+            if result:
+                self.logger.debug("Returning sorted values %r", result)
+                return sorted(result)
+            elif parent_exists:
+                self.logger.debug("Returning empty list")
+                return []
+        elif keyfilter == '':
+            self.logger.debug("Returning empty list")
+            return []
+        self.logger.debug("Returning 'Not Found'")
+        return None
+
+    def cut(self, key):
+        self.logger.debug("Removing key %s", key)
+        query = "DELETE from %s WHERE key=?" % self.table
+        try:
+            conn = sqlite3.connect(self.dburi)
+            with conn:
+                c = conn.cursor()
+                r = c.execute(query, (key,))
+        except sqlite3.Error:
+            self.logger.error("Error removing key %s", key)
+            raise CSStoreError('Error occurred while trying to cut key')
+        self.logger.debug("Key %s %s", key,
+                          "removed" if r.rowcount > 0 else "not found")
+        if r.rowcount > 0:
+            return True
+        return False