csrgen: Add code to generate scripts that generate CSRs

Adds a library that uses jinja2 to format a script that, when run, will
build a CSR. Also adds a CLI command, 'cert-get-requestdata', that uses
this library and builds the script for a given principal. The rules are
read from json files in /usr/share/ipa/csr, but the rule provider is a
separate class so that it can be replaced easily.

https://fedorahosted.org/freeipa/ticket/4899

Reviewed-By: Jan Cholasta <jcholast@redhat.com>

configure.ac

@@ -533,6 +533,7 @@ AC_CONFIG_FILES([
     install/share/Makefile
     install/share/advise/Makefile
     install/share/advise/legacy/Makefile
+    install/share/csrgen/Makefile
     install/share/profiles/Makefile
     install/share/schema.d/Makefile
     install/ui/Makefile

freeipa.spec.in

@@ -153,6 +153,7 @@ BuildRequires:  python-sssdconfig
 BuildRequires:  python-nose
 BuildRequires:  python-paste
 BuildRequires:  systemd-python
+BuildRequires:  python2-jinja2
 
 %if 0%{?with_python3}
 # FIXME: this depedency is missing - server will not work
@@ -190,6 +191,7 @@ BuildRequires:  python3-libsss_nss_idmap
 BuildRequires:  python3-nose
 BuildRequires:  python3-paste
 BuildRequires:  python3-systemd
+BuildRequires:  python3-jinja2
 %endif # with_python3
 %endif # with_lint
 
@@ -489,6 +491,7 @@ Requires: %{name}-client-common = %{version}-%{release}
 Requires: %{name}-common = %{version}-%{release}
 Requires: python2-ipalib = %{version}-%{release}
 Requires: python-dns >= 1.15
+Requires: python2-jinja2
 
 %description -n python2-ipaclient
 IPA is an integrated solution to provide centrally managed Identity (users,
@@ -511,6 +514,7 @@ Requires: %{name}-client-common = %{version}-%{release}
 Requires: %{name}-common = %{version}-%{release}
 Requires: python3-ipalib = %{version}-%{release}
 Requires: python3-dns >= 1.15
+Requires: python3-jinja2
 
 %description -n python3-ipaclient
 IPA is an integrated solution to provide centrally managed Identity (users,
@@ -1217,6 +1221,13 @@ fi
 %{_usr}/share/ipa/advise/legacy/*.template
 %dir %{_usr}/share/ipa/profiles
 %{_usr}/share/ipa/profiles/*.cfg
+%dir %{_usr}/share/ipa/csrgen
+%dir %{_usr}/share/ipa/csrgen/templates
+%{_usr}/share/ipa/csrgen/templates/*.tmpl
+%dir %{_usr}/share/ipa/csrgen/profiles
+%{_usr}/share/ipa/csrgen/profiles/*.json
+%dir %{_usr}/share/ipa/csrgen/rules
+%{_usr}/share/ipa/csrgen/rules/*.json
 %dir %{_usr}/share/ipa/html
 %{_usr}/share/ipa/html/ffconfig.js
 %{_usr}/share/ipa/html/ffconfig_page.js

install/share/Makefile.am

@@ -2,6 +2,7 @@ NULL =
 
 SUBDIRS =  				\
 	advise				\
+	csrgen				\
 	profiles			\
 	schema.d			\
 	$(NULL)

install/share/csr/templates/ipa_macros.tmpl

@@ -0,0 +1,42 @@
+{% set rendersyntax = {} %}
+
+{% set renderdata = {} %}
+
+{# Wrapper for syntax rules. We render the contents of the rule into a
+variable, so that if we find that none of the contained data rules rendered we
+can suppress the whole syntax rule. That is, a syntax rule is rendered either
+if no data rules are specified (unusual) or if at least one of the data rules
+rendered successfully. #}
+{% macro syntaxrule() -%}
+{% do rendersyntax.update(none=true, any=false) -%}
+{% set contents -%}
+{{ caller() -}}
+{% endset -%}
+{% if rendersyntax['none'] or rendersyntax['any'] -%}
+{{ contents -}}
+{% endif -%}
+{% endmacro %}
+
+{# Wrapper for data rules. A data rule is rendered only when all of the data
+fields it contains have data available. #}
+{% macro datarule() -%}
+{% do rendersyntax.update(none=false) -%}
+{% do renderdata.update(all=true) -%}
+{% set contents -%}
+{{ caller() -}}
+{% endset -%}
+{% if renderdata['all'] -%}
+{% do rendersyntax.update(any=true) -%}
+{{ contents -}}
+{% endif -%}
+{% endmacro %}
+
+{# Wrapper for fields in data rules. If any value wrapped by this macro
+produces an empty string, the entire data rule will be suppressed. #}
+{% macro datafield(value) -%}
+{% if value -%}
+{{ value -}}
+{% else -%}
+{% do renderdata.update(all=false) -%}
+{% endif -%}
+{% endmacro %}

install/share/csrgen/Makefile.am

@@ -0,0 +1,27 @@
+NULL =
+
+profiledir = $(IPA_DATA_DIR)/csrgen/profiles
+profile_DATA =				\
+	$(NULL)
+
+ruledir = $(IPA_DATA_DIR)/csrgen/rules
+rule_DATA =				\
+	$(NULL)
+
+templatedir = $(IPA_DATA_DIR)/csrgen/templates
+template_DATA =			\
+	templates/certutil_base.tmpl	\
+	templates/openssl_base.tmpl	\
+	templates/openssl_macros.tmpl	\
+	templates/ipa_macros.tmpl	\
+	$(NULL)
+
+EXTRA_DIST =				\
+	$(profile_DATA)			\
+	$(rule_DATA)			\
+	$(template_DATA)		\
+	$(NULL)
+
+MAINTAINERCLEANFILES =			\
+	*~				\
+	Makefile.in

install/share/csrgen/templates/certutil_base.tmpl

@@ -0,0 +1,14 @@
+{% raw -%}
+{% import "ipa_macros.tmpl" as ipa -%}
+{%- endraw %}
+#!/bin/bash -e
+
+if [[ $# -lt 1 ]]; then
+echo "Usage: $0 <outfile> [<any> <certutil> <args>]"
+echo "Called as: $0 $@"
+exit 1
+fi
+
+CSR="$1"
+shift
+certutil -R -a -z <(head -c 4096 /dev/urandom) -o "$CSR" {{ options|join(' ') }} "$@"

install/share/csrgen/templates/openssl_base.tmpl

@@ -0,0 +1,35 @@
+{% raw -%}
+{% import "openssl_macros.tmpl" as openssl -%}
+{% import "ipa_macros.tmpl" as ipa -%}
+{%- endraw %}
+#!/bin/bash -e
+
+if [[ $# -ne 2 ]]; then
+echo "Usage: $0 <outfile> <keyfile>"
+echo "Called as: $0 $@"
+exit 1
+fi
+
+CONFIG="$(mktemp)"
+CSR="$1"
+shift
+
+echo \
+{% raw %}{% filter quote %}{% endraw -%}
+[ req ]
+prompt = no
+encrypt_key = no
+
+{{ parameters|join('\n') }}
+{% raw %}{% set rendered_extensions -%}{% endraw %}
+{{ extensions|join('\n') }}
+{% raw -%}
+{%- endset -%}
+{% if rendered_extensions -%}
+req_extensions = {% call openssl.section() %}{{ rendered_extensions }}{% endcall %}
+{% endif %}
+{{ openssl.openssl_sections|join('\n\n') }}
+{% endfilter %}{%- endraw %} > "$CONFIG"
+
+openssl req -new -config "$CONFIG" -out "$CSR" -key $1
+rm "$CONFIG"

install/share/csrgen/templates/openssl_macros.tmpl

@@ -0,0 +1,29 @@
+{# List containing rendered sections to be included at end #}
+{% set openssl_sections = [] %}
+
+{#
+List containing one entry for each section name allocated. Because of
+scoping rules, we need to use a list so that it can be a "per-render global"
+that gets updated in place. Real globals are shared by all templates with the
+same environment, and variables defined in the macro don't persist after the
+macro invocation ends.
+#}
+{% set openssl_section_num = [] %}
+
+{% macro section() -%}
+{% set name -%}
+sec{{ openssl_section_num|length -}}
+{% endset -%}
+{% do openssl_section_num.append('') -%}
+{% set contents %}{{ caller() }}{% endset -%}
+{% if contents -%}
+{% set sectiondata = formatsection(name, contents) -%}
+{% do openssl_sections.append(sectiondata) -%}
+{% endif -%}
+{{ name -}}
+{% endmacro %}
+
+{% macro formatsection(name, contents) -%}
+[ {{ name }} ]
+{{ contents -}}
+{% endmacro %}

ipaclient/csrgen.py

@@ -0,0 +1,319 @@
+#
+# Copyright (C) 2016  FreeIPA Contributors see COPYING for license
+#
+
+import collections
+import json
+import os.path
+import pipes
+import traceback
+
+import jinja2
+import jinja2.ext
+import jinja2.sandbox
+import six
+
+from ipalib import api
+from ipalib import errors
+from ipalib.text import _
+from ipaplatform.paths import paths
+from ipapython.ipa_log_manager import log_mgr
+
+if six.PY3:
+    unicode = str
+
+__doc__ = _("""
+Routines for constructing certificate signing requests using IPA data and
+stored templates.
+""")
+
+logger = log_mgr.get_logger(__name__)
+
+
+class IndexableUndefined(jinja2.Undefined):
+    def __getitem__(self, key):
+        return jinja2.Undefined(
+            hint=self._undefined_hint, obj=self._undefined_obj,
+            name=self._undefined_name, exc=self._undefined_exception)
+
+
+class IPAExtension(jinja2.ext.Extension):
+    """Jinja2 extension providing useful features for CSR generation rules."""
+
+    def __init__(self, environment):
+        super(IPAExtension, self).__init__(environment)
+
+        environment.filters.update(
+            quote=self.quote,
+            required=self.required,
+        )
+
+    def quote(self, data):
+        return pipes.quote(data)
+
+    def required(self, data, name):
+        if not data:
+            raise errors.CSRTemplateError(
+                reason=_('Required CSR generation rule %(name)s is missing data') %
+                {'name': name})
+        return data
+
+
+class Formatter(object):
+    """
+    Class for processing a set of CSR generation rules into a template.
+
+    The template can be rendered with user and database data to produce a
+    script, which generates a CSR when run.
+
+    Subclasses of Formatter should set the value of base_template_name to the
+    filename of a base template with spaces for the processed rules.
+    Additionally, they should override the _get_template_params method to
+    produce the correct output for the base template.
+    """
+    base_template_name = None
+
+    def __init__(self, csr_data_dir=paths.CSR_DATA_DIR):
+        self.jinja2 = jinja2.sandbox.SandboxedEnvironment(
+            loader=jinja2.FileSystemLoader(
+                os.path.join(csr_data_dir, 'templates')),
+            extensions=[jinja2.ext.ExprStmtExtension, IPAExtension],
+            keep_trailing_newline=True, undefined=IndexableUndefined)
+
+        self.passthrough_globals = {}
+        self._define_passthrough('ipa.syntaxrule')
+        self._define_passthrough('ipa.datarule')
+
+    def _define_passthrough(self, call):
+
+        def passthrough(caller):
+            return u'{%% call %s() %%}%s{%% endcall %%}' % (call, caller())
+
+        parts = call.split('.')
+        current_level = self.passthrough_globals
+        for part in parts[:-1]:
+            if part not in current_level:
+                current_level[part] = {}
+            current_level = current_level[part]
+        current_level[parts[-1]] = passthrough
+
+    def build_template(self, rules):
+        """
+        Construct a template that can produce CSR generator strings.
+
+        :param rules: list of FieldMapping to use to populate the template.
+
+        :returns: jinja2.Template that can be rendered to produce the CSR data.
+        """
+        syntax_rules = []
+        for description, syntax_rule, data_rules in rules:
+            data_rules_prepared = [
+                self._prepare_data_rule(rule) for rule in data_rules]
+            syntax_rules.append(self._prepare_syntax_rule(
+                syntax_rule, data_rules_prepared, description))
+
+        template_params = self._get_template_params(syntax_rules)
+        base_template = self.jinja2.get_template(
+            self.base_template_name, globals=self.passthrough_globals)
+
+        try:
+            combined_template_source = base_template.render(**template_params)
+        except jinja2.UndefinedError:
+            logger.debug(traceback.format_exc())
+            raise errors.CSRTemplateError(reason=_(
+                'Template error when formatting certificate data'))
+
+        logger.debug(
+            'Formatting with template: %s' % combined_template_source)
+        combined_template = self.jinja2.from_string(combined_template_source)
+
+        return combined_template
+
+    def _wrap_rule(self, rule, rule_type):
+        template = '{%% call ipa.%srule() %%}%s{%% endcall %%}' % (
+            rule_type, rule)
+
+        return template
+
+    def _wrap_required(self, rule, description):
+        template = '{%% filter required("%s") %%}%s{%% endfilter %%}' % (
+            description, rule)
+
+        return template
+
+    def _prepare_data_rule(self, data_rule):
+        return self._wrap_rule(data_rule.template, 'data')
+
+    def _prepare_syntax_rule(self, syntax_rule, data_rules, description):
+        logger.debug('Syntax rule template: %s' % syntax_rule.template)
+        template = self.jinja2.from_string(
+            syntax_rule.template, globals=self.passthrough_globals)
+        is_required = syntax_rule.options.get('required', False)
+        try:
+            rendered = template.render(datarules=data_rules)
+        except jinja2.UndefinedError:
+            logger.debug(traceback.format_exc())
+            raise errors.CSRTemplateError(reason=_(
+                'Template error when formatting certificate data'))
+
+        prepared_template = self._wrap_rule(rendered, 'syntax')
+        if is_required:
+            prepared_template = self._wrap_required(
+                prepared_template, description)
+
+        return prepared_template
+
+    def _get_template_params(self, syntax_rules):
+        """
+        Package the syntax rules into fields expected by the base template.
+
+        :param syntax_rules: list of prepared syntax rules to be included in
+            the template.
+
+        :returns: dict of values needed to render the base template.
+        """
+        raise NotImplementedError('Formatter class must be subclassed')
+
+
+class OpenSSLFormatter(Formatter):
+    """Formatter class supporting the openssl command-line tool."""
+
+    base_template_name = 'openssl_base.tmpl'
+
+    # Syntax rules are wrapped in this data structure, to keep track of whether
+    # each goes in the extension or the root section
+    SyntaxRule = collections.namedtuple(
+        'SyntaxRule', ['template', 'is_extension'])
+
+    def __init__(self):
+        super(OpenSSLFormatter, self).__init__()
+        self._define_passthrough('openssl.section')
+
+    def _get_template_params(self, syntax_rules):
+        parameters = [rule.template for rule in syntax_rules
+                      if not rule.is_extension]
+        extensions = [rule.template for rule in syntax_rules
+                      if rule.is_extension]
+
+        return {'parameters': parameters, 'extensions': extensions}
+
+    def _prepare_syntax_rule(self, syntax_rule, data_rules, description):
+        """Overrides method to pull out whether rule is an extension or not."""
+        prepared_template = super(OpenSSLFormatter, self)._prepare_syntax_rule(
+            syntax_rule, data_rules, description)
+        is_extension = syntax_rule.options.get('extension', False)
+        return self.SyntaxRule(prepared_template, is_extension)
+
+
+class CertutilFormatter(Formatter):
+    base_template_name = 'certutil_base.tmpl'
+
+    def _get_template_params(self, syntax_rules):
+        return {'options': syntax_rules}
+
+
+# FieldMapping - representation of the rules needed to construct a complete
+# certificate field.
+# - description: str, a name or description of this field, to be used in
+#   messages
+# - syntax_rule: Rule, the rule defining the syntax of this field
+# - data_rules: list of Rule, the rules that produce data to be stored in this
+#   field
+FieldMapping = collections.namedtuple(
+    'FieldMapping', ['description', 'syntax_rule', 'data_rules'])
+Rule = collections.namedtuple(
+    'Rule', ['name', 'template', 'options'])
+
+
+class RuleProvider(object):
+    def rules_for_profile(self, profile_id, helper):
+        """
+        Return the rules needed to build a CSR using the given profile.
+
+        :param profile_id: str, name of the CSR generation profile to use
+        :param helper: str, name of tool (e.g. openssl, certutil) that will be
+            used to create CSR
+
+        :returns: list of FieldMapping, filled out with the appropriate rules
+        """
+        raise NotImplementedError('RuleProvider class must be subclassed')
+
+
+class FileRuleProvider(RuleProvider):
+    def __init__(self, csr_data_dir=paths.CSR_DATA_DIR):
+        self.rules = {}
+        self.csr_data_dir = csr_data_dir
+
+    def _rule(self, rule_name, helper):
+        if (rule_name, helper) not in self.rules:
+            rule_path = os.path.join(self.csr_data_dir, 'rules',
+                                     '%s.json' % rule_name)
+            try:
+                with open(rule_path) as rule_file:
+                    ruleset = json.load(rule_file)
+            except IOError:
+                raise errors.NotFound(
+                    reason=_('Ruleset %(ruleset)s does not exist.') %
+                    {'ruleset': rule_name})
+
+            matching_rules = [r for r in ruleset['rules']
+                              if r['helper'] == helper]
+            if len(matching_rules) == 0:
+                raise errors.EmptyResult(
+                    reason=_('No transformation in "%(ruleset)s" rule supports'
+                             ' helper "%(helper)s"') %
+                    {'ruleset': rule_name, 'helper': helper})
+            elif len(matching_rules) > 1:
+                raise errors.RedundantMappingRule(
+                    ruleset=rule_name, helper=helper)
+            rule = matching_rules[0]
+
+            options = {}
+            if 'options' in ruleset:
+                options.update(ruleset['options'])
+            if 'options' in rule:
+                options.update(rule['options'])
+            self.rules[(rule_name, helper)] = Rule(
+                rule_name, rule['template'], options)
+        return self.rules[(rule_name, helper)]
+
+    def rules_for_profile(self, profile_id, helper):
+        profile_path = os.path.join(self.csr_data_dir, 'profiles',
+                                    '%s.json' % profile_id)
+        with open(profile_path) as profile_file:
+            profile = json.load(profile_file)
+
+        field_mappings = []
+        for field in profile:
+            syntax_rule = self._rule(field['syntax'], helper)
+            data_rules = [self._rule(name, helper) for name in field['data']]
+            field_mappings.append(FieldMapping(
+                syntax_rule.name, syntax_rule, data_rules))
+        return field_mappings
+
+
+class CSRGenerator(object):
+    FORMATTERS = {
+        'openssl': OpenSSLFormatter,
+        'certutil': CertutilFormatter,
+    }
+
+    def __init__(self, rule_provider):
+        self.rule_provider = rule_provider
+
+    def csr_script(self, principal, profile_id, helper):
+        config = api.Command.config_show()['result']
+        render_data = {'subject': principal, 'config': config}
+
+        formatter = self.FORMATTERS[helper]()
+        rules = self.rule_provider.rules_for_profile(profile_id, helper)
+        template = formatter.build_template(rules)
+
+        try:
+            script = template.render(render_data)
+        except jinja2.UndefinedError:
+            logger.debug(traceback.format_exc())
+            raise errors.CSRTemplateError(reason=_(
+                'Template error when formatting certificate data'))
+
+        return script

ipaclient/plugins/csrgen.py

@@ -0,0 +1,114 @@
+#
+# Copyright (C) 2016  FreeIPA Contributors see COPYING for license
+#
+
+import six
+
+from ipaclient.csrgen import CSRGenerator, FileRuleProvider
+from ipalib import api
+from ipalib import errors
+from ipalib import output
+from ipalib import util
+from ipalib.frontend import Local, Str
+from ipalib.parameters import Principal
+from ipalib.plugable import Registry
+from ipalib.text import _
+
+if six.PY3:
+    unicode = str
+
+register = Registry()
+
+__doc__ = _("""
+Commands to build certificate requests automatically
+""")
+
+
+@register()
+class cert_get_requestdata(Local):
+    __doc__ = _('Gather data for a certificate signing request.')
+
+    takes_options = (
+        Principal(
+            'principal',
+            label=_('Principal'),
+            doc=_('Principal for this certificate (e.g.'
+                  ' HTTP/test.example.com)'),
+        ),
+        Str(
+            'profile_id',
+            label=_('Profile ID'),
+            doc=_('CSR Generation Profile to use'),
+        ),
+        Str(
+            'helper',
+            label=_('Name of CSR generation tool'),
+            doc=_('Name of tool (e.g. openssl, certutil) that will be used to'
+                  ' create CSR'),
+        ),
+        Str(
+            'out?',
+            doc=_('Write CSR generation script to file'),
+        ),
+    )
+
+    has_output = (
+        output.Output(
+            'result',
+            type=dict,
+            doc=_('Dictionary mapping variable name to value'),
+        ),
+    )
+
+    has_output_params = (
+        Str(
+            'script',
+            label=_('Generation script'),
+        )
+    )
+
+    def execute(self, *args, **options):
+        if 'out' in options:
+            util.check_writable_file(options['out'])
+
+        principal = options.get('principal')
+        profile_id = options.get('profile_id')
+        helper = options.get('helper')
+
+        if self.api.env.in_server:
+            backend = self.api.Backend.ldap2
+        else:
+            backend = self.api.Backend.rpcclient
+        if not backend.isconnected():
+            backend.connect()
+
+        try:
+            if principal.is_host:
+                principal_obj = api.Command.host_show(
+                    principal.hostname, all=True)
+            elif principal.is_service:
+                principal_obj = api.Command.service_show(
+                    unicode(principal), all=True)
+            elif principal.is_user:
+                principal_obj = api.Command.user_show(
+                    principal.username, all=True)
+        except errors.NotFound:
+            raise errors.NotFound(
+                reason=_("The principal for this request doesn't exist."))
+        principal_obj = principal_obj['result']
+
+        generator = CSRGenerator(FileRuleProvider())
+
+        script = generator.csr_script(
+            principal_obj, profile_id, helper)
+
+        result = {}
+        if 'out' in options:
+            with open(options['out'], 'wb') as f:
+                f.write(script)
+        else:
+            result = dict(script=script)
+
+        return dict(
+            result=result
+        )

ipaclient/setup.py

@@ -47,6 +47,7 @@ if __name__ == '__main__':
             "cryptography",
             "ipalib",
             "ipapython",
+            "jinja2",
             "python-nss",
             "python-yubico",
             "pyusb",

ipalib/errors.py

@@ -1422,6 +1422,34 @@ class HTTPRequestError(RemoteRetrieveError):
     format = _('Request failed with status %(status)s: %(reason)s')
 
 
+class RedundantMappingRule(SingleMatchExpected):
+    """
+    **4036** Raised when more than one rule in a CSR generation ruleset matches
+    a particular helper.
+
+    For example:
+
+    >>> raise RedundantMappingRule(ruleset='syntaxSubject', helper='certutil')
+    Traceback (most recent call last):
+      ...
+    RedundantMappingRule: Mapping ruleset "syntaxSubject" has more than one
+    rule for the certutil helper.
+    """
+
+    errno = 4036
+    format = _('Mapping ruleset "%(ruleset)s" has more than one rule for the'
+               ' %(helper)s helper')
+
+
+class CSRTemplateError(ExecutionError):
+    """
+    **4037** Raised when evaluation of a CSR generation template fails
+    """
+
+    errno = 4037
+    format = _('%(reason)s')
+
+
 class BuiltinError(ExecutionError):
     """
     **4100** Base class for builtin execution errors (*4100 - 4199*).

ipaplatform/base/paths.py

@@ -233,6 +233,7 @@ class BasePathNamespace(object):
     SCHEMA_COMPAT_ULDIF = "/usr/share/ipa/schema_compat.uldif"
     IPA_JS_PLUGINS_DIR = "/usr/share/ipa/ui/js/plugins"
     UPDATES_DIR = "/usr/share/ipa/updates/"
+    CSR_DATA_DIR = "/usr/share/ipa/csrgen"
     DICT_WORDS = "/usr/share/dict/words"
     CACHE_IPA_SESSIONS = "/var/cache/ipa/sessions"
     VAR_KERBEROS_KRB5KDC_DIR = "/var/kerberos/krb5kdc/"