Make certificate renewal process synchronized

Synchronization is achieved using a global renewal lock.

https://fedorahosted.org/freeipa/ticket/4803

Reviewed-By: David Kupka <dkupka@redhat.com>

freeipa.spec.in

@@ -658,6 +658,7 @@ fi
 %{_sbindir}/ipa-advise
 %{_sbindir}/ipa-cacert-manage
 %{_libexecdir}/certmonger/dogtag-ipa-ca-renew-agent-submit
+%{_libexecdir}/certmonger/ipa-server-guard
 %{_libexecdir}/ipa-otpd
 %dir %{_libexecdir}/ipa
 %{_libexecdir}/ipa/ipa-dnskeysyncd

install/certmonger/Makefile.am

@@ -3,6 +3,7 @@ NULL =
 appdir = $(libexecdir)/certmonger/
 app_SCRIPTS =					\
 	dogtag-ipa-ca-renew-agent-submit	\
+	ipa-server-guard			\
 	$(NULL)
 
 EXTRA_DIST =				\

install/certmonger/dogtag-ipa-ca-renew-agent-submit

@@ -38,7 +38,7 @@ from ipapython.dn import DN
 from ipalib import api, errors, pkcs10, x509
 from ipaplatform.paths import paths
 from ipaserver.plugins.ldap2 import ldap2
-from ipaserver.install import cainstance
+from ipaserver.install import cainstance, certs
 
 # This is a certmonger CA helper script for IPA CA subsystem cert renewal. See
 # https://git.fedorahosted.org/cgit/certmonger.git/tree/doc/submit.txt for more
@@ -437,6 +437,7 @@ def main():
         return OPERATION_NOT_SUPPORTED_BY_HELPER
 
     tmpdir = tempfile.mkdtemp(prefix="tmp-")
+    certs.renewal_lock.acquire()
     try:
         principal = str('host/%s@%s' % (api.env.host, api.env.realm))
         ipautil.kinit_hostprincipal(paths.KRB5_KEYTAB, tmpdir, principal)
@@ -456,6 +457,7 @@ def main():
             print item
         return res[0]
     finally:
+        certs.renewal_lock.release()
         shutil.rmtree(tmpdir)
 
 try:

install/certmonger/ipa-server-guard

@@ -0,0 +1,55 @@
+#!/usr/bin/python2 -E
+#
+# Authors:
+#   Jan Cholasta <jcholast@redhat.com>
+#
+# Copyright (C) 2015  Red Hat
+# see file 'COPYING' for use and warranty information
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+# Prevent garbage from readline on standard output
+# (see https://fedorahosted.org/freeipa/ticket/4064)
+if not os.isatty(1):
+    os.environ['TERM'] = 'dumb'
+import sys
+import syslog
+import traceback
+
+from ipapython import ipautil
+from ipaserver.install import certs
+
+
+def main():
+    if len(sys.argv) < 2:
+        raise RuntimeError("Not enough arguments")
+
+    with certs.renewal_lock:
+        stdout, stderr, rc = ipautil.run(sys.argv[1:], raiseonerr=False,
+                                         env=os.environ)
+        sys.stdout.write(stdout)
+        sys.stdout.flush()
+        sys.stderr.write(stderr)
+        sys.stderr.flush()
+
+    return rc
+
+
+try:
+    sys.exit(main())
+except Exception, e:
+    syslog.syslog(syslog.LOG_ERR, traceback.format_exc())
+    print "Internal error"
+    sys.exit(3)

install/restart_scripts/renew_ca_cert

@@ -35,7 +35,7 @@ from ipaplatform import services
 from ipaplatform.paths import paths
 
 
-def main():
+def _main():
     nickname = sys.argv[1]
 
     api.bootstrap(context='restart')
@@ -210,6 +210,14 @@ def main():
         syslog.syslog(
             syslog.LOG_NOTICE, "Started %s" % dogtag_service.service_name)
 
+
+def main():
+    try:
+        _main()
+    finally:
+        certs.renewal_lock.release('renew_ca_cert')
+
+
 try:
     main()
 except Exception:

install/restart_scripts/renew_ra_cert

@@ -32,9 +32,10 @@ from ipaserver.install import certs, cainstance
 from ipaplatform import services
 from ipaplatform.paths import paths
 
-nickname = 'ipaCert'
 
-def main():
+def _main():
+    nickname = 'ipaCert'
+
     api.bootstrap(context='restart')
     api.finalize()
 
@@ -68,6 +69,12 @@ def main():
     else:
         syslog.syslog(syslog.LOG_NOTICE, "Restarted httpd")
 
+
+def main():
+    with certs.renewal_lock:
+        _main()
+
+
 try:
     main()
 except Exception:

install/restart_scripts/restart_dirsrv

@@ -24,8 +24,10 @@ import syslog
 import traceback
 from ipalib import api
 from ipaplatform import services
+from ipaserver.install import certs
 
-def main():
+
+def _main():
     try:
         instance = sys.argv[1]
     except IndexError:
@@ -41,6 +43,12 @@ def main():
     except Exception, e:
         syslog.syslog(syslog.LOG_ERR, "Cannot restart dirsrv (instance: '%s'): %s" % (instance, str(e)))
 
+
+def main():
+    with certs.renewal_lock:
+        _main()
+
+
 try:
     main()
 except Exception:

install/restart_scripts/restart_httpd

@@ -22,8 +22,10 @@
 import syslog
 import traceback
 from ipaplatform import services
+from ipaserver.install import certs
 
-def main():
+
+def _main():
     syslog.syslog(syslog.LOG_NOTICE, 'certmonger restarted httpd')
 
     try:
@@ -31,6 +33,12 @@ def main():
     except Exception, e:
         syslog.syslog(syslog.LOG_ERR, "Cannot restart httpd: %s" % str(e))
 
+
+def main():
+    with certs.renewal_lock:
+        _main()
+
+
 try:
     main()
 except Exception:

install/restart_scripts/stop_pkicad

@@ -25,6 +25,8 @@ import traceback
 from ipapython import dogtag
 from ipalib import api
 from ipaplatform import services
+from ipaserver.install import certs
+
 
 def main():
     api.bootstrap(context='restart')
@@ -34,6 +36,8 @@ def main():
     dogtag_service = services.knownservices[configured_constants.SERVICE_NAME]
     dogtag_instance = configured_constants.PKI_INSTANCE_NAME
 
+    certs.renewal_lock.acquire('renew_ca_cert')
+
     syslog.syslog(syslog.LOG_NOTICE, "Stopping %s" % dogtag_service.service_name)
     try:
         dogtag_service.stop(dogtag_instance)

install/tools/ipa-upgradeconfig

@@ -1327,6 +1327,8 @@ def main():
             )
         upgrade_pki(ca, fstore)
 
+    ca.configure_certmonger_renewal_guard()
+
     update_dbmodules(api.env.realm)
     uninstall_ipa_kpasswd()
 
@@ -1339,6 +1341,7 @@ def main():
     http = httpinstance.HTTPInstance(fstore)
     http.configure_selinux_for_httpd()
     http.change_mod_nss_port_from_http()
+    http.configure_certmonger_renewal_guard()
 
     http.stop()
     update_mod_nss_protocol(http)

ipaplatform/base/paths.py

@@ -206,6 +206,7 @@ class BasePathNamespace(object):
     LIBSOFTHSM2_SO_64 = "/usr/lib64/pkcs11/libsofthsm2.so"
     DOGTAG_IPA_CA_RENEW_AGENT_SUBMIT = "/usr/libexec/certmonger/dogtag-ipa-ca-renew-agent-submit"
     DOGTAG_IPA_RENEW_AGENT_SUBMIT = "/usr/libexec/certmonger/dogtag-ipa-renew-agent-submit"
+    IPA_SERVER_GUARD = "/usr/libexec/certmonger/ipa-server-guard"
     IPA_DNSKEYSYNCD_REPLICA = "/usr/libexec/ipa/ipa-dnskeysync-replica"
     IPA_DNSKEYSYNCD = "/usr/libexec/ipa/ipa-dnskeysyncd"
     IPA_ODS_EXPORTER = "/usr/libexec/ipa/ipa-ods-exporter"
@@ -326,6 +327,7 @@ class BasePathNamespace(object):
     VAR_OPENDNSSEC_DIR = "/var/opendnssec"
     OPENDNSSEC_KASP_DB = "/var/opendnssec/kasp.db"
     VAR_RUN_DIRSRV_DIR = "/var/run/dirsrv"
+    IPA_RENEWAL_LOCK = "/var/run/ipa/renewal.lock"
     SVC_LIST_FILE = "/var/run/ipa/services.list"
     IPA_MEMCACHED_DIR = "/var/run/ipa_memcached"
     VAR_RUN_IPA_MEMCACHED = "/var/run/ipa_memcached/ipa_memcached"

ipaserver/install/cainstance.py

@@ -38,6 +38,8 @@ import time
 import tempfile
 import urllib
 import xml.dom.minidom
+import shlex
+import pipes
 
 from ipalib import api
 from ipalib import pkcs10, x509
@@ -1304,6 +1306,16 @@ class CAInstance(DogtagInstance):
         if path:
             iface.remove_known_ca(path)
 
+        helper = self.restore_state('certmonger_dogtag_helper')
+        if helper:
+            path = iface.find_ca_by_nickname('dogtag-ipa-renew-agent')
+            if path:
+                ca_obj = bus.get_object('org.fedorahosted.certmonger', path)
+                ca_iface = dbus.Interface(ca_obj,
+                                          'org.freedesktop.DBus.Properties')
+                ca_iface.Set('org.fedorahosted.certmonger.ca',
+                             'external-helper', helper)
+
         cmonger.stop()
 
         # remove CRL files
@@ -1332,6 +1344,36 @@ class CAInstance(DogtagInstance):
         fd.close()
         os.chmod(location, 0444)
 
+
+    def configure_certmonger_renewal(self):
+        super(CAInstance, self).configure_certmonger_renewal()
+
+        self.configure_certmonger_renewal_guard()
+
+    def configure_certmonger_renewal_guard(self):
+        if not self.is_configured():
+            return
+
+        bus = dbus.SystemBus()
+        obj = bus.get_object('org.fedorahosted.certmonger',
+                             '/org/fedorahosted/certmonger')
+        iface = dbus.Interface(obj, 'org.fedorahosted.certmonger')
+        path = iface.find_ca_by_nickname('dogtag-ipa-renew-agent')
+        if path:
+            ca_obj = bus.get_object('org.fedorahosted.certmonger', path)
+            ca_iface = dbus.Interface(ca_obj,
+                                      'org.freedesktop.DBus.Properties')
+            helper = ca_iface.Get('org.fedorahosted.certmonger.ca',
+                                  'external-helper')
+            if helper:
+                args = shlex.split(helper)
+                if args[0] != paths.IPA_SERVER_GUARD:
+                    self.backup_state('certmonger_dogtag_helper', helper)
+                    args = [paths.IPA_SERVER_GUARD] + args
+                    helper = ' '.join(pipes.quote(a) for a in args)
+                    ca_iface.Set('org.fedorahosted.certmonger.ca',
+                                 'external-helper', helper)
+
     def configure_agent_renewal(self):
         try:
             certmonger.dogtag_start_tracking(

ipaserver/install/certs.py

@@ -26,6 +26,10 @@ import xml.dom.minidom
 import pwd
 import base64
 from hashlib import sha1
+import fcntl
+import time
+import datetime
+import ConfigParser as configparser
 
 from ipapython.ipa_log_manager import root_logger
 from ipapython import dogtag
@@ -647,3 +651,103 @@ class CertDB(object):
 
     def export_pem_cert(self, nickname, location):
         return self.nssdb.export_pem_cert(nickname, location)
+
+
+class _CrossProcessLock(object):
+    _DATETIME_FORMAT = '%Y%m%d%H%M%S%f'
+
+    def __init__(self, filename):
+        self._filename = filename
+
+    def __enter__(self):
+        self.acquire()
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.release()
+
+    def acquire(self, owner=None):
+        self._do(self._acquire, owner)
+
+    def release(self, owner=None):
+        self._do(self._release, owner)
+
+    def _acquire(self, owner):
+        now = datetime.datetime.utcnow()
+
+        if self._locked and now >= self._expire:
+            self._locked = False
+
+        if self._locked:
+            return False
+
+        self._locked = True
+        self._owner = owner
+        self._expire = now + datetime.timedelta(hours=1)
+
+        return True
+
+    def _release(self, owner):
+        if not self._locked or self._owner != owner:
+            raise RuntimeError("lock not acquired by %s" % owner)
+
+        self._locked = False
+        self._owner = None
+        self._expire = None
+
+        return True
+
+    def _do(self, func, owner):
+        if owner is None:
+            owner = '%s[%s]' % (os.path.basename(sys.argv[0]), os.getpid())
+
+        while True:
+            with open(self._filename, 'a+') as f:
+                fcntl.flock(f, fcntl.LOCK_EX)
+
+                f.seek(0)
+                self._read(f)
+
+                if func(owner):
+                    f.seek(0)
+                    f.truncate()
+                    self._write(f)
+                    return
+
+            time.sleep(10)
+
+    def _read(self, fileobj):
+        p = configparser.RawConfigParser()
+        p.readfp(fileobj)
+
+        try:
+            self._locked = p.getboolean('lock', 'locked')
+
+            if self._locked:
+                self._owner = p.get('lock', 'owner')
+
+                expire = p.get('lock', 'expire')
+                try:
+                    self._expire = datetime.datetime.strptime(
+                        expire, self._DATETIME_FORMAT)
+                except ValueError:
+                    raise configparser.Error
+        except configparser.Error:
+            self._locked = False
+            self._owner = None
+            self._expire = None
+
+    def _write(self, fileobj):
+        p = configparser.RawConfigParser()
+        p.add_section('lock')
+
+        locked = '1' if self._locked else '0'
+        p.set('lock', 'locked', locked)
+
+        if self._locked:
+            expire = self._expire.strftime(self._DATETIME_FORMAT)
+            p.set('lock', 'owner', self._owner)
+            p.set('lock', 'expire', expire)
+
+        p.write(fileobj)
+
+renewal_lock = _CrossProcessLock(paths.IPA_RENEWAL_LOCK)

ipaserver/install/dogtaginstance.py

@@ -296,8 +296,7 @@ class DogtagInstance(service.Service):
         with open(paths.HTTPD_IPA_PKI_PROXY_CONF, "w") as fd:
             fd.write(template)
 
-    @staticmethod
+    def configure_certmonger_renewal(self):
-    def configure_certmonger_renewal():
         """
         Create a new CA type for certmonger that will retrieve updated
         certificates from the dogtag master server.

ipaserver/install/httpinstance.py

@@ -23,6 +23,9 @@ import tempfile
 import pwd
 import shutil
 import re
+import dbus
+import shlex
+import pipes
 
 import service
 import certs
@@ -121,6 +124,9 @@ class HTTPInstance(service.Service):
         self.step("enabling mod_nss renegotiate", self.enable_mod_nss_renegotiate)
         self.step("adding URL rewriting rules", self.__add_include)
         self.step("configuring httpd", self.__configure_http)
+        if self.ca_is_configured:
+            self.step("configure certmonger for renewals",
+                      self.configure_certmonger_renewal_guard)
         self.step("setting up ssl", self.__setup_ssl)
         self.step("importing CA certificates from LDAP", self.__import_ca_certs)
         if autoconfig:
@@ -221,6 +227,27 @@ class HTTPInstance(service.Service):
         if installutils.update_file(paths.HTTPD_NSS_CONF, '</VirtualHost>', 'Include conf.d/ipa-rewrite.conf\n</VirtualHost>') != 0:
             print "Adding Include conf.d/ipa-rewrite to %s failed." % paths.HTTPD_NSS_CONF
 
+    def configure_certmonger_renewal_guard(self):
+        bus = dbus.SystemBus()
+        obj = bus.get_object('org.fedorahosted.certmonger',
+                             '/org/fedorahosted/certmonger')
+        iface = dbus.Interface(obj, 'org.fedorahosted.certmonger')
+        path = iface.find_ca_by_nickname('IPA')
+        if path:
+            ca_obj = bus.get_object('org.fedorahosted.certmonger', path)
+            ca_iface = dbus.Interface(ca_obj,
+                                      'org.freedesktop.DBus.Properties')
+            helper = ca_iface.Get('org.fedorahosted.certmonger.ca',
+                                  'external-helper')
+            if helper:
+                args = shlex.split(helper)
+                if args[0] != paths.IPA_SERVER_GUARD:
+                    self.backup_state('certmonger_ipa_helper', helper)
+                    args = [paths.IPA_SERVER_GUARD] + args
+                    helper = ' '.join(pipes.quote(a) for a in args)
+                    ca_iface.Set('org.fedorahosted.certmonger.ca',
+                                 'external-helper', helper)
+
     def __setup_ssl(self):
         fqdn = self.fqdn
 
@@ -355,6 +382,21 @@ class HTTPInstance(service.Service):
             self.stop()
 
         self.stop_tracking_certificates()
+
+        helper = self.restore_state('certmonger_ipa_helper')
+        if helper:
+            bus = dbus.SystemBus()
+            obj = bus.get_object('org.fedorahosted.certmonger',
+                                 '/org/fedorahosted/certmonger')
+            iface = dbus.Interface(obj, 'org.fedorahosted.certmonger')
+            path = iface.find_ca_by_nickname('IPA')
+            if path:
+                ca_obj = bus.get_object('org.fedorahosted.certmonger', path)
+                ca_iface = dbus.Interface(ca_obj,
+                                          'org.freedesktop.DBus.Properties')
+                ca_iface.Set('org.fedorahosted.certmonger.ca',
+                             'external-helper', helper)
+
         if not enabled is None and not enabled:
             self.disable()