Log unhandled exceptions in certificate renewal scripts.

https://fedorahosted.org/freeipa/ticket/4093

Reviewed-By: Petr Viktorin <pviktori@redhat.com>

install/certmonger/dogtag-ipa-retrieve-agent-submit

@@ -31,6 +31,8 @@ import sys
 import shutil
 import tempfile
 import syslog
+import base64
+import traceback
 from ipalib import api
 from ipapython.dn import DN
 from ipalib import errors
@@ -39,45 +41,52 @@ from ipapython import services as ipaservices
 from ipapython import ipautil
 from ipaserver.install import certs
 from ipaserver.plugins.ldap2 import ldap2
-import base64
 
-# We cheat and pass in the nickname as the CA profile to execute against.
+def main():
-# Some way is needed to determine which entry to retrieve from LDAP
+    # We cheat and pass in the nickname as the CA profile to execute against.
-operation = os.environ.get('CERTMONGER_OPERATION')
+    # Some way is needed to determine which entry to retrieve from LDAP
-nickname = os.environ.get('CERTMONGER_CA_PROFILE')
+    operation = os.environ.get('CERTMONGER_OPERATION')
+    nickname = os.environ.get('CERTMONGER_CA_PROFILE')
 
-if operation not in ['SUBMIT', 'POLL']:
+    if operation not in ['SUBMIT', 'POLL']:
-    sys.exit(6) # unsupported operation
+        sys.exit(6) # unsupported operation
 
-api.bootstrap(context='renew')
+    api.bootstrap(context='renew')
-api.finalize()
+    api.finalize()
 
-# Update or add it
+    # Update or add it
-tmpdir = tempfile.mkdtemp(prefix = "tmp-")
+    tmpdir = tempfile.mkdtemp(prefix = "tmp-")
-try:
-    dn = DN(('cn', nickname), ('cn', 'ca_renewal'), ('cn', 'ipa'), ('cn', 'etc'), api.env.basedn)
-    principal = str('host/%s@%s' % (api.env.host, api.env.realm))
-    ccache = ipautil.kinit_hostprincipal('/etc/krb5.keytab', tmpdir, principal)
-    conn = ldap2(shared_instance=False, ldap_uri=api.env.ldap_uri)
-    conn.connect(ccache=ccache)
     try:
-        syslog.syslog(syslog.LOG_NOTICE, "Updating certificate for %s" % nickname)
+        dn = DN(('cn', nickname), ('cn', 'ca_renewal'), ('cn', 'ipa'), ('cn', 'etc'), api.env.basedn)
-        entry_attrs = conn.get_entry(dn, ['usercertificate'])
+        principal = str('host/%s@%s' % (api.env.host, api.env.realm))
-        cert = entry_attrs['usercertificate'][0]
+        ccache = ipautil.kinit_hostprincipal('/etc/krb5.keytab', tmpdir, principal)
-        cert = base64.b64encode(cert)
+        conn = ldap2(shared_instance=False, ldap_uri=api.env.ldap_uri)
-        print x509.make_pem(cert)
+        conn.connect(ccache=ccache)
-    except errors.NotFound:
+        try:
-        syslog.syslog(syslog.LOG_INFO, "Updated certificate for %s not available" % nickname)
+            syslog.syslog(syslog.LOG_NOTICE, "Updating certificate for %s" % nickname)
-        # No cert available yet, tell certmonger to wait another 8 hours
+            entry_attrs = conn.get_entry(dn, ['usercertificate'])
-        print 8 * 60 * 60
+            cert = entry_attrs['usercertificate'][0]
-        sys.exit(5)
+            cert = base64.b64encode(cert)
+            print x509.make_pem(cert)
+        except errors.NotFound:
+            syslog.syslog(syslog.LOG_INFO, "Updated certificate for %s not available" % nickname)
+            # No cert available yet, tell certmonger to wait another 8 hours
+            print 8 * 60 * 60
+            sys.exit(5)
+        finally:
+            conn.disconnect()
+    except Exception, e:
+        syslog.syslog(syslog.LOG_ERR, "Exception trying to retrieve %s: %s" % (nickname, e))
+        # Unhandled error
+        sys.exit(3)
     finally:
-        conn.disconnect()
+        shutil.rmtree(tmpdir)
-except Exception, e:
-    syslog.syslog(syslog.LOG_ERR, "Exception trying to retrieve %s: %s" % (nickname, e))
-    # Unhandled error
-    sys.exit(3)
-finally:
-    shutil.rmtree(tmpdir)
 
-sys.exit(0)
+    sys.exit(0)
+
+try:
+    main()
+except Exception:
+    syslog.syslog(syslog.LOG_ERR, traceback.format_exc())
+    print "Internal error"
+    sys.exit(3)

install/restart_scripts/renew_ca_cert

@@ -26,6 +26,7 @@ import tempfile
 import syslog
 import random
 import time
+import traceback
 from ipalib import api
 from ipapython.dn import DN
 from ipalib import errors
@@ -42,80 +43,86 @@ from ipapython import certmonger
 # has renewed a CA subsystem certificate a copy is put into the replicated
 # tree so it can be shared with the other IPA servers.
 
-nickname = sys.argv[1]
+def main():
+    nickname = sys.argv[1]
 
-api.bootstrap(context='restart')
+    api.bootstrap(context='restart')
-api.finalize()
+    api.finalize()
 
-configured_constants = dogtag.configured_constants(api)
+    configured_constants = dogtag.configured_constants(api)
-alias_dir = configured_constants.ALIAS_DIR
+    alias_dir = configured_constants.ALIAS_DIR
-dogtag_service = ipaservices.knownservices[configured_constants.SERVICE_NAME]
+    dogtag_service = ipaservices.knownservices[configured_constants.SERVICE_NAME]
-dogtag_instance = configured_constants.PKI_INSTANCE_NAME
+    dogtag_instance = configured_constants.PKI_INSTANCE_NAME
 
-# Fetch the new certificate
+    # Fetch the new certificate
-db = certs.CertDB(api.env.realm, nssdir=alias_dir)
-cert = db.get_cert_from_db(nickname, pem=False)
-
-if not cert:
-    syslog.syslog(syslog.LOG_ERR, 'No certificate %s found.' % nickname)
-    sys.exit(1)
-
-# Update or add it
-tmpdir = tempfile.mkdtemp(prefix = "tmp-")
-try:
-    dn = DN(('cn',nickname), ('cn', 'ca_renewal'), ('cn', 'ipa'), ('cn', 'etc'), api.env.basedn)
-    principal = str('host/%s@%s' % (api.env.host, api.env.realm))
-    ccache = ipautil.kinit_hostprincipal('/etc/krb5.keytab', tmpdir, principal)
-    conn = ldap2(shared_instance=False, ldap_uri=api.env.ldap_uri)
-    conn.connect(ccache=ccache)
-    try:
-        entry_attrs = conn.get_entry(dn, ['usercertificate'])
-        entry_attrs['usercertificate'] = cert
-        conn.update_entry(entry_attrs)
-    except errors.NotFound:
-        entry_attrs = conn.make_entry(
-            dn,
-            objectclass=['top', 'pkiuser', 'nscontainer'],
-            usercertificate=[cert])
-        conn.add_entry(entry_attrs)
-    except errors.EmptyModlist:
-        pass
-    conn.disconnect()
-except Exception, e:
-    syslog.syslog(syslog.LOG_ERR, 'Updating renewal certificate failed: %s' % e)
-finally:
-    shutil.rmtree(tmpdir)
-
-# Done withing stopped_service context, CA restarted here
-update_cert_config(nickname, cert)
-
-if nickname == 'subsystemCert cert-pki-ca':
-    update_people_entry('pkidbuser', cert)
-
-if nickname == 'auditSigningCert cert-pki-ca':
-    # Fix trust on the audit cert
     db = certs.CertDB(api.env.realm, nssdir=alias_dir)
-    args = ['-M',
+    cert = db.get_cert_from_db(nickname, pem=False)
-            '-n', nickname,
+
-            '-t', 'u,u,Pu',
+    if not cert:
-           ]
+        syslog.syslog(syslog.LOG_ERR, 'No certificate %s found.' % nickname)
-    try:
+        sys.exit(1)
-        db.run_certutil(args)
+
-        syslog.syslog(syslog.LOG_NOTICE, 'Updated trust on certificate %s in %s' % (nickname, db.secdir))
+    # Update or add it
-    except ipautil.CalledProcessError:
+    tmpdir = tempfile.mkdtemp(prefix = "tmp-")
-         syslog.syslog(syslog.LOG_ERR, 'Updating trust on certificate %s failed in %s' % (nickname, db.secdir))
+    try:
+        dn = DN(('cn',nickname), ('cn', 'ca_renewal'), ('cn', 'ipa'), ('cn', 'etc'), api.env.basedn)
+        principal = str('host/%s@%s' % (api.env.host, api.env.realm))
+        ccache = ipautil.kinit_hostprincipal('/etc/krb5.keytab', tmpdir, principal)
+        conn = ldap2(shared_instance=False, ldap_uri=api.env.ldap_uri)
+        conn.connect(ccache=ccache)
+        try:
+            entry_attrs = conn.get_entry(dn, ['usercertificate'])
+            entry_attrs['usercertificate'] = cert
+            conn.update_entry(entry_attrs)
+        except errors.NotFound:
+            entry_attrs = conn.make_entry(
+                dn,
+                objectclass=['top', 'pkiuser', 'nscontainer'],
+                usercertificate=[cert])
+            conn.add_entry(entry_attrs)
+        except errors.EmptyModlist:
+            pass
+        conn.disconnect()
+    except Exception, e:
+        syslog.syslog(syslog.LOG_ERR, 'Updating renewal certificate failed: %s' % e)
+    finally:
+        shutil.rmtree(tmpdir)
+
+    # Done withing stopped_service context, CA restarted here
+    update_cert_config(nickname, cert)
+
+    if nickname == 'subsystemCert cert-pki-ca':
+        update_people_entry('pkidbuser', cert)
+
+    if nickname == 'auditSigningCert cert-pki-ca':
+        # Fix trust on the audit cert
+        db = certs.CertDB(api.env.realm, nssdir=alias_dir)
+        args = ['-M',
+                '-n', nickname,
+                '-t', 'u,u,Pu',
+               ]
+        try:
+            db.run_certutil(args)
+            syslog.syslog(syslog.LOG_NOTICE, 'Updated trust on certificate %s in %s' % (nickname, db.secdir))
+        except ipautil.CalledProcessError:
+             syslog.syslog(syslog.LOG_ERR, 'Updating trust on certificate %s failed in %s' % (nickname, db.secdir))
+
+    # Now we can start the CA. Using the ipaservices start should fire
+    # off the servlet to verify that the CA is actually up and responding so
+    # when this returns it should be good-to-go. The CA was stopped in the
+    # pre-save state.
+    syslog.syslog(syslog.LOG_NOTICE, 'Starting %s' % dogtag_service.service_name)
+    try:
+        dogtag_service.start(dogtag_instance)
+    except Exception, e:
+        syslog.syslog(
+            syslog.LOG_ERR,
+            "Cannot start %s: %s" % (dogtag_service.service_name, e))
+    else:
+        syslog.syslog(
+            syslog.LOG_NOTICE, "Started %s" % dogtag_service.service_name)
 
-# Now we can start the CA. Using the ipaservices start should fire
-# off the servlet to verify that the CA is actually up and responding so
-# when this returns it should be good-to-go. The CA was stopped in the
-# pre-save state.
-syslog.syslog(syslog.LOG_NOTICE, 'Starting %s' % dogtag_service.service_name)
 try:
-    dogtag_service.start(dogtag_instance)
+    main()
-except Exception, e:
+except Exception:
-    syslog.syslog(
+    syslog.syslog(syslog.LOG_ERR, traceback.format_exc())
-        syslog.LOG_ERR,
-        "Cannot start %s: %s" % (dogtag_service.service_name, e))
-else:
-    syslog.syslog(
-        syslog.LOG_NOTICE, "Started %s" % dogtag_service.service_name)

install/restart_scripts/renew_ra_cert

@@ -24,6 +24,7 @@ import shutil
 import tempfile
 import syslog
 import time
+import traceback
 from ipapython import services as ipaservices
 from ipapython import ipautil
 from ipaserver.install import certs
@@ -33,60 +34,66 @@ from ipapython.dn import DN
 from ipalib import errors
 from ipaserver.plugins.ldap2 import ldap2
 
-api.bootstrap(context='restart')
+def main():
-api.finalize()
+    api.bootstrap(context='restart')
+    api.finalize()
 
-# Fetch the new certificate
+    # Fetch the new certificate
-db = certs.CertDB(api.env.realm)
+    db = certs.CertDB(api.env.realm)
-dercert = db.get_cert_from_db('ipaCert', pem=False)
+    dercert = db.get_cert_from_db('ipaCert', pem=False)
 
-# Load it into dogtag
+    # Load it into dogtag
-update_people_entry('ipara', dercert)
+    update_people_entry('ipara', dercert)
 
-attempts = 0
+    attempts = 0
-updated = False
+    updated = False
 
-# Store it in the IPA LDAP server
+    # Store it in the IPA LDAP server
-while attempts < 10:
+    while attempts < 10:
-    conn = None
+        conn = None
-    tmpdir = None
+        tmpdir = None
-    try:
-        tmpdir = tempfile.mkdtemp(prefix="tmp-")
-        dn = DN(('cn','ipaCert'), ('cn', 'ca_renewal'), ('cn', 'ipa'), ('cn', 'etc'), api.env.basedn)
-        principal = str('host/%s@%s' % (api.env.host, api.env.realm))
-        ccache = ipautil.kinit_hostprincipal('/etc/krb5.keytab', tmpdir, principal)
-        conn = ldap2(shared_instance=False, ldap_uri=api.env.ldap_uri)
-        conn.connect(ccache=ccache)
         try:
-            entry_attrs = conn.get_entry(dn, ['usercertificate'])
+            tmpdir = tempfile.mkdtemp(prefix="tmp-")
-            entry_attrs['usercertificate'] = dercert
+            dn = DN(('cn','ipaCert'), ('cn', 'ca_renewal'), ('cn', 'ipa'), ('cn', 'etc'), api.env.basedn)
-            conn.update_entry(entry_attrs)
+            principal = str('host/%s@%s' % (api.env.host, api.env.realm))
-        except errors.NotFound:
+            ccache = ipautil.kinit_hostprincipal('/etc/krb5.keytab', tmpdir, principal)
-            entry_attrs = conn.make_entry(
+            conn = ldap2(shared_instance=False, ldap_uri=api.env.ldap_uri)
-                dn,
+            conn.connect(ccache=ccache)
-                objectclass=['top', 'pkiuser', 'nscontainer'],
+            try:
-                usercertificate=[dercert])
+                entry_attrs = conn.get_entry(dn, ['usercertificate'])
-            conn.add_entry(entry_attrs)
+                entry_attrs['usercertificate'] = dercert
-        except errors.EmptyModlist:
+                conn.update_entry(entry_attrs)
-            pass
+            except errors.NotFound:
-        updated = True
+                entry_attrs = conn.make_entry(
-        break
+                    dn,
+                    objectclass=['top', 'pkiuser', 'nscontainer'],
+                    usercertificate=[dercert])
+                conn.add_entry(entry_attrs)
+            except errors.EmptyModlist:
+                pass
+            updated = True
+            break
+        except Exception, e:
+            syslog.syslog(syslog.LOG_ERR, 'Updating renewal certificate failed: %s. Sleeping 30s' % e)
+            time.sleep(30)
+            attempts += 1
+        finally:
+            if conn is not None and conn.isconnected():
+                conn.disconnect()
+            if tmpdir is not None:
+                shutil.rmtree(tmpdir)
+
+    if not updated:
+        syslog.syslog(syslog.LOG_ERR, '%s: Giving up. This script may be safely re-executed.' % sys.argv[0])
+        sys.exit(1)
+
+    # Now restart Apache so the new certificate is available
+    try:
+        ipaservices.knownservices.httpd.restart()
     except Exception, e:
-        syslog.syslog(syslog.LOG_ERR, 'Updating renewal certificate failed: %s. Sleeping 30s' % e)
+        syslog.syslog(syslog.LOG_ERR, "Cannot restart httpd: %s" % str(e))
-        time.sleep(30)
-        attempts += 1
-    finally:
-        if conn is not None and conn.isconnected():
-            conn.disconnect()
-        if tmpdir is not None:
-            shutil.rmtree(tmpdir)
 
-if not updated:
-    syslog.syslog(syslog.LOG_ERR, '%s: Giving up. This script may be safely re-executed.' % sys.argv[0])
-    sys.exit(1)
-
-# Now restart Apache so the new certificate is available
 try:
-    ipaservices.knownservices.httpd.restart()
+    main()
-except Exception, e:
+except Exception:
-    syslog.syslog(syslog.LOG_ERR, "Cannot restart httpd: %s" % str(e))
+    syslog.syslog(syslog.LOG_ERR, traceback.format_exc())

install/restart_scripts/restart_dirsrv

@@ -21,20 +21,27 @@
 
 import sys
 import syslog
+import traceback
 from ipapython import services as ipaservices
 from ipalib import api
 
-try:
+def main():
-    instance = sys.argv[1]
+    try:
-except IndexError:
+        instance = sys.argv[1]
-    instance = ""
+    except IndexError:
+        instance = ""
 
-api.bootstrap(context='restart')
+    api.bootstrap(context='restart')
-api.finalize()
+    api.finalize()
 
-syslog.syslog(syslog.LOG_NOTICE, "certmonger restarted dirsrv instance '%s'" % instance)
+    syslog.syslog(syslog.LOG_NOTICE, "certmonger restarted dirsrv instance '%s'" % instance)
+
+    try:
+        ipaservices.knownservices.dirsrv.restart(instance)
+    except Exception, e:
+        syslog.syslog(syslog.LOG_ERR, "Cannot restart dirsrv (instance: '%s'): %s" % (instance, str(e)))
 
 try:
-    ipaservices.knownservices.dirsrv.restart(instance)
+    main()
-except Exception, e:
+except Exception:
-    syslog.syslog(syslog.LOG_ERR, "Cannot restart dirsrv (instance: '%s'): %s" % (instance, str(e)))
+    syslog.syslog(syslog.LOG_ERR, traceback.format_exc())

install/restart_scripts/restart_httpd

@@ -20,11 +20,18 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import syslog
+import traceback
 from ipapython import services as ipaservices
 
-syslog.syslog(syslog.LOG_NOTICE, 'certmonger restarted httpd')
+def main():
+    syslog.syslog(syslog.LOG_NOTICE, 'certmonger restarted httpd')
+
+    try:
+        ipaservices.knownservices.httpd.restart()
+    except Exception, e:
+        syslog.syslog(syslog.LOG_ERR, "Cannot restart httpd: %s" % str(e))
 
 try:
-    ipaservices.knownservices.httpd.restart()
+    main()
-except Exception, e:
+except Exception:
-    syslog.syslog(syslog.LOG_ERR, "Cannot restart httpd: %s" % str(e))
+    syslog.syslog(syslog.LOG_ERR, traceback.format_exc())

install/restart_scripts/restart_pkicad

@@ -21,54 +21,61 @@
 
 import sys
 import syslog
+import traceback
 from ipapython import services as ipaservices
 from ipapython import dogtag
 from ipaserver.install import certs
 from ipalib import api
 
-nickname = sys.argv[1]
+def main():
+    nickname = sys.argv[1]
 
-api.bootstrap(context='restart')
+    api.bootstrap(context='restart')
-api.finalize()
+    api.finalize()
 
-configured_constants = dogtag.configured_constants(api)
+    configured_constants = dogtag.configured_constants(api)
-alias_dir = configured_constants.ALIAS_DIR
+    alias_dir = configured_constants.ALIAS_DIR
-dogtag_service = ipaservices.knownservices[configured_constants.SERVICE_NAME]
+    dogtag_service = ipaservices.knownservices[configured_constants.SERVICE_NAME]
-dogtag_instance = configured_constants.PKI_INSTANCE_NAME
+    dogtag_instance = configured_constants.PKI_INSTANCE_NAME
 
-# dogtag opens its NSS database in read/write mode so we need it
+    # dogtag opens its NSS database in read/write mode so we need it
-# shut down so certmonger can open it read/write mode. This avoids
+    # shut down so certmonger can open it read/write mode. This avoids
-# database corruption. It should already be stopped by the pre-command
+    # database corruption. It should already be stopped by the pre-command
-# but lets be sure.
+    # but lets be sure.
-if dogtag_service.is_running(dogtag_instance):
+    if dogtag_service.is_running(dogtag_instance):
-    syslog.syslog(
+        syslog.syslog(
-        syslog.LOG_NOTICE, "Stopping %s" % dogtag_service.service_name)
+            syslog.LOG_NOTICE, "Stopping %s" % dogtag_service.service_name)
+        try:
+            dogtag_service.stop(dogtag_instance)
+        except Exception, e:
+            syslog.syslog(
+                syslog.LOG_ERR,
+                "Cannot stop %s: %s" % (dogtag_service.service_name, e))
+        else:
+            syslog.syslog(
+                syslog.LOG_NOTICE, "Stopped %s" % dogtag_service.service_name)
+
+    # Fix permissions on the audit cert if we're updating it
+    if nickname == 'auditSigningCert cert-pki-ca':
+        db = certs.CertDB(api.env.realm, nssdir=alias_dir)
+        args = ['-M',
+                '-n', nickname,
+                '-t', 'u,u,Pu',
+               ]
+        db.run_certutil(args)
+
+    syslog.syslog(syslog.LOG_NOTICE, 'Starting %s' % dogtag_service.service_name)
     try:
-        dogtag_service.stop(dogtag_instance)
+        dogtag_service.start(dogtag_instance)
     except Exception, e:
         syslog.syslog(
             syslog.LOG_ERR,
-            "Cannot stop %s: %s" % (dogtag_service.service_name, e))
+            "Cannot start %s: %s" % (dogtag_service.service_name, e))
     else:
         syslog.syslog(
-            syslog.LOG_NOTICE, "Stopped %s" % dogtag_service.service_name)
+            syslog.LOG_NOTICE, "Started %s" % dogtag_service.service_name)
 
-# Fix permissions on the audit cert if we're updating it
-if nickname == 'auditSigningCert cert-pki-ca':
-    db = certs.CertDB(api.env.realm, nssdir=alias_dir)
-    args = ['-M',
-            '-n', nickname,
-            '-t', 'u,u,Pu',
-           ]
-    db.run_certutil(args)
-
-syslog.syslog(syslog.LOG_NOTICE, 'Starting %s' % dogtag_service.service_name)
 try:
-    dogtag_service.start(dogtag_instance)
+    main()
-except Exception, e:
+except Exception:
-    syslog.syslog(
+    syslog.syslog(syslog.LOG_ERR, traceback.format_exc())
-        syslog.LOG_ERR,
-        "Cannot start %s: %s" % (dogtag_service.service_name, e))
-else:
-    syslog.syslog(
-        syslog.LOG_NOTICE, "Started %s" % dogtag_service.service_name)

install/restart_scripts/stop_pkicad

@@ -21,23 +21,30 @@
 
 import sys
 import syslog
+import traceback
 from ipapython import services as ipaservices
 from ipapython import dogtag
 from ipalib import api
 
-api.bootstrap(context='restart')
+def main():
-api.finalize()
+    api.bootstrap(context='restart')
+    api.finalize()
 
-configured_constants = dogtag.configured_constants(api)
+    configured_constants = dogtag.configured_constants(api)
-dogtag_service = ipaservices.knownservices[configured_constants.SERVICE_NAME]
+    dogtag_service = ipaservices.knownservices[configured_constants.SERVICE_NAME]
-dogtag_instance = configured_constants.PKI_INSTANCE_NAME
+    dogtag_instance = configured_constants.PKI_INSTANCE_NAME
+
+    syslog.syslog(syslog.LOG_NOTICE, "Stopping %s" % dogtag_service.service_name)
+    try:
+        dogtag_service.stop(dogtag_instance)
+    except Exception, e:
+        syslog.syslog(
+            syslog.LOG_ERR, "Cannot stop %s: %s" % (dogtag_service.service_name, e))
+    else:
+        syslog.syslog(
+            syslog.LOG_NOTICE, "Stopped %s" % dogtag_service.service_name)
 
-syslog.syslog(syslog.LOG_NOTICE, "Stopping %s" % dogtag_service.service_name)
 try:
-    dogtag_service.stop(dogtag_instance)
+    main()
-except Exception, e:
+except Exception:
-    syslog.syslog(
+    syslog.syslog(syslog.LOG_ERR, traceback.format_exc())
-        syslog.LOG_ERR, "Cannot stop %s: %s" % (dogtag_service.service_name, e))
-else:
-    syslog.syslog(
-        syslog.LOG_NOTICE, "Stopped %s" % dogtag_service.service_name)