From 8e986904096925fc08df8cbdf271d722314c5460 Mon Sep 17 00:00:00 2001 From: Jan Cholasta Date: Thu, 23 Jan 2014 15:33:26 +0100 Subject: [PATCH] Log unhandled exceptions in certificate renewal scripts. https://fedorahosted.org/freeipa/ticket/4093 Reviewed-By: Petr Viktorin --- .../dogtag-ipa-retrieve-agent-submit | 79 ++++++---- install/restart_scripts/renew_ca_cert | 149 +++++++++--------- install/restart_scripts/renew_ra_cert | 105 ++++++------ install/restart_scripts/restart_dirsrv | 27 ++-- install/restart_scripts/restart_httpd | 15 +- install/restart_scripts/restart_pkicad | 77 +++++---- install/restart_scripts/stop_pkicad | 33 ++-- 7 files changed, 268 insertions(+), 217 deletions(-) diff --git a/install/certmonger/dogtag-ipa-retrieve-agent-submit b/install/certmonger/dogtag-ipa-retrieve-agent-submit index 726790197..1422494cc 100644 --- a/install/certmonger/dogtag-ipa-retrieve-agent-submit +++ b/install/certmonger/dogtag-ipa-retrieve-agent-submit @@ -31,6 +31,8 @@ import sys import shutil import tempfile import syslog +import base64 +import traceback from ipalib import api from ipapython.dn import DN from ipalib import errors @@ -39,45 +41,52 @@ from ipapython import services as ipaservices from ipapython import ipautil from ipaserver.install import certs from ipaserver.plugins.ldap2 import ldap2 -import base64 -# We cheat and pass in the nickname as the CA profile to execute against. -# Some way is needed to determine which entry to retrieve from LDAP -operation = os.environ.get('CERTMONGER_OPERATION') -nickname = os.environ.get('CERTMONGER_CA_PROFILE') +def main(): + # We cheat and pass in the nickname as the CA profile to execute against. + # Some way is needed to determine which entry to retrieve from LDAP + operation = os.environ.get('CERTMONGER_OPERATION') + nickname = os.environ.get('CERTMONGER_CA_PROFILE') -if operation not in ['SUBMIT', 'POLL']: - sys.exit(6) # unsupported operation + if operation not in ['SUBMIT', 'POLL']: + sys.exit(6) # unsupported operation -api.bootstrap(context='renew') -api.finalize() + api.bootstrap(context='renew') + api.finalize() -# Update or add it -tmpdir = tempfile.mkdtemp(prefix = "tmp-") -try: - dn = DN(('cn', nickname), ('cn', 'ca_renewal'), ('cn', 'ipa'), ('cn', 'etc'), api.env.basedn) - principal = str('host/%s@%s' % (api.env.host, api.env.realm)) - ccache = ipautil.kinit_hostprincipal('/etc/krb5.keytab', tmpdir, principal) - conn = ldap2(shared_instance=False, ldap_uri=api.env.ldap_uri) - conn.connect(ccache=ccache) + # Update or add it + tmpdir = tempfile.mkdtemp(prefix = "tmp-") try: - syslog.syslog(syslog.LOG_NOTICE, "Updating certificate for %s" % nickname) - entry_attrs = conn.get_entry(dn, ['usercertificate']) - cert = entry_attrs['usercertificate'][0] - cert = base64.b64encode(cert) - print x509.make_pem(cert) - except errors.NotFound: - syslog.syslog(syslog.LOG_INFO, "Updated certificate for %s not available" % nickname) - # No cert available yet, tell certmonger to wait another 8 hours - print 8 * 60 * 60 - sys.exit(5) + dn = DN(('cn', nickname), ('cn', 'ca_renewal'), ('cn', 'ipa'), ('cn', 'etc'), api.env.basedn) + principal = str('host/%s@%s' % (api.env.host, api.env.realm)) + ccache = ipautil.kinit_hostprincipal('/etc/krb5.keytab', tmpdir, principal) + conn = ldap2(shared_instance=False, ldap_uri=api.env.ldap_uri) + conn.connect(ccache=ccache) + try: + syslog.syslog(syslog.LOG_NOTICE, "Updating certificate for %s" % nickname) + entry_attrs = conn.get_entry(dn, ['usercertificate']) + cert = entry_attrs['usercertificate'][0] + cert = base64.b64encode(cert) + print x509.make_pem(cert) + except errors.NotFound: + syslog.syslog(syslog.LOG_INFO, "Updated certificate for %s not available" % nickname) + # No cert available yet, tell certmonger to wait another 8 hours + print 8 * 60 * 60 + sys.exit(5) + finally: + conn.disconnect() + except Exception, e: + syslog.syslog(syslog.LOG_ERR, "Exception trying to retrieve %s: %s" % (nickname, e)) + # Unhandled error + sys.exit(3) finally: - conn.disconnect() -except Exception, e: - syslog.syslog(syslog.LOG_ERR, "Exception trying to retrieve %s: %s" % (nickname, e)) - # Unhandled error - sys.exit(3) -finally: - shutil.rmtree(tmpdir) + shutil.rmtree(tmpdir) -sys.exit(0) + sys.exit(0) + +try: + main() +except Exception: + syslog.syslog(syslog.LOG_ERR, traceback.format_exc()) + print "Internal error" + sys.exit(3) diff --git a/install/restart_scripts/renew_ca_cert b/install/restart_scripts/renew_ca_cert index 09acfc236..2ae869db9 100644 --- a/install/restart_scripts/renew_ca_cert +++ b/install/restart_scripts/renew_ca_cert @@ -26,6 +26,7 @@ import tempfile import syslog import random import time +import traceback from ipalib import api from ipapython.dn import DN from ipalib import errors @@ -42,80 +43,86 @@ from ipapython import certmonger # has renewed a CA subsystem certificate a copy is put into the replicated # tree so it can be shared with the other IPA servers. -nickname = sys.argv[1] +def main(): + nickname = sys.argv[1] -api.bootstrap(context='restart') -api.finalize() + api.bootstrap(context='restart') + api.finalize() -configured_constants = dogtag.configured_constants(api) -alias_dir = configured_constants.ALIAS_DIR -dogtag_service = ipaservices.knownservices[configured_constants.SERVICE_NAME] -dogtag_instance = configured_constants.PKI_INSTANCE_NAME + configured_constants = dogtag.configured_constants(api) + alias_dir = configured_constants.ALIAS_DIR + dogtag_service = ipaservices.knownservices[configured_constants.SERVICE_NAME] + dogtag_instance = configured_constants.PKI_INSTANCE_NAME -# Fetch the new certificate -db = certs.CertDB(api.env.realm, nssdir=alias_dir) -cert = db.get_cert_from_db(nickname, pem=False) - -if not cert: - syslog.syslog(syslog.LOG_ERR, 'No certificate %s found.' % nickname) - sys.exit(1) - -# Update or add it -tmpdir = tempfile.mkdtemp(prefix = "tmp-") -try: - dn = DN(('cn',nickname), ('cn', 'ca_renewal'), ('cn', 'ipa'), ('cn', 'etc'), api.env.basedn) - principal = str('host/%s@%s' % (api.env.host, api.env.realm)) - ccache = ipautil.kinit_hostprincipal('/etc/krb5.keytab', tmpdir, principal) - conn = ldap2(shared_instance=False, ldap_uri=api.env.ldap_uri) - conn.connect(ccache=ccache) - try: - entry_attrs = conn.get_entry(dn, ['usercertificate']) - entry_attrs['usercertificate'] = cert - conn.update_entry(entry_attrs) - except errors.NotFound: - entry_attrs = conn.make_entry( - dn, - objectclass=['top', 'pkiuser', 'nscontainer'], - usercertificate=[cert]) - conn.add_entry(entry_attrs) - except errors.EmptyModlist: - pass - conn.disconnect() -except Exception, e: - syslog.syslog(syslog.LOG_ERR, 'Updating renewal certificate failed: %s' % e) -finally: - shutil.rmtree(tmpdir) - -# Done withing stopped_service context, CA restarted here -update_cert_config(nickname, cert) - -if nickname == 'subsystemCert cert-pki-ca': - update_people_entry('pkidbuser', cert) - -if nickname == 'auditSigningCert cert-pki-ca': - # Fix trust on the audit cert + # Fetch the new certificate db = certs.CertDB(api.env.realm, nssdir=alias_dir) - args = ['-M', - '-n', nickname, - '-t', 'u,u,Pu', - ] - try: - db.run_certutil(args) - syslog.syslog(syslog.LOG_NOTICE, 'Updated trust on certificate %s in %s' % (nickname, db.secdir)) - except ipautil.CalledProcessError: - syslog.syslog(syslog.LOG_ERR, 'Updating trust on certificate %s failed in %s' % (nickname, db.secdir)) + cert = db.get_cert_from_db(nickname, pem=False) + + if not cert: + syslog.syslog(syslog.LOG_ERR, 'No certificate %s found.' % nickname) + sys.exit(1) + + # Update or add it + tmpdir = tempfile.mkdtemp(prefix = "tmp-") + try: + dn = DN(('cn',nickname), ('cn', 'ca_renewal'), ('cn', 'ipa'), ('cn', 'etc'), api.env.basedn) + principal = str('host/%s@%s' % (api.env.host, api.env.realm)) + ccache = ipautil.kinit_hostprincipal('/etc/krb5.keytab', tmpdir, principal) + conn = ldap2(shared_instance=False, ldap_uri=api.env.ldap_uri) + conn.connect(ccache=ccache) + try: + entry_attrs = conn.get_entry(dn, ['usercertificate']) + entry_attrs['usercertificate'] = cert + conn.update_entry(entry_attrs) + except errors.NotFound: + entry_attrs = conn.make_entry( + dn, + objectclass=['top', 'pkiuser', 'nscontainer'], + usercertificate=[cert]) + conn.add_entry(entry_attrs) + except errors.EmptyModlist: + pass + conn.disconnect() + except Exception, e: + syslog.syslog(syslog.LOG_ERR, 'Updating renewal certificate failed: %s' % e) + finally: + shutil.rmtree(tmpdir) + + # Done withing stopped_service context, CA restarted here + update_cert_config(nickname, cert) + + if nickname == 'subsystemCert cert-pki-ca': + update_people_entry('pkidbuser', cert) + + if nickname == 'auditSigningCert cert-pki-ca': + # Fix trust on the audit cert + db = certs.CertDB(api.env.realm, nssdir=alias_dir) + args = ['-M', + '-n', nickname, + '-t', 'u,u,Pu', + ] + try: + db.run_certutil(args) + syslog.syslog(syslog.LOG_NOTICE, 'Updated trust on certificate %s in %s' % (nickname, db.secdir)) + except ipautil.CalledProcessError: + syslog.syslog(syslog.LOG_ERR, 'Updating trust on certificate %s failed in %s' % (nickname, db.secdir)) + + # Now we can start the CA. Using the ipaservices start should fire + # off the servlet to verify that the CA is actually up and responding so + # when this returns it should be good-to-go. The CA was stopped in the + # pre-save state. + syslog.syslog(syslog.LOG_NOTICE, 'Starting %s' % dogtag_service.service_name) + try: + dogtag_service.start(dogtag_instance) + except Exception, e: + syslog.syslog( + syslog.LOG_ERR, + "Cannot start %s: %s" % (dogtag_service.service_name, e)) + else: + syslog.syslog( + syslog.LOG_NOTICE, "Started %s" % dogtag_service.service_name) -# Now we can start the CA. Using the ipaservices start should fire -# off the servlet to verify that the CA is actually up and responding so -# when this returns it should be good-to-go. The CA was stopped in the -# pre-save state. -syslog.syslog(syslog.LOG_NOTICE, 'Starting %s' % dogtag_service.service_name) try: - dogtag_service.start(dogtag_instance) -except Exception, e: - syslog.syslog( - syslog.LOG_ERR, - "Cannot start %s: %s" % (dogtag_service.service_name, e)) -else: - syslog.syslog( - syslog.LOG_NOTICE, "Started %s" % dogtag_service.service_name) + main() +except Exception: + syslog.syslog(syslog.LOG_ERR, traceback.format_exc()) diff --git a/install/restart_scripts/renew_ra_cert b/install/restart_scripts/renew_ra_cert index cb3e3683b..0d731144b 100644 --- a/install/restart_scripts/renew_ra_cert +++ b/install/restart_scripts/renew_ra_cert @@ -24,6 +24,7 @@ import shutil import tempfile import syslog import time +import traceback from ipapython import services as ipaservices from ipapython import ipautil from ipaserver.install import certs @@ -33,60 +34,66 @@ from ipapython.dn import DN from ipalib import errors from ipaserver.plugins.ldap2 import ldap2 -api.bootstrap(context='restart') -api.finalize() +def main(): + api.bootstrap(context='restart') + api.finalize() -# Fetch the new certificate -db = certs.CertDB(api.env.realm) -dercert = db.get_cert_from_db('ipaCert', pem=False) + # Fetch the new certificate + db = certs.CertDB(api.env.realm) + dercert = db.get_cert_from_db('ipaCert', pem=False) -# Load it into dogtag -update_people_entry('ipara', dercert) + # Load it into dogtag + update_people_entry('ipara', dercert) -attempts = 0 -updated = False + attempts = 0 + updated = False -# Store it in the IPA LDAP server -while attempts < 10: - conn = None - tmpdir = None - try: - tmpdir = tempfile.mkdtemp(prefix="tmp-") - dn = DN(('cn','ipaCert'), ('cn', 'ca_renewal'), ('cn', 'ipa'), ('cn', 'etc'), api.env.basedn) - principal = str('host/%s@%s' % (api.env.host, api.env.realm)) - ccache = ipautil.kinit_hostprincipal('/etc/krb5.keytab', tmpdir, principal) - conn = ldap2(shared_instance=False, ldap_uri=api.env.ldap_uri) - conn.connect(ccache=ccache) + # Store it in the IPA LDAP server + while attempts < 10: + conn = None + tmpdir = None try: - entry_attrs = conn.get_entry(dn, ['usercertificate']) - entry_attrs['usercertificate'] = dercert - conn.update_entry(entry_attrs) - except errors.NotFound: - entry_attrs = conn.make_entry( - dn, - objectclass=['top', 'pkiuser', 'nscontainer'], - usercertificate=[dercert]) - conn.add_entry(entry_attrs) - except errors.EmptyModlist: - pass - updated = True - break + tmpdir = tempfile.mkdtemp(prefix="tmp-") + dn = DN(('cn','ipaCert'), ('cn', 'ca_renewal'), ('cn', 'ipa'), ('cn', 'etc'), api.env.basedn) + principal = str('host/%s@%s' % (api.env.host, api.env.realm)) + ccache = ipautil.kinit_hostprincipal('/etc/krb5.keytab', tmpdir, principal) + conn = ldap2(shared_instance=False, ldap_uri=api.env.ldap_uri) + conn.connect(ccache=ccache) + try: + entry_attrs = conn.get_entry(dn, ['usercertificate']) + entry_attrs['usercertificate'] = dercert + conn.update_entry(entry_attrs) + except errors.NotFound: + entry_attrs = conn.make_entry( + dn, + objectclass=['top', 'pkiuser', 'nscontainer'], + usercertificate=[dercert]) + conn.add_entry(entry_attrs) + except errors.EmptyModlist: + pass + updated = True + break + except Exception, e: + syslog.syslog(syslog.LOG_ERR, 'Updating renewal certificate failed: %s. Sleeping 30s' % e) + time.sleep(30) + attempts += 1 + finally: + if conn is not None and conn.isconnected(): + conn.disconnect() + if tmpdir is not None: + shutil.rmtree(tmpdir) + + if not updated: + syslog.syslog(syslog.LOG_ERR, '%s: Giving up. This script may be safely re-executed.' % sys.argv[0]) + sys.exit(1) + + # Now restart Apache so the new certificate is available + try: + ipaservices.knownservices.httpd.restart() except Exception, e: - syslog.syslog(syslog.LOG_ERR, 'Updating renewal certificate failed: %s. Sleeping 30s' % e) - time.sleep(30) - attempts += 1 - finally: - if conn is not None and conn.isconnected(): - conn.disconnect() - if tmpdir is not None: - shutil.rmtree(tmpdir) + syslog.syslog(syslog.LOG_ERR, "Cannot restart httpd: %s" % str(e)) -if not updated: - syslog.syslog(syslog.LOG_ERR, '%s: Giving up. This script may be safely re-executed.' % sys.argv[0]) - sys.exit(1) - -# Now restart Apache so the new certificate is available try: - ipaservices.knownservices.httpd.restart() -except Exception, e: - syslog.syslog(syslog.LOG_ERR, "Cannot restart httpd: %s" % str(e)) + main() +except Exception: + syslog.syslog(syslog.LOG_ERR, traceback.format_exc()) diff --git a/install/restart_scripts/restart_dirsrv b/install/restart_scripts/restart_dirsrv index 9b22d08c1..03e6a5c42 100644 --- a/install/restart_scripts/restart_dirsrv +++ b/install/restart_scripts/restart_dirsrv @@ -21,20 +21,27 @@ import sys import syslog +import traceback from ipapython import services as ipaservices from ipalib import api -try: - instance = sys.argv[1] -except IndexError: - instance = "" +def main(): + try: + instance = sys.argv[1] + except IndexError: + instance = "" -api.bootstrap(context='restart') -api.finalize() + api.bootstrap(context='restart') + api.finalize() -syslog.syslog(syslog.LOG_NOTICE, "certmonger restarted dirsrv instance '%s'" % instance) + syslog.syslog(syslog.LOG_NOTICE, "certmonger restarted dirsrv instance '%s'" % instance) + + try: + ipaservices.knownservices.dirsrv.restart(instance) + except Exception, e: + syslog.syslog(syslog.LOG_ERR, "Cannot restart dirsrv (instance: '%s'): %s" % (instance, str(e))) try: - ipaservices.knownservices.dirsrv.restart(instance) -except Exception, e: - syslog.syslog(syslog.LOG_ERR, "Cannot restart dirsrv (instance: '%s'): %s" % (instance, str(e))) + main() +except Exception: + syslog.syslog(syslog.LOG_ERR, traceback.format_exc()) diff --git a/install/restart_scripts/restart_httpd b/install/restart_scripts/restart_httpd index 16a41ee5b..98337bc85 100644 --- a/install/restart_scripts/restart_httpd +++ b/install/restart_scripts/restart_httpd @@ -20,11 +20,18 @@ # along with this program. If not, see . import syslog +import traceback from ipapython import services as ipaservices -syslog.syslog(syslog.LOG_NOTICE, 'certmonger restarted httpd') +def main(): + syslog.syslog(syslog.LOG_NOTICE, 'certmonger restarted httpd') + + try: + ipaservices.knownservices.httpd.restart() + except Exception, e: + syslog.syslog(syslog.LOG_ERR, "Cannot restart httpd: %s" % str(e)) try: - ipaservices.knownservices.httpd.restart() -except Exception, e: - syslog.syslog(syslog.LOG_ERR, "Cannot restart httpd: %s" % str(e)) + main() +except Exception: + syslog.syslog(syslog.LOG_ERR, traceback.format_exc()) diff --git a/install/restart_scripts/restart_pkicad b/install/restart_scripts/restart_pkicad index 9a3d48057..4e14577ae 100644 --- a/install/restart_scripts/restart_pkicad +++ b/install/restart_scripts/restart_pkicad @@ -21,54 +21,61 @@ import sys import syslog +import traceback from ipapython import services as ipaservices from ipapython import dogtag from ipaserver.install import certs from ipalib import api -nickname = sys.argv[1] +def main(): + nickname = sys.argv[1] -api.bootstrap(context='restart') -api.finalize() + api.bootstrap(context='restart') + api.finalize() -configured_constants = dogtag.configured_constants(api) -alias_dir = configured_constants.ALIAS_DIR -dogtag_service = ipaservices.knownservices[configured_constants.SERVICE_NAME] -dogtag_instance = configured_constants.PKI_INSTANCE_NAME + configured_constants = dogtag.configured_constants(api) + alias_dir = configured_constants.ALIAS_DIR + dogtag_service = ipaservices.knownservices[configured_constants.SERVICE_NAME] + dogtag_instance = configured_constants.PKI_INSTANCE_NAME -# dogtag opens its NSS database in read/write mode so we need it -# shut down so certmonger can open it read/write mode. This avoids -# database corruption. It should already be stopped by the pre-command -# but lets be sure. -if dogtag_service.is_running(dogtag_instance): - syslog.syslog( - syslog.LOG_NOTICE, "Stopping %s" % dogtag_service.service_name) + # dogtag opens its NSS database in read/write mode so we need it + # shut down so certmonger can open it read/write mode. This avoids + # database corruption. It should already be stopped by the pre-command + # but lets be sure. + if dogtag_service.is_running(dogtag_instance): + syslog.syslog( + syslog.LOG_NOTICE, "Stopping %s" % dogtag_service.service_name) + try: + dogtag_service.stop(dogtag_instance) + except Exception, e: + syslog.syslog( + syslog.LOG_ERR, + "Cannot stop %s: %s" % (dogtag_service.service_name, e)) + else: + syslog.syslog( + syslog.LOG_NOTICE, "Stopped %s" % dogtag_service.service_name) + + # Fix permissions on the audit cert if we're updating it + if nickname == 'auditSigningCert cert-pki-ca': + db = certs.CertDB(api.env.realm, nssdir=alias_dir) + args = ['-M', + '-n', nickname, + '-t', 'u,u,Pu', + ] + db.run_certutil(args) + + syslog.syslog(syslog.LOG_NOTICE, 'Starting %s' % dogtag_service.service_name) try: - dogtag_service.stop(dogtag_instance) + dogtag_service.start(dogtag_instance) except Exception, e: syslog.syslog( syslog.LOG_ERR, - "Cannot stop %s: %s" % (dogtag_service.service_name, e)) + "Cannot start %s: %s" % (dogtag_service.service_name, e)) else: syslog.syslog( - syslog.LOG_NOTICE, "Stopped %s" % dogtag_service.service_name) + syslog.LOG_NOTICE, "Started %s" % dogtag_service.service_name) -# Fix permissions on the audit cert if we're updating it -if nickname == 'auditSigningCert cert-pki-ca': - db = certs.CertDB(api.env.realm, nssdir=alias_dir) - args = ['-M', - '-n', nickname, - '-t', 'u,u,Pu', - ] - db.run_certutil(args) - -syslog.syslog(syslog.LOG_NOTICE, 'Starting %s' % dogtag_service.service_name) try: - dogtag_service.start(dogtag_instance) -except Exception, e: - syslog.syslog( - syslog.LOG_ERR, - "Cannot start %s: %s" % (dogtag_service.service_name, e)) -else: - syslog.syslog( - syslog.LOG_NOTICE, "Started %s" % dogtag_service.service_name) + main() +except Exception: + syslog.syslog(syslog.LOG_ERR, traceback.format_exc()) diff --git a/install/restart_scripts/stop_pkicad b/install/restart_scripts/stop_pkicad index c275eaee4..e58e04401 100644 --- a/install/restart_scripts/stop_pkicad +++ b/install/restart_scripts/stop_pkicad @@ -21,23 +21,30 @@ import sys import syslog +import traceback from ipapython import services as ipaservices from ipapython import dogtag from ipalib import api -api.bootstrap(context='restart') -api.finalize() +def main(): + api.bootstrap(context='restart') + api.finalize() -configured_constants = dogtag.configured_constants(api) -dogtag_service = ipaservices.knownservices[configured_constants.SERVICE_NAME] -dogtag_instance = configured_constants.PKI_INSTANCE_NAME + configured_constants = dogtag.configured_constants(api) + dogtag_service = ipaservices.knownservices[configured_constants.SERVICE_NAME] + dogtag_instance = configured_constants.PKI_INSTANCE_NAME + + syslog.syslog(syslog.LOG_NOTICE, "Stopping %s" % dogtag_service.service_name) + try: + dogtag_service.stop(dogtag_instance) + except Exception, e: + syslog.syslog( + syslog.LOG_ERR, "Cannot stop %s: %s" % (dogtag_service.service_name, e)) + else: + syslog.syslog( + syslog.LOG_NOTICE, "Stopped %s" % dogtag_service.service_name) -syslog.syslog(syslog.LOG_NOTICE, "Stopping %s" % dogtag_service.service_name) try: - dogtag_service.stop(dogtag_instance) -except Exception, e: - syslog.syslog( - syslog.LOG_ERR, "Cannot stop %s: %s" % (dogtag_service.service_name, e)) -else: - syslog.syslog( - syslog.LOG_NOTICE, "Stopped %s" % dogtag_service.service_name) + main() +except Exception: + syslog.syslog(syslog.LOG_ERR, traceback.format_exc())