Use new certmonger locking to prevent NSS database corruption.

dogtag opens its NSS database in read/write mode so we need to be very
careful during renewal that we don't also open it up read/write. We
basically need to serialize access to the database. certmonger does the
majority of this work via internal locking from the point where it generates
a new key/submits a rewewal through the pre_save and releases the lock after
the post_save command. This lock is held per NSS database so we're save
from certmonger. dogtag needs to be shutdown in the pre_save state so
certmonger can safely add the certificate and we can manipulate trust
in the post_save command.

Fix a number of bugs in renewal. The CA wasn't actually being restarted
at all due to a naming change upstream. In python we need to reference
services using python-ish names but the service is pki-cad. We need a
translation for non-Fedora systems as well.

Update the CA ou=People entry when he CA subsystem certificate is
renewed. This certificate is used as an identity certificate to bind
to the DS instance.

https://fedorahosted.org/freeipa/ticket/3292
https://fedorahosted.org/freeipa/ticket/3322
This commit is contained in:
Rob Crittenden 2014-12-02 13:18:36 -05:00
parent b382a77fc3
commit 045b6e6ed9
9 changed files with 274 additions and 100 deletions

View File

@ -112,7 +112,7 @@ Requires: python-memcached
Requires: systemd-units >= 36-3
Requires(pre): systemd-units
Requires(post): systemd-units
Requires: selinux-policy >= 3.11.1-60
Requires: selinux-policy >= 3.11.1-71
Requires(post): selinux-policy-base
Requires: slapi-nis >= 0.44
Requires: pki-ca >= 10.0.0-0.54.b3
@ -769,6 +769,12 @@ fi
%ghost %attr(0644,root,apache) %config(noreplace) %{_sysconfdir}/ipa/ca.crt
%changelog
* Tue Jan 29 2013 Rob Crittenden <rcritten@redhat.com> - 3.0.99-13
- Set certmonger minimum version to 0.65 for NSS locking during
renewal
- Set selinux-policy to 3.11.1-73 so certmonger can run in post
scriptlet
* Thu Jan 24 2013 Rob Crittenden <rcritten@redhat.com> - 3.0.99-12
- Add certmonger condrestart to server post scriptlet
- Make certmonger a (pre) Requires on the server subpackage

View File

@ -7,6 +7,7 @@ app_DATA = \
restart_pkicad \
renew_ca_cert \
renew_ra_cert \
stop_pkicad \
$(NULL)
EXTRA_DIST = \

View File

@ -34,8 +34,10 @@ from ipapython import services as ipaservices
from ipapython import ipautil
from ipapython import dogtag
from ipaserver.install import certs
from ipaserver.install.cainstance import update_people_entry
from ipaserver.plugins.ldap2 import ldap2
from ipaserver.install.cainstance import update_cert_config
from ipapython import certmonger
# This script a post-cert-install command for certmonger. When certmonger
# has renewed a CA subsystem certificate a copy is put into the replicated
@ -82,8 +84,13 @@ except Exception, e:
finally:
shutil.rmtree(tmpdir)
# Fix permissions on the audit cert if we're updating it
update_cert_config(nickname, cert)
if nickname == 'subsystemCert cert-pki-ca':
update_people_entry('pkidbuser', cert)
if nickname == 'auditSigningCert cert-pki-ca':
# Fix trust on the audit cert
db = certs.CertDB(api.env.realm, nssdir=alias_dir)
args = ['-M',
'-n', nickname,
@ -91,25 +98,20 @@ if nickname == 'auditSigningCert cert-pki-ca':
]
try:
db.run_certutil(args)
syslog.syslog(syslog.LOG_NOTICE, 'Updated trust on certificate %s in %s' % (nickname, db.secdir))
except ipautil.CalledProcessError:
syslog.syslog(syslog.LOG_ERR, 'Updating trust on certificate %s failed in %s' % (nickname, db.secdir))
syslog.syslog(syslog.LOG_ERR, 'Updating trust on certificate %s failed in %s' % (nickname, db.secdir))
update_cert_config(nickname, cert)
syslog.syslog(
syslog.LOG_NOTICE, 'certmonger restarted %sd instance %s to renew %s' %
(dogtag_instance, dogtag_instance, nickname))
# We monitor 3 certs that are all likely to be renewed by certmonger more or
# less at the same time. Each cert renewal is going to need to restart
# the CA. Add a bit of randomness in this so not all three try to start it
# at the same time. A restart is needed for each because there is no guarantee
# that they will all be renewed at the same time.
pause = random.randint(10,360)
syslog.syslog(syslog.LOG_NOTICE, 'Pausing %d seconds to restart pki-ca' % pause)
time.sleep(pause)
# Now we can start the CA. Using the ipaservices start should fire
# off the servlet to verify that the CA is actually up and responding so
# when this returns it should be good-to-go. The CA was stopped in the
# pre-save state.
syslog.syslog(syslog.LOG_NOTICE, 'Starting %sd' % dogtag_instance)
try:
ipaservices.knownservices.pki_cad.restart(dogtag_instance)
if configured_constants.DOGTAG_VERSION == 9:
ipaservices.knownservices.pki_cad.start(dogtag_instance)
else:
ipaservices.knownservices.pki_tomcatd.start(dogtag_instance)
except Exception, e:
syslog.syslog(syslog.LOG_ERR, "Cannot restart %sd: %s" %
syslog.syslog(syslog.LOG_ERR, "Cannot start %sd: %s" %
(dogtag_instance, str(e)))

View File

@ -25,13 +25,11 @@ import tempfile
import syslog
import time
from ipapython import services as ipaservices
from ipapython.certmonger import get_pin
from ipapython import ipautil
from ipaserver.install import certs
from ipaserver.install.cainstance import DEFAULT_DSPORT
from ipaserver.install.cainstance import update_people_entry
from ipalib import api
from ipapython.dn import DN
from ipalib import x509
from ipalib import errors
from ipaserver.plugins.ldap2 import ldap2
import ldap as _ldap
@ -41,52 +39,10 @@ api.finalize()
# Fetch the new certificate
db = certs.CertDB(api.env.realm)
cert = db.get_cert_from_db('ipaCert', pem=False)
serial_number = x509.get_serial_number(cert, datatype=x509.DER)
subject = x509.get_subject(cert, datatype=x509.DER)
issuer = x509.get_issuer(cert, datatype=x509.DER)
dercert = db.get_cert_from_db('ipaCert', pem=False)
# Load it into dogtag
dn = DN(('uid','ipara'),('ou','People'),('o','ipaca'))
try:
dm_password = get_pin('internaldb')
except IOError, e:
syslog.syslog(syslog.LOG_ERR, 'Unable to determine PIN for CA instance: %s' % e)
sys.exit(1)
attempts = 0
dogtag_uri='ldap://localhost:%d' % DEFAULT_DSPORT
updated = False
while attempts < 10:
conn = None
try:
conn = ldap2(shared_instance=False, ldap_uri=dogtag_uri)
conn.connect(bind_dn=DN(('cn', 'directory manager')), bind_pw=dm_password)
(entry_dn, entry_attrs) = conn.get_entry(dn, ['usercertificate'], normalize=False)
entry_attrs['usercertificate'].append(cert)
entry_attrs['description'] = '2;%d;%s;%s' % (serial_number, issuer, subject)
conn.update_entry(dn, entry_attrs, normalize=False)
updated = True
break
except errors.NetworkError:
syslog.syslog(syslog.LOG_ERR, 'Connection to %s failed, sleeping 30s' % dogtag_uri)
time.sleep(30)
attempts += 1
except errors.EmptyModlist:
updated = True
break
except Exception, e:
syslog.syslog(syslog.LOG_ERR, 'Updating agent entry failed: %s' % e)
break
finally:
if conn.isconnected():
conn.disconnect()
if not updated:
syslog.syslog(syslog.LOG_ERR, '%s: Giving up. This script may be safely re-executed.' % sys.argv[0])
sys.exit(1)
update_people_entry('ipara', dercert)
attempts = 0
updated = False
@ -104,11 +60,11 @@ while attempts < 10:
conn.connect(ccache=ccache)
try:
(entry_dn, entry_attrs) = conn.get_entry(dn, ['usercertificate'])
entry_attrs['usercertificate'] = cert
entry_attrs['usercertificate'] = dercert
conn.update_entry(dn, entry_attrs, normalize=False)
except errors.NotFound:
entry_attrs = dict(objectclass=['top', 'pkiuser', 'nscontainer'],
usercertificate=cert)
usercertificate=dercert)
conn.add_entry(dn, entry_attrs, normalize=False)
except errors.EmptyModlist:
pass

View File

@ -35,8 +35,16 @@ configured_constants = dogtag.configured_constants(api)
alias_dir = configured_constants.ALIAS_DIR
dogtag_instance = configured_constants.PKI_INSTANCE_NAME
syslog.syslog(syslog.LOG_NOTICE, "certmonger restarted %sd, nickname '%s'" %
(dogtag_instance, nickname))
# dogtag opens its NSS database in read/write mode so we need it
# shut down so certmonger can open it read/write mode. This avoids
# database corruption. It should already be stopped by the pre-command
# but lets be sure.
if ipaservices.knownservices.pki_cad.is_running(dogtag_instance):
try:
ipaservices.knownservices.pki_cad.stop(dogtag_instance)
except Exception, e:
syslog.syslog(syslog.LOG_ERR, "Cannot stop %sd: %s" %
(dogtag_instance, str(e)))
# Fix permissions on the audit cert if we're updating it
if nickname == 'auditSigningCert cert-pki-ca':
@ -48,10 +56,13 @@ if nickname == 'auditSigningCert cert-pki-ca':
db.run_certutil(args)
try:
# I've seen times where systemd restart does not actually restart
# the process. A full stop/start is required. This works around that
ipaservices.knownservices.pki_cad.stop(dogtag_instance)
ipaservices.knownservices.pki_cad.start(dogtag_instance)
if configured_constants.DOGTAG_VERSION == 9:
ipaservices.knownservices.pki_cad.start(dogtag_instance)
else:
ipaservices.knownservices.pki_tomcatd.start(dogtag_instance)
except Exception, e:
syslog.syslog(syslog.LOG_ERR, "Cannot restart %sd: %s" %
syslog.syslog(syslog.LOG_ERR, "Cannot start %sd: %s" %
(dogtag_instance, str(e)))
else:
syslog.syslog(syslog.LOG_NOTICE, "certmonger started %sd, nickname '%s'" %
(dogtag_instance, nickname))

View File

@ -0,0 +1,43 @@
#!/usr/bin/python -E
#
# Authors:
# Rob Crittenden <rcritten@redhat.com>
#
# Copyright (C) 2012 Red Hat
# see file 'COPYING' for use and warranty information
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import sys
import syslog
from ipapython import services as ipaservices
from ipapython import dogtag
from ipalib import api
api.bootstrap(context='restart')
api.finalize()
configured_constants = dogtag.configured_constants(api)
dogtag_instance = configured_constants.PKI_INSTANCE_NAME
syslog.syslog(syslog.LOG_NOTICE, "certmonger stopping %sd" % dogtag_instance)
try:
if configured_constants.DOGTAG_VERSION == 9:
ipaservices.knownservices.pki_cad.start(dogtag_instance)
else:
ipaservices.knownservices.pki_tomcatd.start(dogtag_instance)
except Exception, e:
syslog.syslog(syslog.LOG_ERR, "Cannot stop %sd: %s" %
(dogtag_instance, str(e)))

View File

@ -493,6 +493,53 @@ def enable_certificate_renewal(ca):
return False
def certificate_renewal_stop_ca(ca):
"""
Validate the certmonger configuration on certificates that already
have renewal configured.
As of certmonger 0.65 it now does locking from the point where it
generates the CSR to the end of the post-command. This is to ensure
that only one certmonger renewal, and hopefully, one process at a
time holds the NSS database open in read/write.
"""
root_logger.info('[Certificate renewal should stop the CA]')
if not ca.is_configured():
root_logger.info('CA is not configured')
return False
nss_dir = dogtag.configured_constants().ALIAS_DIR
# Using the nickname find the certmonger request_id
criteria = (('cert_storage_location', nss_dir, certmonger.NPATH),('cert_nickname', 'auditSigningCert cert-pki-ca', None))
id = certmonger.get_request_id(criteria)
if id is None:
root_logger.error('Unable to find certmonger request ID for auditSigning Cert')
return False
if sysupgrade.get_upgrade_state('dogtag', 'stop_ca_during_renewal'):
return False
# State not set, lets see if we are already configured
pre_command = certmonger.get_request_value(id, 'pre_certsave_command')
if pre_command is not None:
if pre_command.strip().endswith('stop_pkicad'):
root_logger.info('Already configured to stop CA')
return False
# Ok, now we need to stop tracking, then we can start tracking them
# again with new configuration:
cainstance.stop_tracking_certificates(dogtag.configured_constants())
if ca.is_master():
ca.configure_renewal()
else:
ca.configure_certmonger_renewal()
ca.configure_clone_renewal()
ca.configure_agent_renewal()
ca.track_servercert()
sysupgrade.set_upgrade_state('dogtag', 'stop_ca_during_renewal', True)
root_logger.debug('CA subsystem certificate renewal configured to stop the CA')
return True
def copy_crl_file(old_path, new_path=None):
"""
Copy CRL to new location, update permissions and SELinux context
@ -711,7 +758,12 @@ def main():
bind.restart()
except ipautil.CalledProcessError, e:
root_logger.error("Failed to restart %s: %s", bind.service_name, e)
ca_restart = ca_restart or enable_certificate_renewal(ca) or upgrade_ipa_profile(ca, api.env.domain, fqdn)
ca_restart = any([
ca_restart,
enable_certificate_renewal(ca),
upgrade_ipa_profile(ca, api.env.domain, fqdn),
certificate_renewal_stop_ca(ca),
])
if ca_restart:
root_logger.info('pki-ca configuration changed, restart pki-ca')

View File

@ -261,7 +261,7 @@ def stop_tracking(secdir, request_id=None, nickname=None):
# Fall back to trying to stop tracking using nickname
pass
args = ['/usr/bin/ipa-getcert',
args = ['/usr/bin/getcert',
'stop-tracking',
]
if request_id:
@ -368,7 +368,8 @@ def get_pin(token, dogtag_constants=None):
return pin.strip()
return None
def dogtag_start_tracking(ca, nickname, pin, pinfile, secdir, command):
def dogtag_start_tracking(ca, nickname, pin, pinfile, secdir, pre_command,
post_command):
"""
Tell certmonger to start tracking a dogtag CA certificate. These
are handled differently because their renewal must be done directly
@ -377,7 +378,10 @@ def dogtag_start_tracking(ca, nickname, pin, pinfile, secdir, command):
This uses the generic certmonger command getcert so we can specify
a different helper.
command is the script to execute.
pre_command is the script to execute before a renewal is done.
post_command is the script to execute after a renewal is done.
Both commands can be None.
Returns the stdout, stderr and returncode from running ipa-getcert
@ -386,20 +390,32 @@ def dogtag_start_tracking(ca, nickname, pin, pinfile, secdir, command):
if not cert_exists(nickname, os.path.abspath(secdir)):
raise RuntimeError('Nickname "%s" doesn\'t exist in NSS database "%s"' % (nickname, secdir))
if command is not None and not os.path.isabs(command):
if sys.maxsize > 2**32:
libpath = 'lib64'
else:
libpath = 'lib'
command = '/usr/%s/ipa/certmonger/%s' % (libpath, command)
args = ["/usr/bin/getcert", "start-tracking",
"-d", os.path.abspath(secdir),
"-n", nickname,
"-c", ca,
"-C", command,
]
if pre_command is not None:
if not os.path.isabs(pre_command):
if sys.maxsize > 2**32:
libpath = 'lib64'
else:
libpath = 'lib'
pre_command = '/usr/%s/ipa/certmonger/%s' % (libpath, pre_command)
args.append("-B")
args.append(pre_command)
if post_command is not None:
if not os.path.isabs(post_command):
if sys.maxsize > 2**32:
libpath = 'lib64'
else:
libpath = 'lib'
post_command = '/usr/%s/ipa/certmonger/%s' % (libpath, post_command)
args.append("-C")
args.append(post_command)
if pinfile:
args.append("-p")
args.append(pinfile)

View File

@ -35,11 +35,13 @@ import urllib
import xml.dom.minidom
import stat
import socket
import syslog
import ConfigParser
from ipapython import dogtag
from ipapython.certdb import get_ca_nickname
from ipapython import certmonger
from ipalib import pkcs10, x509
from ipalib import errors
from ipapython.dn import DN
import subprocess
import traceback
@ -1048,7 +1050,11 @@ class CAInstance(service.Service):
On upgrades this needs to be called from ipa-upgradeconfig.
"""
certmonger.dogtag_start_tracking('dogtag-ipa-retrieve-agent-submit', 'ipaCert', None, '/etc/httpd/alias/pwdfile.txt', '/etc/httpd/alias', 'restart_httpd')
try:
certmonger.dogtag_start_tracking('dogtag-ipa-retrieve-agent-submit', 'ipaCert', None, '/etc/httpd/alias/pwdfile.txt', '/etc/httpd/alias', None, 'restart_httpd')
except (ipautil.CalledProcessError, RuntimeError), e:
root_logger.error(
"certmonger failed to start tracking certificate: %s" % str(e))
def __configure_ra(self):
# Create an RA user in the CA LDAP server and add that user to
@ -1534,11 +1540,19 @@ class CAInstance(service.Service):
'Unable to determine PIN for CA instance: %s' % str(e))
def track_servercert(self):
"""
Specifically do not tell certmonger to restart the CA. This will be
done by the renewal script, renew_ca_cert once all the subsystem
certificates are renewed.
"""
pin = self.__get_ca_pin()
certmonger.dogtag_start_tracking(
'dogtag-ipa-renew-agent', 'Server-Cert cert-pki-ca', pin, None,
self.dogtag_constants.ALIAS_DIR,
'restart_pkicad "Server-Cert cert-pki-ca"')
try:
certmonger.dogtag_start_tracking(
'dogtag-ipa-renew-agent', 'Server-Cert cert-pki-ca', pin, None,
self.dogtag_constants.ALIAS_DIR, None, None)
except (ipautil.CalledProcessError, RuntimeError), e:
root_logger.error(
"certmonger failed to start tracking certificate: %s" % str(e))
def configure_renewal(self):
cmonger = ipaservices.knownservices.certmonger
@ -1552,12 +1566,20 @@ class CAInstance(service.Service):
for nickname in ['auditSigningCert cert-pki-ca',
'ocspSigningCert cert-pki-ca',
'subsystemCert cert-pki-ca']:
certmonger.dogtag_start_tracking(
'dogtag-ipa-renew-agent', nickname, pin, None,
self.dogtag_constants.ALIAS_DIR, 'renew_ca_cert "%s"' % nickname)
try:
certmonger.dogtag_start_tracking(
'dogtag-ipa-renew-agent', nickname, pin, None,
self.dogtag_constants.ALIAS_DIR, 'stop_pkicad', 'renew_ca_cert "%s"' % nickname)
except (ipautil.CalledProcessError, RuntimeError), e:
root_logger.error(
"certmonger failed to start tracking certificate: %s" % str(e))
# Set up the agent cert for renewal
certmonger.dogtag_start_tracking('dogtag-ipa-renew-agent', 'ipaCert', None, '/etc/httpd/alias/pwdfile.txt', '/etc/httpd/alias', 'renew_ra_cert')
try:
certmonger.dogtag_start_tracking('dogtag-ipa-renew-agent', 'ipaCert', None, '/etc/httpd/alias/pwdfile.txt', '/etc/httpd/alias', None, 'renew_ra_cert')
except (ipautil.CalledProcessError, RuntimeError), e:
root_logger.error(
"certmonger failed to start tracking certificate: %s" % str(e))
def configure_certmonger_renewal(self):
"""
@ -1595,10 +1617,14 @@ class CAInstance(service.Service):
for nickname in ['auditSigningCert cert-pki-ca',
'ocspSigningCert cert-pki-ca',
'subsystemCert cert-pki-ca']:
certmonger.dogtag_start_tracking(
'dogtag-ipa-retrieve-agent-submit', nickname, pin, None,
self.dogtag_constants.ALIAS_DIR,
'restart_pkicad "%s"' % nickname)
try:
certmonger.dogtag_start_tracking(
'dogtag-ipa-retrieve-agent-submit', nickname, pin, None,
self.dogtag_constants.ALIAS_DIR, 'stop_pkicad',
'restart_pkicad "%s"' % nickname)
except (ipautil.CalledProcessError, RuntimeError), e:
root_logger.error(
"certmonger failed to start tracking certificate: %s" % str(e))
# The agent renewal is configured in import_ra_cert which is called
# after the HTTP instance is created.
@ -1861,6 +1887,67 @@ def update_cert_config(nickname, cert):
base64.b64encode(cert),
quotes=False, separator='=')
def update_people_entry(uid, dercert):
"""
Update the userCerticate for an entry in the dogtag ou=People. This
is needed when a certificate is renewed.
uid: uid of user to update
dercert: An X509.3 certificate in DER format
Logging is done via syslog
Returns True or False
"""
dn = DN(('uid',uid),('ou','People'),('o','ipaca'))
serial_number = x509.get_serial_number(dercert, datatype=x509.DER)
subject = x509.get_subject(dercert, datatype=x509.DER)
issuer = x509.get_issuer(dercert, datatype=x509.DER)
attempts = 0
dogtag_uri='ldap://localhost:%d' % DEFAULT_DSPORT
updated = False
try:
dm_password = certmonger.get_pin('internaldb')
except IOError, e:
syslog.syslog(syslog.LOG_ERR, 'Unable to determine PIN for CA instance: %s' % e)
return False
while attempts < 10:
conn = None
try:
conn = ldap2.ldap2(shared_instance=False, ldap_uri=dogtag_uri)
conn.connect(bind_dn=DN(('cn', 'directory manager')),
bind_pw=dm_password)
(entry_dn, entry_attrs) = conn.get_entry(dn, ['usercertificate'],
normalize=False)
entry_attrs['usercertificate'].append(dercert)
entry_attrs['description'] = '2;%d;%s;%s' % (serial_number, issuer,
subject)
conn.update_entry(dn, entry_attrs, normalize=False)
updated = True
break
except errors.NetworkError:
syslog.syslog(syslog.LOG_ERR, 'Connection to %s failed, sleeping 30s' % dogtag_uri)
time.sleep(30)
attempts += 1
except errors.EmptyModlist:
updated = True
break
except Exception, e:
syslog.syslog(syslog.LOG_ERR, 'Updating %s entry failed: %s' % (str(dn), e))
break
finally:
if conn.isconnected():
conn.disconnect()
if not updated:
syslog.syslog(syslog.LOG_ERR, 'Update failed.')
return False
return True
if __name__ == "__main__":
standard_logging_setup("install.log")
if not dogtag.install_constants.SHARED_DB: