Move the HTTP/S request code to a common library

This moves code that does HTTP and HTTPS requests into a common library
that can be used by both the installer and the dogtag plugin.

These functions are not generic HTTP/S clients, they are designed
specifically to talk to dogtag, so use accordingly.

ipapython/dogtag.py

@@ -20,6 +20,12 @@
 from ipalib import api, errors
 import httplib
 import xml.dom.minidom
+from ipapython import nsslib
+import nss.nss as nss
+from ipalib.errors import NetworkError, CertificateOperationError
+from urllib import urlencode
+import socket
+import logging
 
 def get_ca_certchain(ca_host=None):
     """
@@ -28,7 +34,7 @@ def get_ca_certchain(ca_host=None):
     if ca_host is None:
         ca_host = api.env.ca_host
     chain = None
-    conn = httplib.HTTPConnection(ca_host, 9180)
+    conn = httplib.HTTPConnection(ca_host, api.env.ca_port)
     conn.request("GET", "/ca/ee/ca/getCertChain")
     res = conn.getresponse()
     if res.status == 200:
@@ -50,3 +56,74 @@ def get_ca_certchain(ca_host=None):
             doc.unlink()
 
     return chain
+
+def https_request(host, port, url, secdir, password, nickname, **kw):
+    """
+    :param url: The URL to post to.
+    :param kw:  Keyword arguments to encode into POST body.
+    :return:   (http_status, http_reason_phrase, http_headers, http_body)
+               as (integer, unicode, dict, str)
+
+    Perform a client authenticated HTTPS request
+    """
+    uri = 'https://%s:%d%s' % (host, port, url)
+    post = urlencode(kw)
+    logging.info('sslget %r', uri)
+    logging.debug('sslget post %r', post)
+    request_headers = {"Content-type": "application/x-www-form-urlencoded",
+                       "Accept": "text/plain"}
+    try:
+        conn = nsslib.NSSConnection(host, port, dbdir=secdir)
+        conn.sslsock.set_client_auth_data_callback(nsslib.client_auth_data_callback,
+                                                   nickname,
+                                                   password, nss.get_default_certdb())
+        conn.set_debuglevel(0)
+        conn.request("POST", url, post, request_headers)
+
+        res = conn.getresponse()
+
+        http_status = res.status
+        http_reason_phrase = unicode(res.reason, 'utf-8')
+        http_headers = res.msg.dict
+        http_body = res.read()
+        conn.close()
+    except Exception, e:
+        raise NetworkError(uri=uri, error=str(e))
+
+    return http_status, http_reason_phrase, http_headers, http_body
+
+def http_request(host, port, url, **kw):
+        """
+        :param url: The URL to post to.
+        :param kw: Keyword arguments to encode into POST body.
+        :return:   (http_status, http_reason_phrase, http_headers, http_body)
+                   as (integer, unicode, dict, str)
+
+        Perform an HTTP request.
+        """
+        uri = 'http://%s:%s%s' % (host, port, url)
+        post = urlencode(kw)
+        logging.info('request %r', uri)
+        logging.debug('request post %r', post)
+        conn = httplib.HTTPConnection(host, port)
+        try:
+            conn.request('POST', url,
+                body=post,
+                headers={'Content-type': 'application/x-www-form-urlencoded'},
+            )
+            res = conn.getresponse()
+
+            http_status = res.status
+            http_reason_phrase = unicode(res.reason, 'utf-8')
+            http_headers = res.msg.dict
+            http_body = res.read()
+            conn.close()
+        except socket.error, e:
+            raise NetworkError(uri=uri, error=e.args[1])
+
+        logging.debug('request status %d',        http_status)
+        logging.debug('request reason_phrase %r', http_reason_phrase)
+        logging.debug('request headers %s',       http_headers)
+        logging.debug('request body %r',          http_body)
+
+        return http_status, http_reason_phrase, http_headers, http_body

ipaserver/install/certs.py

@@ -29,6 +29,7 @@ import fcntl
 import base64
 
 from ipapython import nsslib
+from ipapython import dogtag
 from ipapython import sysrestore
 from ipapython import ipautil
 from ConfigParser import RawConfigParser
@@ -553,31 +554,25 @@ class CertDB(object):
             if s >= 0:
                 csr = csr[s:]
 
-            params = urllib.urlencode({'profileId': 'caRAserverCert',
+            params = {'profileId': 'caRAserverCert',
                     'cert_request_type': 'pkcs10',
                     'requestor_name': 'IPA Installer',
                     'cert_request': csr,
-                    'xmlOutput': 'true'})
+                    'xmlOutput': 'true'}
-            headers = {"Content-type": "application/x-www-form-urlencoded",
-                       "Accept": "text/plain"}
 
             # Send the request to the CA
             f = open(self.passwd_fname, "r")
             password = f.readline()
             f.close()
-            conn = nsslib.NSSConnection(self.host_name, api.env.ca_agent_port, dbdir=self.secdir)
+            http_status, http_reason_phrase, http_headers, http_body = \
-            conn.sslsock.set_client_auth_data_callback(client_auth_data_callback, "ipaCert", password, nss.get_default_certdb())
+                dogtag.https_request(self.host_name, api.env.ca_agent_port, "/ca/agent/ca/profileSubmitSSLClient", self.secdir, password, "ipaCert", **params)
-            conn.set_debuglevel(0)
 
-            conn.request("POST", "/ca/agent/ca/profileSubmitSSLClient", params, headers)
+            if http_status != 200:
-            res = conn.getresponse()
+                raise CertificateOperationError(error=_('Unable to communicate with CMS (%s)') % \
-            data = res.read()
+                      http_reason_phrase)
-            conn.close()
-            if res.status != 200:
-                raise RuntimeError("Unable to submit cert request")
 
             # The result is an XML blob. Pull the certificate out of that
-            doc = xml.dom.minidom.parseString(data)
+            doc = xml.dom.minidom.parseString(http_body)
             item_node = doc.getElementsByTagName("b64")
             try:
                 try:
@@ -586,7 +581,6 @@ class CertDB(object):
                     raise RuntimeError("Certificate issuance failed")
             finally:
                 doc.unlink()
-                conn.close()
 
             # base64-decode the result for uniformity
             cert = base64.b64decode(cert)
@@ -647,35 +641,26 @@ class CertDB(object):
             if s >= 0:
                 csr = csr[s:]
 
-            params = urllib.urlencode({'profileId': 'caJarSigningCert',
+            params = {'profileId': 'caJarSigningCert',
                     'cert_request_type': 'pkcs10',
                     'requestor_name': 'IPA Installer',
                     'cert_request': csr,
-                    'xmlOutput': 'true'})
+                    'xmlOutput': 'true'}
-            headers = {"Content-type": "application/x-www-form-urlencoded",
-                       "Accept": "text/plain"}
 
             # Send the request to the CA
             f = open(self.passwd_fname, "r")
             password = f.readline()
             f.close()
-            conn = nsslib.NSSConnection(self.host_name, api.env.ca_agent_port, dbdir=self.secdir)
+            http_status, http_reason_phrase, http_headers, http_body = \
-            conn.sslsock.set_client_auth_data_callback(client_auth_data_callback, "ipaCert", password, nss.get_default_certdb())
+                dogtag.https_request(self.host_name, api.env.ca_agent_port, "/ca/agent/ca/profileSubmitSSLClient", self.secdir, password, "ipaCert", **params)
-            conn.set_debuglevel(0)
+            if http_status != 200:
-
-            conn.request("POST", "/ca/agent/ca/profileSubmitSSLClient", params, headers)
-            res = conn.getresponse()
-            data = res.read()
-            conn.close()
-            if res.status != 200:
                 raise RuntimeError("Unable to submit cert request")
 
             # The result is an XML blob. Pull the certificate out of that
-            doc = xml.dom.minidom.parseString(data)
+            doc = xml.dom.minidom.parseString(http_body)
             item_node = doc.getElementsByTagName("b64")
             cert = item_node[0].childNodes[0].data
             doc.unlink()
-            conn.close()
 
             # base64-decode the cert for uniformity
             cert = base64.b64decode(cert)

ipaserver/plugins/dogtag.py

@@ -1197,14 +1197,10 @@ if api.env.ra_plugin != 'dogtag':
     # In this case, abort loading this plugin module...
     raise SkipPluginModule(reason='dogtag not selected as RA plugin')
 import os
-from httplib import HTTPConnection
-from urllib import urlencode
 from ipaserver.plugins import rabase
-import socket
 from ipalib.errors import NetworkError, CertificateOperationError
 from ipalib.constants import TYPE_ERROR
-from ipapython import nsslib
+from ipapython import dogtag
-import nss.nss as nss
 from ipalib.request import ugettext as _
 
 class ra(rabase.rabase):
@@ -1239,32 +1235,7 @@ class ra(rabase.rabase):
 
         Perform an HTTP request.
         """
-        uri = 'http://%s:%s%s' % (self.env.ca_host, port, url)
+        return dogtag.http_request(self.env.ca_host, port, url, **kw)
-        post = urlencode(kw)
-        self.info('request %r', uri)
-        self.debug('request post %r', post)
-        conn = HTTPConnection(self.env.ca_host, port)
-        try:
-            conn.request('POST', url,
-                body=post,
-                headers={'Content-type': 'application/x-www-form-urlencoded'},
-            )
-            res = conn.getresponse()
-
-            http_status = res.status
-            http_reason_phrase = unicode(res.reason, 'utf-8')
-            http_headers = res.msg.dict
-            http_body = res.read()
-            conn.close()
-        except socket.error, e:
-            raise NetworkError(uri=uri, error=e.args[1])
-
-        self.debug('request status %d',        http_status)
-        self.debug('request reason_phrase %r', http_reason_phrase)
-        self.debug('request headers %s',       http_headers)
-        self.debug('request body %r',          http_body)
-
-        return http_status, http_reason_phrase, http_headers, http_body
 
     def _sslget(self, url, port, **kw):
         """
@@ -1275,36 +1246,8 @@ class ra(rabase.rabase):
 
         Perform an HTTPS request
         """
-        uri = 'https://%s:%d%s' % (self.env.ca_host, port, url)
-        post = urlencode(kw)
-        self.info('sslget %r', uri)
-        self.debug('sslget post %r', post)
-        request_headers = {"Content-type": "application/x-www-form-urlencoded",
-                           "Accept": "text/plain"}
-        try:
-            conn = nsslib.NSSConnection(self.env.ca_host, port, dbdir=self.sec_dir)
-            conn.sslsock.set_client_auth_data_callback(nsslib.client_auth_data_callback,
-                                                       self.ipa_certificate_nickname,
-                                                       self.password, nss.get_default_certdb())
-            conn.set_debuglevel(10)
-            conn.request("POST", url, post, request_headers)
 
-            res = conn.getresponse()
+        return dogtag.https_request(self.env.ca_host, port, url, self.sec_dir, self.password, self.ipa_certificate_nickname, **kw)
-
-            http_status = res.status
-            http_reason_phrase = unicode(res.reason, 'utf-8')
-            http_headers = res.msg.dict
-            http_body = res.read()
-            conn.close()
-        except Exception, e:
-            raise NetworkError(uri=uri, error=str(e))
-
-        self.debug('sslget status %d',        http_status)
-        self.debug('sslget reason_phrase %r', http_reason_phrase)
-        self.debug('sslget headers %s',       http_headers)
-        self.debug('sslget body %r',          http_body)
-
-        return http_status, http_reason_phrase, http_headers, http_body
 
     def get_parse_result_xml(self, xml_text, parse_func):
         '''