From a48f6511f6f463488452c320dc1370b0ac620646 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Wed, 11 Oct 2017 12:09:30 +0200 Subject: [PATCH] Use namespace-aware meta importer for ipaplatform Instead of symlinks and build-time configuration the ipaplatform module is now able to auto-detect platforms on import time. The meta importer uses the platform 'ID' from /etc/os-releases. It falls back to 'ID_LIKE' on platforms like CentOS, which has ID=centos and ID_LIKE="rhel fedora". The meta importer is able to handle namespace packages and the ipaplatform package has been turned into a namespace package in order to support external platform specifications. https://fedorahosted.org/freeipa/ticket/6474 Signed-off-by: Christian Heimes Reviewed-By: Rob Crittenden --- .gitignore | 6 +- .travis.yml | 4 +- Makefile.am | 22 ++- configure.ac | 8 - freeipa.spec.in | 2 + ipalib/config.py | 13 +- ipalib/setup.py | 1 + ipalib/util.py | 7 +- ipaplatform/Makefile.am | 11 ++ ipaplatform/__init__.py | 11 ++ ipaplatform/_importhook.py | 150 ++++++++++++++++++ ipaplatform/base/constants.py | 3 + ipaplatform/base/paths.py | 4 +- ipaplatform/base/services.py | 8 +- ipaplatform/base/tasks.py | 6 + ipaplatform/constants.py | 8 + ipaplatform/override.py.in | 1 + ipaplatform/paths.py | 8 + ipaplatform/services.py | 8 + ipaplatform/setup.py | 1 + ipaplatform/tasks.py | 8 + ipapython/certdb.py | 28 ++-- ipapython/config.py | 13 +- ipapython/setup.py | 1 + ipasetup.py.in | 13 +- ipatests/setup.py | 3 + ipatests/test_integration/test_ipalib_util.py | 23 --- ipatests/test_ipaplatform/__init__.py | 0 .../test_ipaplatform/data/os-release-centos | 15 ++ .../test_ipaplatform/data/os-release-fedora | 16 ++ .../test_ipaplatform/data/os-release-ubuntu | 11 ++ ipatests/test_ipaplatform/test_importhook.py | 54 +++++++ pylint_plugins.py | 43 +++++ pylintrc | 6 +- pypi/Makefile.am | 1 - pypi/ipaplatform/Makefile.am | 3 - pypi/ipaplatform/README.txt | 2 - pypi/ipaplatform/ipaplatform/__init__.py | 5 - pypi/ipaplatform/setup.cfg | 6 - pypi/ipaplatform/setup.py | 26 --- pypi/test_placeholder.py | 4 +- 41 files changed, 422 insertions(+), 141 deletions(-) create mode 100644 ipaplatform/__init__.py create mode 100644 ipaplatform/_importhook.py create mode 100644 ipaplatform/constants.py create mode 100644 ipaplatform/override.py.in create mode 100644 ipaplatform/paths.py create mode 100644 ipaplatform/services.py create mode 100644 ipaplatform/tasks.py delete mode 100644 ipatests/test_integration/test_ipalib_util.py create mode 100644 ipatests/test_ipaplatform/__init__.py create mode 100644 ipatests/test_ipaplatform/data/os-release-centos create mode 100644 ipatests/test_ipaplatform/data/os-release-fedora create mode 100644 ipatests/test_ipaplatform/data/os-release-ubuntu create mode 100644 ipatests/test_ipaplatform/test_importhook.py delete mode 100644 pypi/ipaplatform/Makefile.am delete mode 100644 pypi/ipaplatform/README.txt delete mode 100644 pypi/ipaplatform/ipaplatform/__init__.py delete mode 100644 pypi/ipaplatform/setup.cfg delete mode 100755 pypi/ipaplatform/setup.py diff --git a/.gitignore b/.gitignore index 8f4c2aa7a..cb1d98a56 100644 --- a/.gitignore +++ b/.gitignore @@ -108,11 +108,7 @@ freeipa2-dev-doc /client/ipa-join /client/ipa-rmkeytab +/ipaplatform/override.py /ipapython/version.py /ipapython/.DEFAULT_PLUGINS -/ipaplatform/__init__.py -/ipaplatform/constants.py -/ipaplatform/paths.py -/ipaplatform/services.py -/ipaplatform/tasks.py diff --git a/.travis.yml b/.travis.yml index 8875b7760..b59a42c14 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,9 +28,9 @@ env: test_install test_ipaclient test_ipalib + test_ipaplatform test_ipapython test_ipaserver - test_integration/test_ipalib_util.py test_xmlrpc/test_[l-z]*.py" - TASK_TO_RUN="run-tests" PYTHON=/usr/bin/python3 @@ -43,9 +43,9 @@ env: test_install test_ipaclient test_ipalib + test_ipaplatform test_ipapython test_ipaserver - test_integration/test_ipalib_util.py test_xmlrpc/test_[l-uw-z]*.py" # FIXME: add vault tests once PKI finally fixes vault install: diff --git a/Makefile.am b/Makefile.am index 90a4875a1..e16a5379f 100644 --- a/Makefile.am +++ b/Makefile.am @@ -8,10 +8,14 @@ if WITH_IPATESTS IPATESTS_SUBDIRS = ipatests endif -IPACLIENT_SUBDIRS = ipaclient ipalib ipapython -IPA_PLACEHOLDERS = freeipa ipa ipaplatform ipaserver ipatests +IPACLIENT_SUBDIRS = ipaclient ipalib ipaplatform ipapython +IPA_PLACEHOLDERS = freeipa ipa ipaserver ipatests SUBDIRS = asn1 util client contrib po pypi \ - $(IPACLIENT_SUBDIRS) ipaplatform $(IPATESTS_SUBDIRS) $(SERVER_SUBDIRS) + $(IPACLIENT_SUBDIRS) $(IPATESTS_SUBDIRS) $(SERVER_SUBDIRS) + +GENERATED_PYTHON_FILES = \ + $(top_builddir)/ipaplatform/override.py \ + $(top_builddir)/ipapython/version.py MOSTLYCLEANFILES = ipasetup.pyc ipasetup.pyo \ pylint_plugins.pyc pylint_plugins.pyo @@ -133,7 +137,7 @@ _srpms-body: _rpms-prep rm -f rm -f $(top_builddir)/.version .PHONY: lite-server -lite-server: $(top_builddir)/ipapython/version.py +lite-server: $(GENERATED_PYTHON_FILES) +$(MAKE) -C $(top_builddir)/install/ui PYTHONPATH=$(top_srcdir) $(PYTHON) -bb \ contrib/lite-server.py $(LITESERVER_ARGS) @@ -179,16 +183,20 @@ else endif @echo "All tests passed." +.PHONY: $(top_builddir)/ipaplatform/override.py +$(top_builddir)/ipaplatform/override.py: + (cd $(top_builddir)/ipaplatform && make override.py) + .PHONY: $(top_builddir)/ipapython/version.py $(top_builddir)/ipapython/version.py: (cd $(top_builddir)/ipapython && make version.py) .PHONY: acilint -acilint: $(top_builddir)/ipapython/version.py +acilint: $(GENERATED_PYTHON_FILES) cd $(srcdir); ./makeaci --validate .PHONY: apilint -apilint: $(top_builddir)/ipapython/version.py +apilint: $(GENERATED_PYTHON_FILES) cd $(srcdir); ./makeapi --validate .PHONY: polint @@ -203,7 +211,7 @@ polint: .PHONY: pylint if WITH_PYLINT -pylint: $(top_builddir)/ipapython/version.py ipasetup.py +pylint: $(GENERATED_PYTHON_FILES) ipasetup.py FILES=`find $(top_srcdir) \ -type d -exec test -e '{}/__init__.py' \; -print -prune -o \ -path './rpmbuild' -prune -o \ diff --git a/configure.ac b/configure.ac index f098eb1da..699fd64cb 100644 --- a/configure.ac +++ b/configure.ac @@ -520,13 +520,6 @@ AC_SUBST(LDFLAGS) # Files -AC_CONFIG_LINKS([ipaplatform/__init__.py:ipaplatform/$IPAPLATFORM/__init__.py - ipaplatform/constants.py:ipaplatform/$IPAPLATFORM/constants.py - ipaplatform/paths.py:ipaplatform/$IPAPLATFORM/paths.py - ipaplatform/services.py:ipaplatform/$IPAPLATFORM/services.py - ipaplatform/tasks.py:ipaplatform/$IPAPLATFORM/tasks.py - ]) - AC_CONFIG_FILES([ Makefile asn1/Makefile @@ -594,7 +587,6 @@ AC_CONFIG_FILES([ pypi/Makefile pypi/freeipa/Makefile pypi/ipa/Makefile - pypi/ipaplatform/Makefile pypi/ipaserver/Makefile pypi/ipatests/Makefile po/Makefile.in diff --git a/freeipa.spec.in b/freeipa.spec.in index 35cc671aa..eefaba72c 100644 --- a/freeipa.spec.in +++ b/freeipa.spec.in @@ -1637,6 +1637,7 @@ fi %{python_sitelib}/ipapython-*.egg-info %{python_sitelib}/ipalib-*.egg-info %{python_sitelib}/ipaplatform-*.egg-info +%{python_sitelib}/ipaplatform-*-nspkg.pth %files common -f %{gettext_domain}.lang @@ -1658,6 +1659,7 @@ fi %{python3_sitelib}/ipapython-*.egg-info %{python3_sitelib}/ipalib-*.egg-info %{python3_sitelib}/ipaplatform-*.egg-info +%{python3_sitelib}/ipaplatform-*-nspkg.pth %endif # with_python3 diff --git a/ipalib/config.py b/ipalib/config.py index 151c4b4a8..b6c17fa1b 100644 --- a/ipalib/config.py +++ b/ipalib/config.py @@ -28,6 +28,7 @@ of the process. For the per-request thread-local information, see `ipalib.request`. """ +from __future__ import absolute_import import os from os import path @@ -39,6 +40,7 @@ from six.moves.urllib.parse import urlparse, urlunparse from six.moves.configparser import RawConfigParser, ParsingError # pylint: enable=import-error +from ipaplatform.tasks import tasks from ipapython.dn import DN from ipalib.base import check_name from ipalib.constants import ( @@ -47,12 +49,6 @@ from ipalib.constants import ( TLS_VERSIONS ) from ipalib import errors -try: - # pylint: disable=ipa-forbidden-import - from ipaplatform.tasks import tasks - # pylint: enable=ipa-forbidden-import -except ImportError: - tasks = None if six.PY3: unicode = str @@ -451,10 +447,7 @@ class Env(object): self.script = path.abspath(sys.argv[0]) self.bin = path.dirname(self.script) self.home = os.environ.get('HOME', None) - - # Set fips_mode only if ipaplatform module was loaded - if tasks is not None: - self.fips_mode = tasks.is_fips_enabled() + self.fips_mode = tasks.is_fips_enabled() # Merge in overrides: self._merge(**overrides) diff --git a/ipalib/setup.py b/ipalib/setup.py index cdbd61c01..722fcebd3 100644 --- a/ipalib/setup.py +++ b/ipalib/setup.py @@ -37,6 +37,7 @@ if __name__ == '__main__': "ipalib.install", ], install_requires=[ + "ipaplatform", "ipapython", "netaddr", "pyasn1", diff --git a/ipalib/util.py b/ipalib/util.py index 65b3eceda..a02be89f3 100644 --- a/ipalib/util.py +++ b/ipalib/util.py @@ -55,6 +55,7 @@ from ipalib.constants import ( TLS_VERSIONS, TLS_VERSION_MINIMAL, TLS_HIGH_CIPHERS ) from ipalib.text import _ +from ipaplatform.paths import paths from ipapython.ssh import SSHPublicKey from ipapython.dn import DN, RDN from ipapython.dnsutil import DNSName @@ -1101,9 +1102,9 @@ def check_client_configuration(): """ Check if IPA client is configured on the system. """ - if (not os.path.isfile(_IPA_DEFAULT_CONF) or - not os.path.isdir(_IPA_CLIENT_SYSRESTORE) or - not os.listdir(_IPA_CLIENT_SYSRESTORE)): + if (not os.path.isfile(paths.IPA_DEFAULT_CONF) or + not os.path.isdir(paths.IPA_CLIENT_SYSRESTORE) or + not os.listdir(paths.IPA_CLIENT_SYSRESTORE)): raise ScriptError('IPA client is not configured on this system') diff --git a/ipaplatform/Makefile.am b/ipaplatform/Makefile.am index 8be72b25d..ddee56c84 100644 --- a/ipaplatform/Makefile.am +++ b/ipaplatform/Makefile.am @@ -1 +1,12 @@ include $(top_srcdir)/Makefile.python.am + +EXTRA_DIST = override.py.in + +all-local: override.py +dist-hook: override.py +install-exec-local: override.py + +override.py: override.py.in $(top_builddir)/$(CONFIG_STATUS) + $(AM_V_GEN)sed \ + -e 's|@IPAPLATFORM[@]|$(IPAPLATFORM)|g' \ + $< > $@ diff --git a/ipaplatform/__init__.py b/ipaplatform/__init__.py new file mode 100644 index 000000000..06397fda1 --- /dev/null +++ b/ipaplatform/__init__.py @@ -0,0 +1,11 @@ +# +# Copyright (C) 2017 FreeIPA Contributors see COPYING for license +# +"""ipaplatform namespace package + +In the presence of a namespace package, any code in this module will be +ignore. +""" +__import__('pkg_resources').declare_namespace(__name__) + +NAME = None # initialized by IpaMetaImporter diff --git a/ipaplatform/_importhook.py b/ipaplatform/_importhook.py new file mode 100644 index 000000000..8e4e58b7a --- /dev/null +++ b/ipaplatform/_importhook.py @@ -0,0 +1,150 @@ +# +# Copyright (C) 2017 FreeIPA Contributors see COPYING for license +# +"""Meta import hook for ipaplatform. + +Known Linux distros with /etc/os-release +---------------------------------------- + +- alpine +- centos (like rhel, fedora) +- debian +- fedora +- rhel +- ubuntu (like debian) +""" + +import importlib +import io +import re +import sys +import warnings + + +import ipaplatform +try: + from ipaplatform.override import OVERRIDE +except ImportError: + OVERRIDE = None + + +_osrelease_line = re.compile( + u"^(?!#)(?P[a-zA-Z0-9_]+)=" + u"(?P[\"\']?)(?P.+)(?P=quote)$" +) + + +class IpaMetaImporter(object): + """Meta import hook and platform detector. + + The meta import hook uses /etc/os-release to auto-detects the best + matching ipaplatform provider. It is compatible with external namespace + packages, too. + """ + modules = { + 'ipaplatform.constants', + 'ipaplatform.paths', + 'ipaplatform.services', + 'ipaplatform.tasks' + } + + bsd_family = ( + 'freebsd', + 'openbsd', + 'netbsd', + 'dragonfly', + 'gnukfreebsd' + ) + + def __init__(self, override=OVERRIDE): + self.override = override + self.platform_ids = self._get_platform_ids(self.override) + self.platform = self._get_platform(self.platform_ids) + + def _get_platform_ids(self, override): + platforms = [] + # allow RPM and Debian packages to override platform + if override is not None: + platforms.append(override) + + if sys.platform.startswith('linux'): + # Linux, get distribution from /etc/os-release + try: + platforms.extend(self._parse_osrelease()) + except Exception as e: + warnings.warn("Failed to read /etc/os-release: {}".format(e)) + elif sys.platform == 'win32': + # Windows 32 or 64bit platform + platforms.append('win32') + elif sys.platform == 'darwin': + # macOS + platforms.append('macos') + elif sys.platform.startswith(self.bsd_family): + # BSD family, look for e.g. ['freebsd10', 'freebsd'] + platforms.append(sys.platform) + simple = sys.platform.rstrip('0123456789') + if simple != sys.platform: + platforms.append(simple) + + if not platforms: + raise ValueError("Unsupported platform: {}".format(sys.platform)) + + return platforms + + def _parse_osrelease(self, filename='/etc/os-release'): + release = {} + with io.open(filename, encoding='utf-8') as f: + for line in f: + mo = _osrelease_line.match(line) + if mo is not None: + release[mo.group('name')] = mo.group('value') + + platforms = [ + release['ID'], + ] + if "ID_LIKE" in release: + platforms.extend( + v.strip() for v in release['ID_LIKE'].split(' ') if v.strip() + ) + + return platforms + + def _get_platform(self, platform_ids): + for platform in platform_ids: + try: + importlib.import_module('ipaplatform.{}'.format(platform)) + except ImportError: + pass + else: + return platform + raise ImportError('No ipaplatform available for "{}"'.format( + ', '.join(platform_ids))) + + def find_module(self, fullname, path=None): + """Meta importer hook""" + if fullname in self.modules: + return self + return None + + def load_module(self, fullname): + """Meta importer hook""" + suffix = fullname.split('.', 1)[1] + alias = 'ipaplatform.{}.{}'.format(self.platform, suffix) + platform_mod = importlib.import_module(alias) + base_mod = sys.modules.get(fullname) + if base_mod is not None: + # module has been imported before, update its __dict__ + base_mod.__dict__.update(platform_mod.__dict__) + for key in list(base_mod.__dict__): + if not hasattr(platform_mod, key): + delattr(base_mod, key) + else: + sys.modules[fullname] = platform_mod + return platform_mod + + +metaimporter = IpaMetaImporter() +sys.meta_path.insert(0, metaimporter) + +fixup_module = metaimporter.load_module +ipaplatform.NAME = metaimporter.platform diff --git a/ipaplatform/base/constants.py b/ipaplatform/base/constants.py index 6592c63d9..5f52b94f6 100644 --- a/ipaplatform/base/constants.py +++ b/ipaplatform/base/constants.py @@ -37,3 +37,6 @@ class BaseConstantsNamespace(object): 'httpd_dbus_sssd': 'on', } SSSD_USER = "sssd" + + +constants = BaseConstantsNamespace() diff --git a/ipaplatform/base/paths.py b/ipaplatform/base/paths.py index 98372478e..6facc81e3 100644 --- a/ipaplatform/base/paths.py +++ b/ipaplatform/base/paths.py @@ -355,5 +355,7 @@ class BasePathNamespace(object): GSSPROXY_CONF = '/etc/gssproxy/10-ipa.conf' KRB5CC_HTTPD = '/tmp/krb5cc-httpd' IF_INET6 = '/proc/net/if_inet6' + AUTHCONFIG = None -path_namespace = BasePathNamespace + +paths = BasePathNamespace() diff --git a/ipaplatform/base/services.py b/ipaplatform/base/services.py index 875931b42..3cd2565ef 100644 --- a/ipaplatform/base/services.py +++ b/ipaplatform/base/services.py @@ -505,8 +505,12 @@ class SystemdService(PlatformService): # Objects below are expected to be exported by platform module -service = None -knownservices = None +def base_service_class_factory(name, api=None): + raise NotImplementedError + + +service = base_service_class_factory +knownservices = KnownServices({}) # System may support more time&date services. FreeIPA supports ntpd only, other # services will be disabled during IPA installation diff --git a/ipaplatform/base/tasks.py b/ipaplatform/base/tasks.py index dc3cacc23..8f73eaddc 100644 --- a/ipaplatform/base/tasks.py +++ b/ipaplatform/base/tasks.py @@ -204,6 +204,9 @@ class BaseTaskNamespace(object): """Configure httpd service to work with IPA""" raise NotImplementedError() + def configure_http_gssproxy_conf(self, ipauser): + raise NotImplementedError() + def remove_httpd_service_ipa_conf(self): """Remove configuration of httpd service of IPA""" raise NotImplementedError() @@ -219,3 +222,6 @@ class BaseTaskNamespace(object): logger.debug('Done adding user to group') except ipautil.CalledProcessError as e: logger.debug('Failed to add user to group: %s', e) + + +tasks = BaseTaskNamespace() diff --git a/ipaplatform/constants.py b/ipaplatform/constants.py new file mode 100644 index 000000000..cc43cfb1d --- /dev/null +++ b/ipaplatform/constants.py @@ -0,0 +1,8 @@ +# +# Copyright (C) 2017 FreeIPA Contributors see COPYING for license +# +"""IpaMetaImporter replaces this module with ipaplatform.$NAME.constants. +""" +import ipaplatform._importhook + +ipaplatform._importhook.fixup_module('ipaplatform.constants') diff --git a/ipaplatform/override.py.in b/ipaplatform/override.py.in new file mode 100644 index 000000000..1bd363ab2 --- /dev/null +++ b/ipaplatform/override.py.in @@ -0,0 +1 @@ +OVERRIDE = '@IPAPLATFORM@' diff --git a/ipaplatform/paths.py b/ipaplatform/paths.py new file mode 100644 index 000000000..2fcb477d4 --- /dev/null +++ b/ipaplatform/paths.py @@ -0,0 +1,8 @@ +# +# Copyright (C) 2017 FreeIPA Contributors see COPYING for license +# +"""IpaMetaImporter replaces this module with ipaplatform.$NAME.paths. +""" +import ipaplatform._importhook + +ipaplatform._importhook.fixup_module('ipaplatform.paths') diff --git a/ipaplatform/services.py b/ipaplatform/services.py new file mode 100644 index 000000000..0d40f6443 --- /dev/null +++ b/ipaplatform/services.py @@ -0,0 +1,8 @@ +# +# Copyright (C) 2017 FreeIPA Contributors see COPYING for license +# +"""IpaMetaImporter replaces this module with ipaplatform.$NAME.services. +""" +import ipaplatform._importhook + +ipaplatform._importhook.fixup_module('ipaplatform.services') diff --git a/ipaplatform/setup.py b/ipaplatform/setup.py index 501e2bc56..1098ab6f1 100644 --- a/ipaplatform/setup.py +++ b/ipaplatform/setup.py @@ -32,6 +32,7 @@ if __name__ == '__main__': name="ipaplatform", doc=__doc__, package_dir={'ipaplatform': ''}, + namespace_packages=['ipaplatform'], packages=[ "ipaplatform", "ipaplatform.base", diff --git a/ipaplatform/tasks.py b/ipaplatform/tasks.py new file mode 100644 index 000000000..23c785909 --- /dev/null +++ b/ipaplatform/tasks.py @@ -0,0 +1,8 @@ +# +# Copyright (C) 2017 FreeIPA Contributors see COPYING for license +# +"""IpaMetaImporter replaces this module with ipaplatform.$NAME.tasks. +""" +import ipaplatform._importhook + +ipaplatform._importhook.fixup_module('ipaplatform.tasks') diff --git a/ipapython/certdb.py b/ipapython/certdb.py index 92da7829a..14e6adf69 100644 --- a/ipapython/certdb.py +++ b/ipapython/certdb.py @@ -16,6 +16,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # +from __future__ import absolute_import import collections import logging @@ -30,24 +31,12 @@ import shutil import cryptography.x509 +from ipaplatform.paths import paths from ipapython.dn import DN from ipapython.kerberos import Principal from ipapython import ipautil from ipalib import x509 # pylint: disable=ipa-forbidden-import -try: - # pylint: disable=import-error,ipa-forbidden-import - from ipaplatform.paths import paths - # pylint: enable=import-error,ipa-forbidden-import -except ImportError: - CERTUTIL = '/usr/bin/certutil' - PK12UTIL = '/usr/bin/pk12util' - OPENSSL = '/usr/bin/openssl' -else: - CERTUTIL = paths.CERTUTIL - PK12UTIL = paths.PK12UTIL - OPENSSL = paths.OPENSSL - logger = logging.getLogger(__name__) @@ -188,7 +177,8 @@ def verify_kdc_cert_validity(kdc_cert, ca_certs, realm): try: ipautil.run( - [OPENSSL, 'verify', '-CAfile', ca_file.name, kdc_file.name], + [paths.OPENSSL, 'verify', '-CAfile', ca_file.name, + kdc_file.name], capture_output=True) except ipautil.CalledProcessError as e: raise ValueError(e.output) @@ -244,7 +234,7 @@ class NSSDatabase(object): self.close() def run_certutil(self, args, stdin=None, **kwargs): - new_args = [CERTUTIL, "-d", self.secdir] + new_args = [paths.CERTUTIL, "-d", self.secdir] new_args = new_args + args new_args.extend(['-f', self.pwd_file]) return ipautil.run(new_args, stdin, **kwargs) @@ -367,7 +357,7 @@ class NSSDatabase(object): return root_nicknames def export_pkcs12(self, nickname, pkcs12_filename, pkcs12_passwd=None): - args = [PK12UTIL, "-d", self.secdir, + args = [paths.PK12UTIL, "-d", self.secdir, "-o", pkcs12_filename, "-n", nickname, "-k", self.pwd_file] @@ -391,7 +381,7 @@ class NSSDatabase(object): pkcs12_password_file.close() def import_pkcs12(self, pkcs12_filename, pkcs12_passwd=None): - args = [PK12UTIL, "-d", self.secdir, + args = [paths.PK12UTIL, "-d", self.secdir, "-i", pkcs12_filename, "-k", self.pwd_file, '-v'] pkcs12_password_file = None @@ -501,7 +491,7 @@ class NSSDatabase(object): (key_file, filename)) args = [ - OPENSSL, 'pkcs8', + paths.OPENSSL, 'pkcs8', '-topk8', '-passout', 'file:' + self.pwd_file, ] @@ -588,7 +578,7 @@ class NSSDatabase(object): out_password = ipautil.ipa_generate_password() out_pwdfile = ipautil.write_tmp_file(out_password) args = [ - OPENSSL, 'pkcs12', + paths.OPENSSL, 'pkcs12', '-export', '-in', in_file.name, '-out', out_file.name, diff --git a/ipapython/config.py b/ipapython/config.py index aa4b3e48f..0b3c36978 100644 --- a/ipapython/config.py +++ b/ipapython/config.py @@ -16,6 +16,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # +from __future__ import absolute_import # pylint: disable=deprecated-module from optparse import ( @@ -33,18 +34,10 @@ from six.moves.configparser import SafeConfigParser from six.moves.urllib.parse import urlsplit # pylint: enable=import-error +from ipaplatform.paths import paths from ipapython.dn import DN from ipapython.ipautil import CheckedIPAddress, CheckedIPAddressLoopback -try: - # pylint: disable=ipa-forbidden-import - from ipaplatform.paths import paths - # pylint: enable=ipa-forbidden-import -except ImportError: - IPA_DEFAULT_CONF = '/etc/ipa/default.conf' -else: - IPA_DEFAULT_CONF = paths.IPA_DEFAULT_CONF - class IPAConfigError(Exception): def __init__(self, msg=''): @@ -188,7 +181,7 @@ config = IPAConfig() def __parse_config(discover_server = True): p = SafeConfigParser() - p.read(IPA_DEFAULT_CONF) + p.read(paths.IPA_DEFAULT_CONF) try: if not config.default_realm: diff --git a/ipapython/setup.py b/ipapython/setup.py index 4f7153039..b982577d9 100755 --- a/ipapython/setup.py +++ b/ipapython/setup.py @@ -42,6 +42,7 @@ if __name__ == '__main__': "dnspython", "gssapi", # "ipalib", # circular dependency + "ipaplatform", "netaddr", "netifaces", "six", diff --git a/ipasetup.py.in b/ipasetup.py.in index 2862ae234..426cbdb6e 100644 --- a/ipasetup.py.in +++ b/ipasetup.py.in @@ -25,22 +25,27 @@ class build_py(setuptools_build_py): """ def initialize_options(self): setuptools_build_py.initialize_options(self) - self.skip_package = None + self.skip_modules = () def finalize_options(self): setuptools_build_py.finalize_options(self) omit = os.environ.get('IPA_OMIT_INSTALL', '0') if omit == '1': distname = self.distribution.metadata.name - self.skip_package = '{}.install'.format(distname) + self.skip_modules = ( + # *.install.* subpackages + '{}.install'.format(distname), + # platform override module + 'ipaplatform.override', + ) log.warn("bdist_wheel: Ignore package: %s", - self.skip_package) + ', '.join(self.skip_modules)) def build_module(self, module, module_file, package): if isinstance(package, str): package = package.split('.') name = '.'.join(list(package) + [module]) - if self.skip_package and name.startswith(self.skip_package): + if self.skip_modules and name.startswith(self.skip_modules): # remove file in case it has been copied to build/lib before outfile = self.get_module_outfile(self.build_lib, package, module) try: diff --git a/ipatests/setup.py b/ipatests/setup.py index b576284b1..de98648f9 100644 --- a/ipatests/setup.py +++ b/ipatests/setup.py @@ -41,6 +41,7 @@ if __name__ == '__main__': "ipatests.test_integration", "ipatests.test_ipaclient", "ipatests.test_ipalib", + "ipatests.test_ipaplatform", "ipatests.test_ipapython", "ipatests.test_ipaserver", "ipatests.test_ipaserver.test_install", @@ -54,6 +55,7 @@ if __name__ == '__main__': 'ipatests.test_integration': ['scripts/*'], 'ipatests.test_ipaclient': ['data/*/*/*'], 'ipatests.test_ipalib': ['data/*'], + 'ipatests.test_ipaplatform': ['data/*'], "ipatests.test_ipaserver": ['data/*'], 'ipatests.test_xmlrpc': ['data/*'], }, @@ -63,6 +65,7 @@ if __name__ == '__main__': "gssapi", "ipaclient", "ipalib", + "ipaplatform", "ipapython", "nose", "polib", diff --git a/ipatests/test_integration/test_ipalib_util.py b/ipatests/test_integration/test_ipalib_util.py deleted file mode 100644 index 92fe5dbec..000000000 --- a/ipatests/test_integration/test_ipalib_util.py +++ /dev/null @@ -1,23 +0,0 @@ -# -# Copyright (C) 2017 FreeIPA Contributors see COPYING for license -# - -""" -Test the `ipalib.util` module. -This tests is in test_integration beucase we only can import ipaplatform here. -""" - -from ipalib import util -from ipaplatform.paths import paths - -import pytest -pytestmark = pytest.mark.tier0 - - -def test_hardcoded_paths_are_right(): - """ - Test if constants created in ipalib.util are in sync with - paths declared in ipaplatform.paths - """ - assert util._IPA_CLIENT_SYSRESTORE == paths.IPA_CLIENT_SYSRESTORE - assert util._IPA_DEFAULT_CONF == paths.IPA_DEFAULT_CONF diff --git a/ipatests/test_ipaplatform/__init__.py b/ipatests/test_ipaplatform/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/ipatests/test_ipaplatform/data/os-release-centos b/ipatests/test_ipaplatform/data/os-release-centos new file mode 100644 index 000000000..c276e3ae5 --- /dev/null +++ b/ipatests/test_ipaplatform/data/os-release-centos @@ -0,0 +1,15 @@ +NAME="CentOS Linux" +VERSION="7 (Core)" +ID="centos" +ID_LIKE="rhel fedora" +VERSION_ID="7" +PRETTY_NAME="CentOS Linux 7 (Core)" +ANSI_COLOR="0;31" +CPE_NAME="cpe:/o:centos:centos:7" +HOME_URL="https://www.centos.org/" +BUG_REPORT_URL="https://bugs.centos.org/" + +CENTOS_MANTISBT_PROJECT="CentOS-7" +CENTOS_MANTISBT_PROJECT_VERSION="7" +REDHAT_SUPPORT_PRODUCT="centos" +REDHAT_SUPPORT_PRODUCT_VERSION="7" diff --git a/ipatests/test_ipaplatform/data/os-release-fedora b/ipatests/test_ipaplatform/data/os-release-fedora new file mode 100644 index 000000000..a270edbf5 --- /dev/null +++ b/ipatests/test_ipaplatform/data/os-release-fedora @@ -0,0 +1,16 @@ +NAME=Fedora +VERSION="26 (Workstation Edition)" +ID=fedora +VERSION_ID=26 +PRETTY_NAME="Fedora 26 (Workstation Edition)" +ANSI_COLOR="0;34" +CPE_NAME="cpe:/o:fedoraproject:fedora:26" +HOME_URL="https://fedoraproject.org/" +BUG_REPORT_URL="https://bugzilla.redhat.com/" +REDHAT_BUGZILLA_PRODUCT="Fedora" +REDHAT_BUGZILLA_PRODUCT_VERSION=26 +REDHAT_SUPPORT_PRODUCT="Fedora" +REDHAT_SUPPORT_PRODUCT_VERSION=26 +PRIVACY_POLICY_URL=https://fedoraproject.org/wiki/Legal:PrivacyPolicy +VARIANT="Workstation Edition" +VARIANT_ID=workstation diff --git a/ipatests/test_ipaplatform/data/os-release-ubuntu b/ipatests/test_ipaplatform/data/os-release-ubuntu new file mode 100644 index 000000000..fc9a2bfb2 --- /dev/null +++ b/ipatests/test_ipaplatform/data/os-release-ubuntu @@ -0,0 +1,11 @@ +NAME="Ubuntu" +VERSION="16.04.3 LTS (Xenial Xerus)" +ID=ubuntu +ID_LIKE=debian +PRETTY_NAME="Ubuntu 16.04.3 LTS" +VERSION_ID="16.04" +HOME_URL="http://www.ubuntu.com/" +SUPPORT_URL="http://help.ubuntu.com/" +BUG_REPORT_URL="http://bugs.launchpad.net/ubuntu/" +VERSION_CODENAME=xenial +UBUNTU_CODENAME=xenial diff --git a/ipatests/test_ipaplatform/test_importhook.py b/ipatests/test_ipaplatform/test_importhook.py new file mode 100644 index 000000000..ca55af158 --- /dev/null +++ b/ipatests/test_ipaplatform/test_importhook.py @@ -0,0 +1,54 @@ +# +# Copyright (C) 2017 FreeIPA Contributors see COPYING for license +# +import os +import sys + +import pytest + +import ipaplatform.constants +import ipaplatform.paths +import ipaplatform.services +import ipaplatform.tasks +from ipaplatform._importhook import metaimporter +try: + from ipaplatform.override import OVERRIDE +except ImportError: + OVERRIDE = None + + +HERE = os.path.dirname(os.path.abspath(__file__)) +DATA = os.path.join(HERE, 'data') + + +@pytest.mark.skipif(OVERRIDE is None, + reason='test requires override') +def test_override(): + assert OVERRIDE == metaimporter.platform_ids[0] + assert OVERRIDE == metaimporter.platform + + +@pytest.mark.parametrize('mod, name', [ + (ipaplatform.constants, 'ipaplatform.constants'), + (ipaplatform.paths, 'ipaplatform.paths'), + (ipaplatform.services, 'ipaplatform.services'), + (ipaplatform.tasks, 'ipaplatform.tasks'), +]) +def test_importhook(mod, name): + assert name in metaimporter.modules + prefix, suffix = name.split('.') + assert prefix == 'ipaplatform' + override = '.'.join((prefix, metaimporter.platform, suffix)) + assert mod.__name__ == override + # dicts are equal, modules may not be identical + assert mod.__dict__ == sys.modules[override].__dict__ + + +@pytest.mark.parametrize('filename, expected_platforms', [ + (os.path.join(DATA, 'os-release-centos'), ['centos', 'rhel', 'fedora']), + (os.path.join(DATA, 'os-release-fedora'), ['fedora']), + (os.path.join(DATA, 'os-release-ubuntu'), ['ubuntu', 'debian']), +]) +def test_parse_os_release(filename, expected_platforms): + parsed = metaimporter._parse_osrelease(filename) + assert parsed == expected_platforms diff --git a/pylint_plugins.py b/pylint_plugins.py index e5c999861..594393a36 100644 --- a/pylint_plugins.py +++ b/pylint_plugins.py @@ -269,6 +269,49 @@ def pytest_config_transform(): register_module_extender(MANAGER, 'pytest', pytest_config_transform) +def ipaplatform_constants_transform(): + return AstroidBuilder(MANAGER).string_build(textwrap.dedent(''' + from ipaplatform.base.constants import constants + __all__ = ('constants',) + ''')) + + +def ipaplatform_paths_transform(): + return AstroidBuilder(MANAGER).string_build(textwrap.dedent(''' + from ipaplatform.base.paths import paths + __all__ = ('paths',) + ''')) + + +def ipaplatform_services_transform(): + return AstroidBuilder(MANAGER).string_build(textwrap.dedent(''' + from ipaplatform.base.services import knownservices + from ipaplatform.base.services import timedate_services + from ipaplatform.base.services import service + from ipaplatform.base.services import wellknownservices + from ipaplatform.base.services import wellknownports + __all__ = ('knownservices', 'timedate_services', 'service', + 'wellknownservices', 'wellknownports') + ''')) + + +def ipaplatform_tasks_transform(): + return AstroidBuilder(MANAGER).string_build(textwrap.dedent(''' + from ipaplatform.base.tasks import tasks + __all__ = ('tasks',) + ''')) + + +register_module_extender(MANAGER, 'ipaplatform.constants', + ipaplatform_constants_transform) +register_module_extender(MANAGER, 'ipaplatform.paths', + ipaplatform_paths_transform) +register_module_extender(MANAGER, 'ipaplatform.services', + ipaplatform_services_transform) +register_module_extender(MANAGER, 'ipaplatform.tasks', + ipaplatform_tasks_transform) + + class IPAChecker(BaseChecker): __implements__ = IAstroidChecker diff --git a/pylintrc b/pylintrc index 462b96cb0..8cd7c870b 100644 --- a/pylintrc +++ b/pylintrc @@ -116,9 +116,9 @@ dummy-variables-rgx=_.+ [IPA] forbidden-imports= client/:ipaserver, - ipaclient/:ipaclient.install:ipalib.install:ipaplatform:ipaserver, + ipaclient/:ipaclient.install:ipalib.install:ipaserver, ipaclient/install/:ipaserver, - ipalib/:ipaclient.install:ipalib.install:ipaplatform:ipaserver, + ipalib/:ipaclient.install:ipalib.install:ipaserver, ipalib/install/:ipaserver, ipaplatform/:ipaclient:ipalib:ipaserver, - ipapython/:ipaclient:ipalib:ipaplatform:ipaserver + ipapython/:ipaclient:ipalib:ipaserver diff --git a/pypi/Makefile.am b/pypi/Makefile.am index 5d8be9c1f..bcbe1ea41 100644 --- a/pypi/Makefile.am +++ b/pypi/Makefile.am @@ -7,7 +7,6 @@ NULL = SUBDIRS = \ freeipa \ ipa \ - ipaplatform \ ipaserver \ ipatests \ $(NULL) diff --git a/pypi/ipaplatform/Makefile.am b/pypi/ipaplatform/Makefile.am deleted file mode 100644 index 15d86ce0c..000000000 --- a/pypi/ipaplatform/Makefile.am +++ /dev/null @@ -1,3 +0,0 @@ -include $(top_srcdir)/Makefile.python.am - -pkginstall = false diff --git a/pypi/ipaplatform/README.txt b/pypi/ipaplatform/README.txt deleted file mode 100644 index 15064b0b0..000000000 --- a/pypi/ipaplatform/README.txt +++ /dev/null @@ -1,2 +0,0 @@ -This is a dummy package for FreeIPA's ipaplatform. - diff --git a/pypi/ipaplatform/ipaplatform/__init__.py b/pypi/ipaplatform/ipaplatform/__init__.py deleted file mode 100644 index 3b12c8c74..000000000 --- a/pypi/ipaplatform/ipaplatform/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -# -# Copyright (C) 2017 FreeIPA Contributors see COPYING for license -# - -raise ImportError("ipaplatform is not yet supported as PyPI package.") diff --git a/pypi/ipaplatform/setup.cfg b/pypi/ipaplatform/setup.cfg deleted file mode 100644 index 62f65c719..000000000 --- a/pypi/ipaplatform/setup.cfg +++ /dev/null @@ -1,6 +0,0 @@ -[bdist_wheel] -universal = 1 - -[aliases] -packages = clean --all egg_info bdist_wheel -release = packages register upload diff --git a/pypi/ipaplatform/setup.py b/pypi/ipaplatform/setup.py deleted file mode 100755 index f0fca2c70..000000000 --- a/pypi/ipaplatform/setup.py +++ /dev/null @@ -1,26 +0,0 @@ -# -# Copyright (C) 2017 FreeIPA Contributors see COPYING for license -# -"""Dummy package for FreeIPA - -ipaplatform is not yet available as PyPI package. -""" - -from os.path import abspath, dirname -import sys - -if __name__ == '__main__': - # include ../../ for ipasetup.py - sys.path.append(dirname(dirname(dirname(abspath(__file__))))) - from ipasetup import ipasetup # noqa: E402 - - ipasetup( - name='ipaplatform', - doc = __doc__, - packages=[ - "ipaplatform", - ], - install_requires=[ - "ipaclient", - ] - ) diff --git a/pypi/test_placeholder.py b/pypi/test_placeholder.py index d17b23af4..2196f683c 100644 --- a/pypi/test_placeholder.py +++ b/pypi/test_placeholder.py @@ -9,13 +9,14 @@ import pytest @pytest.mark.parametrize("modname", [ # placeholder packages raise ImportError - 'ipaplatform', 'ipaserver', 'ipatests', # PyPI packages do not have install subpackage 'ipaclient.install', 'ipalib.install', 'ipapython.install', + # override module should not be shipped in wheels + 'ipaplatform.override', ]) def test_fail_import(modname): try: @@ -29,6 +30,7 @@ def test_fail_import(modname): @pytest.mark.parametrize("modname", [ 'ipaclient', 'ipalib', + 'ipaplatform', 'ipapython', ]) def test_import(modname):