diff --git a/Makefile b/Makefile index d2857ae47..8c1a5dea7 100644 --- a/Makefile +++ b/Makefile @@ -54,7 +54,9 @@ LIBDIR ?= /usr/lib DEVELOPER_MODE ?= 0 ifneq ($(DEVELOPER_MODE),0) -LINT_OPTIONS=--no-fail +LINT_IGNORE_FAIL=true +else +LINT_IGNORE_FAIL=false endif PYTHON ?= $(shell rpm -E %__python || echo /usr/bin/python2) @@ -127,8 +129,14 @@ client-dirs: fi lint: bootstrap-autogen - ./make-lint $(LINT_OPTIONS) - $(MAKE) -C install/po validate-src-strings + # find all python modules and executable python files outside modules for pylint check + FILES=`find . \ + -type d -exec test -e '{}/__init__.py' \; -print -prune -o \ + -name \*.py -print -o \ + -type f \! -path '*/.*' \! -name '*~' -exec grep -qsm1 '^#!.*\bpython' '{}' \; -print`; \ + echo "Pylint is running, please wait ..."; \ + PYTHONPATH=. pylint --rcfile=pylintrc $(PYLINTFLAGS) $$FILES || $(LINT_IGNORE_FAIL) + $(MAKE) -C install/po validate-src-strings || $(LINT_IGNORE_FAIL) test: diff --git a/ipalib/plugins/vault.py b/ipalib/plugins/vault.py index cbfa1e630..22fb3e2a3 100644 --- a/ipalib/plugins/vault.py +++ b/ipalib/plugins/vault.py @@ -1653,7 +1653,9 @@ class vault_archive(PKQuery, Local): session_key = slot.key_gen(mechanism, None, key_length) # wrap session key with transport certificate + # pylint: disable=no-member public_key = nss_transport_cert.subject_public_key_info.public_key + # pylint: enable=no-member wrapped_session_key = nss.pub_wrap_sym_key(mechanism, public_key, session_key) @@ -1857,7 +1859,9 @@ class vault_retrieve(PKQuery, Local): session_key = slot.key_gen(mechanism, None, key_length) # wrap session key with transport certificate + # pylint: disable=no-member public_key = nss_transport_cert.subject_public_key_info.public_key + # pylint: enable=no-member wrapped_session_key = nss.pub_wrap_sym_key(mechanism, public_key, session_key) diff --git a/ipatests/test_ipapython/test_ipautil.py b/ipatests/test_ipapython/test_ipautil.py index f91b730c5..a945179b5 100644 --- a/ipatests/test_ipapython/test_ipautil.py +++ b/ipatests/test_ipapython/test_ipautil.py @@ -386,6 +386,8 @@ class TestTimeParser(object): nose.tools.assert_equal(800000, time.microsecond) def test_time_zones(self): + # pylint: disable=no-member + timestr = "20051213141205Z" time = ipautil.parse_generalized_time(timestr) diff --git a/make-lint b/make-lint deleted file mode 100755 index 74d20691a..000000000 --- a/make-lint +++ /dev/null @@ -1,293 +0,0 @@ -#!/usr/bin/python2 -# -# Authors: -# Jakub Hrozek -# Jan Cholasta -# -# Copyright (C) 2011 Red Hat -# see file 'COPYING' for use and warranty information -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from __future__ import print_function - -import os -import sys -from optparse import OptionParser -from fnmatch import fnmatch, fnmatchcase -import subprocess - -try: - from pylint import checkers - from pylint.lint import PyLinter - from pylint.checkers.typecheck import TypeChecker - from pylint.checkers.utils import safe_infer - from astroid import Class, Instance, Module, InferenceError, Function - from pylint.reporters.text import TextReporter -except ImportError: - print("To use {0}, please install pylint.".format(sys.argv[0]), file=sys.stderr) - sys.exit(32) - -# File names to ignore when searching for python source files -IGNORE_FILES = ('.*', '*~', '*.in', '*.pyc', '*.pyo') -IGNORE_PATHS = ( - 'build', 'rpmbuild', 'dist', 'install/po/test_i18n.py', 'lite-server.py') - -class IPATypeChecker(TypeChecker): - NAMESPACE_ATTRS = ['Command', 'Object', 'Method', 'Backend', 'Updater', - 'Advice'] - LOGGING_ATTRS = ['log', 'debug', 'info', 'warning', 'error', 'exception', - 'critical'] - - # 'class or module': ['generated', 'properties'] - ignore = { - # Python standard library & 3rd party classes - 'socket._socketobject': ['sendall'], - # should be 'subprocess.Popen' - '.Popen': ['stdin', 'stdout', 'stderr', 'pid', 'returncode', 'poll', - 'wait', 'communicate'], - 'urlparse.ResultMixin': ['scheme', 'netloc', 'path', 'query', - 'fragment', 'username', 'password', 'hostname', 'port'], - 'urlparse.ParseResult': ['params'], - 'pytest': ['fixture', 'raises', 'skip', 'yield_fixture', 'mark', 'fail'], - 'unittest.case': ['assertEqual', 'assertRaises'], - 'nose.tools': ['assert_equal', 'assert_raises'], - 'datetime.tzinfo': ['houroffset', 'minoffset', 'utcoffset', 'dst'], - 'nss.nss.subject_public_key_info': ['public_key'], - - # IPA classes - 'ipalib.base.NameSpace': ['add', 'mod', 'del', 'show', 'find'], - 'ipalib.cli.Collector': ['__options'], - 'ipalib.config.Env': ['*'], - 'ipalib.parameters.Param': ['cli_name', 'cli_short_name', 'label', - 'default', 'doc', 'required', 'multivalue', 'primary_key', - 'normalizer', 'default_from', 'autofill', 'query', 'attribute', - 'include', 'exclude', 'flags', 'hint', 'alwaysask', 'sortorder', - 'csv', 'option_group'], - 'ipalib.parameters.Bool': ['truths', 'falsehoods'], - 'ipalib.parameters.Data': ['minlength', 'maxlength', 'length', - 'pattern', 'pattern_errmsg'], - 'ipalib.parameters.Str': ['noextrawhitespace'], - 'ipalib.parameters.Password': ['confirm'], - 'ipalib.parameters.File': ['stdin_if_missing'], - 'ipalib.plugins.dns.DNSRecord': ['validatedns', 'normalizedns'], - 'ipalib.parameters.Enum': ['values'], - 'ipalib.parameters.Number': ['minvalue', 'maxvalue'], - 'ipalib.parameters.Decimal': ['precision', 'exponential', - 'numberclass'], - 'ipalib.parameters.DNSNameParam': ['only_absolute', 'only_relative'], - 'ipalib.plugable.API': NAMESPACE_ATTRS + LOGGING_ATTRS, - 'ipalib.plugable.Plugin': ['api', 'env'] + NAMESPACE_ATTRS + - LOGGING_ATTRS, - 'ipalib.session.AuthManager': LOGGING_ATTRS, - 'ipalib.session.SessionAuthManager': LOGGING_ATTRS, - 'ipalib.session.SessionManager': LOGGING_ATTRS, - 'ipaserver.install.ldapupdate.LDAPUpdate': LOGGING_ATTRS, - 'ipaserver.rpcserver.KerberosSession': ['api'] + LOGGING_ATTRS, - 'ipatests.test_integration.base.IntegrationTest': [ - 'domain', 'master', 'replicas', 'clients', 'ad_domains'] - } - - def _related_classes(self, klass): - yield klass - for base in klass.ancestors(): - yield base - - def _class_full_name(self, klass): - return klass.root().name + '.' + klass.name - - def _find_ignored_attrs(self, owner): - attrs = [] - for klass in self._related_classes(owner): - name = self._class_full_name(klass) - if name in self.ignore: - attrs += self.ignore[name] - return attrs - - def visit_getattr(self, node): - try: - inferred = list(node.expr.infer()) - except InferenceError: - inferred = [] - - for owner in inferred: - if isinstance(owner, Module): - if node.attrname in self.ignore.get(owner.name, ()): - return - - elif isinstance(owner, Class) or type(owner) is Instance: - ignored = self._find_ignored_attrs(owner) - for pattern in ignored: - if fnmatchcase(node.attrname, pattern): - return - - super(IPATypeChecker, self).visit_getattr(node) - - def visit_callfunc(self, node): - called = safe_infer(node.func) - if isinstance(called, Function): - if called.name in self.ignore.get(called.root().name, []): - return - - super(IPATypeChecker, self).visit_callfunc(node) - -class IPALinter(PyLinter): - ignore = (TypeChecker,) - - def __init__(self): - super(IPALinter, self).__init__() - - self.missing = set() - - def register_checker(self, checker): - if type(checker) in self.ignore: - return - super(IPALinter, self).register_checker(checker) - - def add_message(self, msg_id, line=None, node=None, args=None, confidence=None): - if line is None and node is not None: - line = node.fromlineno - - # Record missing packages - if msg_id == 'F0401' and self.is_message_enabled(msg_id, line): - self.missing.add(args) - - super(IPALinter, self).add_message(msg_id, line, node, args) - -def find_files(path, basepath): - entries = os.listdir(path) - - # If this directory is a python package, look no further - if '__init__.py' in entries: - return [path] - - result = [] - for filename in entries: - filepath = os.path.join(path, filename) - - for pattern in IGNORE_FILES: - if fnmatch(filename, pattern): - filename = None - break - if filename is None: - continue - - for pattern in IGNORE_PATHS: - patpath = os.path.join(basepath, pattern).replace(os.sep, '/') - if filepath == patpath: - filename = None - break - if filename is None: - continue - - if os.path.islink(filepath): - continue - - # Recurse into subdirectories - if os.path.isdir(filepath): - result += find_files(filepath, basepath) - continue - - # Add all *.py files - if filename.endswith('.py'): - result.append(filepath) - continue - - # Add any other files beginning with a shebang and having - # the word "python" on the first line - file = open(filepath, 'r') - line = file.readline(128) - file.close() - - if line[:2] == '#!' and line.find('python') >= 0: - result.append(filepath) - - return result - -def main(): - optparser = OptionParser() - optparser.add_option('--no-fail', help='report success even if errors were found', - dest='fail', default=True, action='store_false') - optparser.add_option('--enable-noerror', help='enable warnings and other non-error messages', - dest='errors_only', default=True, action='store_false') - optparser.add_option('--no-py3k', help='Do not check for Python 3 porting issues', - dest='py3k', default=True, action='store_false') - optparser.add_option('--no-lint', help='Skip the main lint check', - dest='do_lint', default=True, action='store_false') - - options, args = optparser.parse_args() - cwd = os.getcwd() - - if len(args) == 0: - files = find_files(cwd, cwd) - else: - files = args - - for filename in files: - dirname = os.path.dirname(filename) - if dirname not in sys.path: - sys.path.insert(0, dirname) - - linter = IPALinter() - checkers.initialize(linter) - linter.register_checker(IPATypeChecker(linter)) - - if options.errors_only: - linter.disable_noerror_messages() - linter.enable('F') - linter.set_reporter(TextReporter()) - linter.set_option('msg-template', - '{path}:{line}: [{msg_id}({symbol}), {obj}] {msg})') - linter.set_option('reports', False) - linter.set_option('persistent', False) - linter.set_option('disable', 'python3') - - if options.do_lint: - linter.check(files) - - if linter.msg_status != 0: - print(""" -=============================================================================== -Errors were found during the static code check. -""", file=sys.stderr) - - if len(linter.missing) > 0: - print("There are some missing imports:", file=sys.stderr) - for mod in sorted(linter.missing): - print(" " + mod, file=sys.stderr) - print(""" -Please make sure all of the required and optional (python-gssapi, python-rhsm) -python packages are installed. -""", file=sys.stderr) - - print("""\ -If you are certain that any of the reported errors are false positives, please -mark them in the source code according to the pylint documentation. -=============================================================================== -""", file=sys.stderr) - - if options.fail and linter.msg_status != 0: - return linter.msg_status - - if options.py3k: - args = ['pylint', '--py3k', '-d', 'no-absolute-import', '--reports=n'] - args.extend(files) - returncode = subprocess.call(args) - if options.fail and returncode != 0: - return returncode - - return 0 - -if __name__ == "__main__": - sys.exit(main()) diff --git a/pylint_plugins.py b/pylint_plugins.py new file mode 100644 index 000000000..c4bc1f014 --- /dev/null +++ b/pylint_plugins.py @@ -0,0 +1,210 @@ +# +# Copyright (C) 2015 FreeIPA Contributors see COPYING for license +# + +from __future__ import print_function + +import copy +import sys + +from astroid import MANAGER +from astroid import scoped_nodes + + +def register(linter): + pass + + +def _warning_already_exists(cls, member): + print( + "WARNING: member '{member}' in '{cls}' already exists".format( + cls="{}.{}".format(cls.root().name, cls.name), member=member), + file=sys.stderr + ) + + +def fake_class(name_or_class_obj, members=()): + if isinstance(name_or_class_obj, scoped_nodes.Class): + cl = name_or_class_obj + else: + cl = scoped_nodes.Class(name_or_class_obj, None) + + for m in members: + if isinstance(m, str): + if m in cl.locals: + _warning_already_exists(cl, m) + else: + cl.locals[m] = [scoped_nodes.Class(m, None)] + elif isinstance(m, dict): + for key, val in m.items(): + assert isinstance(key, str), "key must be string" + if key in cl.locals: + _warning_already_exists(cl, key) + fake_class(cl.locals[key], val) + else: + cl.locals[key] = [fake_class(key, val)] + else: + # here can be used any astroid type + if m.name in cl.locals: + _warning_already_exists(cl, m.name) + else: + cl.locals[m.name] = [copy.copy(m)] + return cl + + +fake_backend = {'Backend': [ + {'wsgi_dispatch': ['mount']}, +]} + +NAMESPACE_ATTRS = ['Command', 'Object', 'Method', fake_backend, 'Updater', + 'Advice'] +fake_api_env = {'env': [ + 'host', + 'realm', + 'session_auth_duration', + 'session_duration_type', +]} + +# this is due ipaserver.rpcserver.KerberosSession where api is undefined +fake_api = {'api': [fake_api_env] + NAMESPACE_ATTRS} + +_LOGGING_ATTRS = ['debug', 'info', 'warning', 'error', 'exception', + 'critical', 'warn'] +LOGGING_ATTRS = [ + {'log': _LOGGING_ATTRS}, +] + _LOGGING_ATTRS + +# 'class': ['generated', 'properties'] +ipa_class_members = { + # Python standard library & 3rd party classes + 'socket._socketobject': ['sendall'], + + # IPA classes + 'ipalib.base.NameSpace': [ + 'add', + 'mod', + 'del', + 'show', + 'find' + ], + 'ipalib.cli.Collector': ['__options'], + 'ipalib.config.Env': [ + {'__d': ['get']}, + {'__done': ['add']}, + 'xmlrpc_uri', + 'validate_api', + 'startup_traceback', + 'verbose' + ] + LOGGING_ATTRS, + 'ipalib.parameters.Param': [ + 'cli_name', + 'cli_short_name', + 'label', + 'default', + 'doc', + 'required', + 'multivalue', + 'primary_key', + 'normalizer', + 'default_from', + 'autofill', + 'query', + 'attribute', + 'include', + 'exclude', + 'flags', + 'hint', + 'alwaysask', + 'sortorder', + 'csv', + 'option_group', + ], + 'ipalib.parameters.Bool': [ + 'truths', + 'falsehoods'], + 'ipalib.parameters.Data': [ + 'minlength', + 'maxlength', + 'length', + 'pattern', + 'pattern_errmsg', + ], + 'ipalib.parameters.Str': ['noextrawhitespace'], + 'ipalib.parameters.Password': ['confirm'], + 'ipalib.parameters.File': ['stdin_if_missing'], + 'ipalib.plugins.dns.DNSRecord': [ + 'validatedns', + 'normalizedns', + ], + 'ipalib.parameters.Enum': ['values'], + 'ipalib.parameters.Number': [ + 'minvalue', + 'maxvalue', + ], + 'ipalib.parameters.Decimal': [ + 'precision', + 'exponential', + 'numberclass', + ], + 'ipalib.parameters.DNSNameParam': [ + 'only_absolute', + 'only_relative', + ], + 'ipalib.plugable.API': [ + fake_api_env, + ] + NAMESPACE_ATTRS + LOGGING_ATTRS, + 'ipalib.plugable.Plugin': [ + 'Object', + 'Method', + 'Updater', + 'Advice', + ] + LOGGING_ATTRS, + 'ipalib.session.AuthManager': LOGGING_ATTRS, + 'ipalib.session.SessionAuthManager': LOGGING_ATTRS, + 'ipalib.session.SessionManager': LOGGING_ATTRS, + 'ipaserver.install.ldapupdate.LDAPUpdate': LOGGING_ATTRS, + 'ipaserver.rpcserver.KerberosSession': [ + fake_api, + ] + LOGGING_ATTRS, + 'ipatests.test_integration.base.IntegrationTest': [ + 'domain', + {'master': [ + {'config': [ + {'dirman_password': dir(str)}, + {'admin_password': dir(str)}, + {'admin_name': dir(str)}, + {'dns_forwarder': dir(str)}, + {'test_dir': dir(str)}, + {'ad_admin_name': dir(str)}, + {'ad_admin_password': dir(str)}, + {'domain_level': dir(str)}, + ]}, + {'domain': [ + {'realm': dir(str)}, + {'name': dir(str)}, + ]}, + 'hostname', + 'ip', + 'collect_log', + {'run_command': [ + {'stdout_text': dir(str)}, + 'stderr_text', + 'returncode', + ]}, + {'transport': ['put_file']}, + 'put_file_contents', + 'get_file_contents', + ]}, + 'replicas', + 'clients', + 'ad_domains', + ] +} + + +def fix_ipa_classes(cls): + class_name_with_module = "{}.{}".format(cls.root().name, cls.name) + if class_name_with_module in ipa_class_members: + fake_class(cls, ipa_class_members[class_name_with_module]) + +MANAGER.register_transform(scoped_nodes.Class, fix_ipa_classes) diff --git a/pylintrc b/pylintrc new file mode 100644 index 000000000..01ae889ab --- /dev/null +++ b/pylintrc @@ -0,0 +1,97 @@ +[MASTER] +# Pickle collected data for later comparisons. +persistent=no + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +load-plugins=pylint_plugins + +# Use multiple processes to speed up Pylint. +jobs=1 + +[MESSAGES CONTROL] + +enable= + all, + python3 + +disable= + R, + I, + invalid-name, + import-error, + abstract-method, + anomalous-backslash-in-string, + arguments-differ, + attribute-defined-outside-init, + bad-builtin, + bad-indentation, + bare-except, + broad-except, + dangerous-default-value, + eval-used, + exec-used, + fixme, + global-statement, + global-variable-not-assigned, + global-variable-undefined, + no-init, + pointless-except, + pointless-statement, + pointless-string-statement, + protected-access, + redefine-in-handler, + redefined-builtin, + redefined-outer-name, + reimported, + relative-import, + super-init-not-called, + undefined-loop-variable, + unnecessary-lambda, + unnecessary-semicolon, + unused-argument, + unused-import, + unused-variable, + unused-wildcard-import, + useless-else-on-loop, + bad-classmethod-argument, + bad-continuation, + bad-mcs-classmethod-argument, + bad-mcs-method-argument, + bad-whitespace, + blacklisted-name, + invalid-name, + line-too-long, + missing-docstring, + multiple-imports, + multiple-statements, + old-style-class, + superfluous-parens, + too-many-lines, + unidiomatic-typecheck, + no-absolute-import, + wildcard-import, + unnecessary-pass, + expression-not-assigned, + unbalanced-tuple-unpacking, + missing-final-newline, + unpacking-non-sequence, + lost-exception, + empty-docstring, + trailing-whitespace, + duplicate-key, + unused-format-string-key + +[REPORTS] + +# Set the output format. Available formats are text, parseable, colorized, msvs +# (visual studio) and html. You can also give a reporter class, eg +# mypackage.mymodule.MyReporterClass. +output-format=colorized + +# Tells whether to display a full report or only the messages +reports=no + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details +msg-template='{path}:{line}: [{msg_id}({symbol}), {obj}] {msg})'