diff --git a/prospector/finder.py b/prospector/finder.py new file mode 100644 index 00000000..066bf0db --- /dev/null +++ b/prospector/finder.py @@ -0,0 +1,84 @@ +import os + + +class FoundFiles(object): + def __init__(self, rootpath, files, modules, packages, directories): + self.rootpath = rootpath + self.files = files + self.modules = modules + self.packages = packages + self.directories = directories + + def check_module(self, filepath, abspath=True): + path = os.path.relpath(filepath, self.rootpath) if abspath else filepath + return path in self.modules + + def check_package(self, filepath, abspath=True): + path = os.path.relpath(filepath, self.rootpath) if abspath else filepath + return path in self.packages + + def check_file(self, filepath, abspath=True): + path = os.path.relpath(filepath, self.rootpath) if abspath else filepath + return path in self.files + + def iter_file_paths(self): + for filepath in self.files: + yield(os.path.abspath(os.path.join(self.rootpath, filepath))) + + def iter_package_paths(self): + for package in self.packages: + yield(os.path.abspath(os.path.join(self.rootpath, package))) + + def iter_module_paths(self): + for module in self.modules: + yield(os.path.abspath(os.path.join(self.rootpath, module))) + + +def _find_paths(ignore, curpath, rootpath): + files, modules, packages, directories = [], [], [], [] + + for filename in os.listdir(curpath): + if filename.startswith('.'): + continue + + fullpath = os.path.join(curpath, filename) + relpath = os.path.relpath(fullpath, rootpath) + + if any([m.search(relpath) for m in ignore]): + continue + + if os.path.islink(fullpath): + continue + + if os.path.isdir(fullpath): + # this is a directory, is it also a package? + directories.append(relpath) + initpy = os.path.join(fullpath, '__init__.py') + if os.path.exists(initpy) and os.path.isfile(initpy): + packages.append(relpath) + + # do the same for this directory + recurse = _find_paths(ignore, os.path.join(curpath, filename), rootpath) + files += recurse[0] + modules += recurse[1] + packages += recurse[2] + directories += recurse[3] + + else: + # this is a file, is it a python module? + if fullpath.endswith('.py'): + modules.append(relpath) + files.append(relpath) + + return files, modules, packages, directories + + +def find_python(ignores, dirpath): + """ + Returns a FoundFiles class containing a list of files, packages, directories, + where files are simply all python (.py) files, packages are directories + containing an `__init__.py` file, and directories is a list of all directories. + All paths are relative to the dirpath argument. + """ + files, modules, directories, packages = _find_paths(ignores, dirpath, dirpath) + return FoundFiles(dirpath, files, modules, directories, packages) diff --git a/prospector/run.py b/prospector/run.py index 2df89808..69fb7da0 100644 --- a/prospector/run.py +++ b/prospector/run.py @@ -11,6 +11,7 @@ from prospector.autodetect import autodetect_libraries from prospector.formatters import FORMATTERS from prospector.message import Location, Message +from prospector.finder import find_python __all__ = ( @@ -126,9 +127,13 @@ def execute(self): 'tools': self.config.tools, } + # Find the files and packages in a common way, so that each tool + # gets the same list. + found_files = find_python(self.ignores, self.path) + # Prep the tools. for tool in self.tool_runners: - tool.prepare(self.path, self.ignores, self.config, self.adaptors) + tool.prepare(found_files, self.config, self.adaptors) # Run the tools messages = [] diff --git a/prospector/tools/base.py b/prospector/tools/base.py index 278a467d..65dabd0b 100644 --- a/prospector/tools/base.py +++ b/prospector/tools/base.py @@ -2,7 +2,7 @@ class ToolBase(object): - def prepare(self, rootpath, ignore, args, adaptors): + def prepare(self, found_files, args, adaptors): pass def run(self): diff --git a/prospector/tools/dodgy/__init__.py b/prospector/tools/dodgy/__init__.py index aefd4f9d..da66eea9 100644 --- a/prospector/tools/dodgy/__init__.py +++ b/prospector/tools/dodgy/__init__.py @@ -1,8 +1,8 @@ from __future__ import absolute_import - -from dodgy.run import run_checks +import mimetypes import os import re +from dodgy.run import check_file from prospector.message import Location, Message from prospector.tools.base import ToolBase @@ -15,14 +15,19 @@ def module_from_path(path): class DodgyTool(ToolBase): - def prepare(self, rootpath, ignore, args, adaptors): - self.rootpath = rootpath - self.ignore = ignore + def prepare(self, found_files, args, adaptors): + self._files = found_files def run(self): - warnings = run_checks(self.rootpath, self.ignore) - messages = [] + warnings = [] + for filepath in self._files.iter_file_paths(): + mimetype = mimetypes.guess_type(filepath) + if mimetype[0] is None or not mimetype[0].startswith('text/'): + continue + warnings += check_file(filepath) + + messages = [] for warning in warnings: path = warning['path'] loc = Location(path, module_from_path(path), '', warning['line'], 0, absolute_path=False) diff --git a/prospector/tools/frosted/__init__.py b/prospector/tools/frosted/__init__.py index 4ea47c71..e120eb3b 100644 --- a/prospector/tools/frosted/__init__.py +++ b/prospector/tools/frosted/__init__.py @@ -74,10 +74,8 @@ def __init__(self, *args, **kwargs): self._paths = [] self._ignores = [] - def prepare(self, rootpath, ignore, args, adaptors): - self._paths = [rootpath] - self._rootpath = rootpath - self._ignores = ignore + def prepare(self, found_files, args, adaptors): + self._files = found_files for adaptor in adaptors: adaptor.adapt_frosted(self) @@ -85,11 +83,7 @@ def prepare(self, rootpath, ignore, args, adaptors): def run(self): reporter = ProspectorReporter(ignore=self.ignore_codes) - for filepath in iter_source_code(self._paths): - relpath = os.path.relpath(filepath, self._rootpath) - if any([ip.search(relpath) for ip in self._ignores]): - continue - + for filepath in self._files.iter_module_paths(): # Frosted cannot handle non-utf-8 encoded files at the moment - # see https://github.com/timothycrosley/frosted/issues/53 # Therefore (since pyflakes overlaps heavily and does not have the same diff --git a/prospector/tools/mccabe/__init__.py b/prospector/tools/mccabe/__init__.py index 6f9f82e5..54001098 100644 --- a/prospector/tools/mccabe/__init__.py +++ b/prospector/tools/mccabe/__init__.py @@ -1,8 +1,6 @@ from __future__ import absolute_import import ast -import os.path - from mccabe import PathGraphingAstVisitor from prospector.message import Location, Message @@ -14,19 +12,6 @@ ) -def _find_code_files(rootpath, ignores): - code_files = [] - - for root, _, files in os.walk(rootpath): - for potential in files: - fullpath = os.path.join(root, potential) - relpath = os.path.relpath(fullpath, rootpath) - if potential.endswith('.py') and not any([ip.search(relpath) for ip in ignores]): - code_files.append(fullpath) - - return code_files - - class McCabeTool(ToolBase): def __init__(self, *args, **kwargs): super(McCabeTool, self).__init__(*args, **kwargs) @@ -34,8 +19,8 @@ def __init__(self, *args, **kwargs): self.ignore_codes = () self.max_complexity = 10 - def prepare(self, rootpath, ignore, args, adaptors): - self._code_files = _find_code_files(rootpath, ignore) + def prepare(self, found_files, args, adaptors): + self._code_files = list(found_files.iter_module_paths()) for adaptor in adaptors: adaptor.adapt_mccabe(self) diff --git a/prospector/tools/pep8/__init__.py b/prospector/tools/pep8/__init__.py index 931fc02d..08c82bcf 100644 --- a/prospector/tools/pep8/__init__.py +++ b/prospector/tools/pep8/__init__.py @@ -59,10 +59,8 @@ def get_messages(self): class ProspectorStyleGuide(StyleGuide): - def __init__(self, rootpath, *args, **kwargs): - # Remember the ignore patterns for later. - self._rootpath = rootpath - self._ignore_patterns = kwargs.pop('ignore_patterns', []) + def __init__(self, found_files, *args, **kwargs): + self._files = found_files # Override the default reporter with our custom one. kwargs['reporter'] = ProspectorReport @@ -75,12 +73,11 @@ def excluded(self, filename, parent=None): # If the file survived pep8's exclusion rules, check it against # prospector's patterns. - fullpath = os.path.join(parent, filename) if parent else filename - relpath = os.path.relpath(fullpath, self._rootpath) - if any([ip.search(relpath) for ip in self._ignore_patterns]): - return True + if os.path.isdir(os.path.join(self._files.rootpath, filename)): + return False - return False + fullpath = os.path.join(self._files.rootpath, parent, filename) if parent else filename + return fullpath not in self._files.iter_module_paths() class Pep8Tool(ToolBase): @@ -88,14 +85,14 @@ def __init__(self, *args, **kwargs): super(Pep8Tool, self).__init__(*args, **kwargs) self.checker = None - def prepare(self, rootpath, ignore, args, adaptors): + def prepare(self, found_files, args, adaptors): # figure out if we should use a pre-existing config file # such as setup.cfg or tox.ini external_config = None # 'none' means we ignore any external config, so just carry on if args.external_config != 'none': - paths = [os.path.join(rootpath, name) for name in PROJECT_CONFIG] + paths = [os.path.join(found_files.rootpath, name) for name in PROJECT_CONFIG] paths.append(DEFAULT_CONFIG) for conf_path in paths: @@ -124,9 +121,8 @@ def prepare(self, rootpath, ignore, args, adaptors): # Instantiate our custom pep8 checker. self.checker = ProspectorStyleGuide( - rootpath=rootpath, - paths=[rootpath], - ignore_patterns=ignore, + paths=list(found_files.iter_package_paths()), + found_files=found_files, config_file=external_config ) diff --git a/prospector/tools/pyflakes/__init__.py b/prospector/tools/pyflakes/__init__.py index 01277231..ff8adf39 100644 --- a/prospector/tools/pyflakes/__init__.py +++ b/prospector/tools/pyflakes/__init__.py @@ -103,22 +103,15 @@ def __init__(self, *args, **kwargs): self._paths = [] self._ignores = [] - def prepare(self, rootpath, ignore, args, adaptors): - self._paths = [rootpath] - self._rootpath = rootpath - self._ignores = ignore + def prepare(self, found_files, args, adaptors): + self._files = found_files for adaptor in adaptors: adaptor.adapt_pyflakes(self) def run(self): reporter = ProspectorReporter(ignore=self.ignore_codes) - - for filepath in iterSourceCode(self._paths): - relpath = os.path.relpath(filepath, self._rootpath) - if any([ip.search(relpath) for ip in self._ignores]): - continue - + for filepath in self._files.iter_module_paths(): checkPath(filepath, reporter) return reporter.get_messages() diff --git a/prospector/tools/pylint/__init__.py b/prospector/tools/pylint/__init__.py index 59eee83a..c55f29c2 100644 --- a/prospector/tools/pylint/__init__.py +++ b/prospector/tools/pylint/__init__.py @@ -79,11 +79,41 @@ def __init__(self): self._collector = self._linter = None self._orig_sys_path = [] - def prepare(self, rootpath, ignore, args, adaptors): - linter = ProspectorLinter(ignore, rootpath) + def prepare(self, found_files, args, adaptors): + + linter = ProspectorLinter(found_files) linter.load_default_plugins() - extra_sys_path, check_paths = _find_package_paths(ignore, rootpath) + extra_sys_path = set() + extra_sys_path |= set(found_files.iter_package_paths()) + for filepath in found_files.iter_module_paths(): + extra_sys_path.add(os.path.dirname(filepath)) + + # create a list of packages, but don't include packages which are + # subpackages of others as checks will be duplicated + packages = [p.split(os.path.sep) for p in found_files.packages] + packages.sort(key=lambda x: len(x)) + check_paths = set() + for package in packages: + package_path = os.path.join(*package) + if len(package) == 1: + check_paths.add(package_path) + continue + for i in range(1, len(package)): + if os.path.join(*package[:-i]) in check_paths: + break + else: + check_paths.add(package_path) + + for filepath in found_files.modules: + package = os.path.dirname(filepath).split(os.path.sep) + for i in range(0, len(package)): + if os.path.join(*package[:i+1]) in check_paths: + break + else: + check_paths.add(filepath) + + check_paths = [os.path.abspath(os.path.join(found_files.rootpath, p)) for p in check_paths] # insert the target path into the system path to get correct behaviour self._orig_sys_path = sys.path diff --git a/prospector/tools/pylint/linter.py b/prospector/tools/pylint/linter.py index bf079481..845bca27 100644 --- a/prospector/tools/pylint/linter.py +++ b/prospector/tools/pylint/linter.py @@ -7,9 +7,8 @@ class ProspectorLinter(PyLinter): # pylint: disable=R0901,R0904 - def __init__(self, ignore, rootpath, *args, **kwargs): - self._ignore = ignore - self._rootpath = rootpath + def __init__(self, found_files, *args, **kwargs): + self._files = found_files # set up the standard PyLint linter PyLinter.__init__(self, *args, **kwargs) @@ -25,8 +24,6 @@ def expand_files(self, modules): expanded = PyLinter.expand_files(self, modules) filtered = [] for module in expanded: - rel_path = os.path.relpath(module['path'], self._rootpath) - if any([m.search(rel_path) for m in self._ignore]): - continue - filtered.append(module) + if self._files.check_module(module['path']): + filtered.append(module) return filtered