Adding a central "file finder" to normalise the finding and exclusion…

…/inclusion of files, modules and packages [refs #26]

prospector/finder.py

@@ -0,0 +1,84 @@
+import os
+
+
+class FoundFiles(object):
+    def __init__(self, rootpath, files, modules, packages, directories):
+        self.rootpath = rootpath
+        self.files = files
+        self.modules = modules
+        self.packages = packages
+        self.directories = directories
+
+    def check_module(self, filepath, abspath=True):
+        path = os.path.relpath(filepath, self.rootpath) if abspath else filepath
+        return path in self.modules
+
+    def check_package(self, filepath, abspath=True):
+        path = os.path.relpath(filepath, self.rootpath) if abspath else filepath
+        return path in self.packages
+
+    def check_file(self, filepath, abspath=True):
+        path = os.path.relpath(filepath, self.rootpath) if abspath else filepath
+        return path in self.files
+
+    def iter_file_paths(self):
+        for filepath in self.files:
+            yield(os.path.abspath(os.path.join(self.rootpath, filepath)))
+
+    def iter_package_paths(self):
+        for package in self.packages:
+            yield(os.path.abspath(os.path.join(self.rootpath, package)))
+
+    def iter_module_paths(self):
+        for module in self.modules:
+            yield(os.path.abspath(os.path.join(self.rootpath, module)))
+
+
+def _find_paths(ignore, curpath, rootpath):
+    files, modules, packages, directories = [], [], [], []
+
+    for filename in os.listdir(curpath):
+        if filename.startswith('.'):
+            continue
+
+        fullpath = os.path.join(curpath, filename)
+        relpath = os.path.relpath(fullpath, rootpath)
+
+        if any([m.search(relpath) for m in ignore]):
+            continue
+
+        if os.path.islink(fullpath):
+            continue
+
+        if os.path.isdir(fullpath):
+            # this is a directory, is it also a package?
+            directories.append(relpath)
+            initpy = os.path.join(fullpath, '__init__.py')
+            if os.path.exists(initpy) and os.path.isfile(initpy):
+                packages.append(relpath)
+
+            # do the same for this directory
+            recurse = _find_paths(ignore, os.path.join(curpath, filename), rootpath)
+            files += recurse[0]
+            modules += recurse[1]
+            packages += recurse[2]
+            directories += recurse[3]
+
+        else:
+            # this is a file, is it a python module?
+            if fullpath.endswith('.py'):
+                modules.append(relpath)
+            files.append(relpath)
+
+    return files, modules, packages, directories
+
+
+def find_python(ignores, dirpath):
+    """
+    Returns a FoundFiles class containing a list of files, packages, directories,
+    where files are simply all python (.py) files, packages are directories
+    containing an `__init__.py` file, and directories is a list of all directories.
+    All paths are relative to the dirpath argument.
+    """
+    files, modules, directories, packages = _find_paths(ignores, dirpath, dirpath)
+    return FoundFiles(dirpath, files, modules, directories, packages)

prospector/run.py

@@ -11,6 +11,7 @@
 from prospector.autodetect import autodetect_libraries
 from prospector.formatters import FORMATTERS
 from prospector.message import Location, Message
+from prospector.finder import find_python
 
 
 __all__ = (
@@ -126,9 +127,13 @@ def execute(self):
             'tools': self.config.tools,
         }
 
+        # Find the files and packages in a common way, so that each tool
+        # gets the same list.
+        found_files = find_python(self.ignores, self.path)
+
         # Prep the tools.
         for tool in self.tool_runners:
-            tool.prepare(self.path, self.ignores, self.config, self.adaptors)
+            tool.prepare(found_files, self.config, self.adaptors)
 
         # Run the tools
         messages = []

prospector/tools/base.py

@@ -2,7 +2,7 @@
 
 class ToolBase(object):
 
-    def prepare(self, rootpath, ignore, args, adaptors):
+    def prepare(self, found_files, args, adaptors):
         pass
 
     def run(self):

prospector/tools/dodgy/__init__.py

@@ -1,8 +1,8 @@
 from __future__ import absolute_import
-
-from dodgy.run import run_checks
+import mimetypes
 import os
 import re
+from dodgy.run import check_file
 from prospector.message import Location, Message
 from prospector.tools.base import ToolBase
 
@@ -15,14 +15,19 @@ def module_from_path(path):
 
 class DodgyTool(ToolBase):
 
-    def prepare(self, rootpath, ignore, args, adaptors):
-        self.rootpath = rootpath
-        self.ignore = ignore
+    def prepare(self, found_files, args, adaptors):
+        self._files = found_files
 
     def run(self):
-        warnings = run_checks(self.rootpath, self.ignore)
-        messages = []
 
+        warnings = []
+        for filepath in self._files.iter_file_paths():
+            mimetype = mimetypes.guess_type(filepath)
+            if mimetype[0] is None or not mimetype[0].startswith('text/'):
+                continue
+            warnings += check_file(filepath)
+
+        messages = []
         for warning in warnings:
             path = warning['path']
             loc = Location(path, module_from_path(path), '', warning['line'], 0, absolute_path=False)

prospector/tools/frosted/__init__.py

@@ -74,22 +74,16 @@ def __init__(self, *args, **kwargs):
         self._paths = []
         self._ignores = []
 
-    def prepare(self, rootpath, ignore, args, adaptors):
-        self._paths = [rootpath]
-        self._rootpath = rootpath
-        self._ignores = ignore
+    def prepare(self, found_files, args, adaptors):
+        self._files = found_files
 
         for adaptor in adaptors:
             adaptor.adapt_frosted(self)
 
     def run(self):
         reporter = ProspectorReporter(ignore=self.ignore_codes)
 
-        for filepath in iter_source_code(self._paths):
-            relpath = os.path.relpath(filepath, self._rootpath)
-            if any([ip.search(relpath) for ip in self._ignores]):
-                continue
-
+        for filepath in self._files.iter_module_paths():
             # Frosted cannot handle non-utf-8 encoded files at the moment -
             # see https://github.com/timothycrosley/frosted/issues/53
             # Therefore (since pyflakes overlaps heavily and does not have the same

prospector/tools/mccabe/__init__.py

@@ -1,8 +1,6 @@
 from __future__ import absolute_import
 
 import ast
-import os.path
-
 from mccabe import PathGraphingAstVisitor
 
 from prospector.message import Location, Message
@@ -14,28 +12,15 @@
 )
 
 
-def _find_code_files(rootpath, ignores):
-    code_files = []
-
-    for root, _, files in os.walk(rootpath):
-        for potential in files:
-            fullpath = os.path.join(root, potential)
-            relpath = os.path.relpath(fullpath, rootpath)
-            if potential.endswith('.py') and not any([ip.search(relpath) for ip in ignores]):
-                code_files.append(fullpath)
-
-    return code_files
-
-
 class McCabeTool(ToolBase):
     def __init__(self, *args, **kwargs):
         super(McCabeTool, self).__init__(*args, **kwargs)
         self._code_files = []
         self.ignore_codes = ()
         self.max_complexity = 10
 
-    def prepare(self, rootpath, ignore, args, adaptors):
-        self._code_files = _find_code_files(rootpath, ignore)
+    def prepare(self, found_files, args, adaptors):
+        self._code_files = list(found_files.iter_module_paths())
 
         for adaptor in adaptors:
             adaptor.adapt_mccabe(self)

prospector/tools/pep8/__init__.py

@@ -59,10 +59,8 @@ def get_messages(self):
 
 
 class ProspectorStyleGuide(StyleGuide):
-    def __init__(self, rootpath, *args, **kwargs):
-        # Remember the ignore patterns for later.
-        self._rootpath = rootpath
-        self._ignore_patterns = kwargs.pop('ignore_patterns', [])
+    def __init__(self, found_files, *args, **kwargs):
+        self._files = found_files
 
         # Override the default reporter with our custom one.
         kwargs['reporter'] = ProspectorReport
@@ -75,27 +73,26 @@ def excluded(self, filename, parent=None):
 
         # If the file survived pep8's exclusion rules, check it against
         # prospector's patterns.
-        fullpath = os.path.join(parent, filename) if parent else filename
-        relpath = os.path.relpath(fullpath, self._rootpath)
-        if any([ip.search(relpath) for ip in self._ignore_patterns]):
-            return True
+        if os.path.isdir(os.path.join(self._files.rootpath, filename)):
+            return False
 
-        return False
+        fullpath = os.path.join(self._files.rootpath, parent, filename) if parent else filename
+        return fullpath not in self._files.iter_module_paths()
 
 
 class Pep8Tool(ToolBase):
     def __init__(self, *args, **kwargs):
         super(Pep8Tool, self).__init__(*args, **kwargs)
         self.checker = None
 
-    def prepare(self, rootpath, ignore, args, adaptors):
+    def prepare(self, found_files, args, adaptors):
         # figure out if we should use a pre-existing config file
         # such as setup.cfg or tox.ini
         external_config = None
 
         # 'none' means we ignore any external config, so just carry on
         if args.external_config != 'none':
-            paths = [os.path.join(rootpath, name) for name in PROJECT_CONFIG]
+            paths = [os.path.join(found_files.rootpath, name) for name in PROJECT_CONFIG]
             paths.append(DEFAULT_CONFIG)
 
             for conf_path in paths:
@@ -124,9 +121,8 @@ def prepare(self, rootpath, ignore, args, adaptors):
 
         # Instantiate our custom pep8 checker.
         self.checker = ProspectorStyleGuide(
-            rootpath=rootpath,
-            paths=[rootpath],
-            ignore_patterns=ignore,
+            paths=list(found_files.iter_package_paths()),
+            found_files=found_files,
             config_file=external_config
         )
 

prospector/tools/pyflakes/__init__.py

@@ -103,22 +103,15 @@ def __init__(self, *args, **kwargs):
         self._paths = []
         self._ignores = []
 
-    def prepare(self, rootpath, ignore, args, adaptors):
-        self._paths = [rootpath]
-        self._rootpath = rootpath
-        self._ignores = ignore
+    def prepare(self, found_files, args, adaptors):
+        self._files = found_files
 
         for adaptor in adaptors:
             adaptor.adapt_pyflakes(self)
 
     def run(self):
         reporter = ProspectorReporter(ignore=self.ignore_codes)
-
-        for filepath in iterSourceCode(self._paths):
-            relpath = os.path.relpath(filepath, self._rootpath)
-            if any([ip.search(relpath) for ip in self._ignores]):
-                continue
-
+        for filepath in self._files.iter_module_paths():
             checkPath(filepath, reporter)
 
         return reporter.get_messages()

prospector/tools/pylint/__init__.py

@@ -79,11 +79,41 @@ def __init__(self):
         self._collector = self._linter = None
         self._orig_sys_path = []
 
-    def prepare(self, rootpath, ignore, args, adaptors):
-        linter = ProspectorLinter(ignore, rootpath)
+    def prepare(self, found_files, args, adaptors):
+
+        linter = ProspectorLinter(found_files)
         linter.load_default_plugins()
 
-        extra_sys_path, check_paths = _find_package_paths(ignore, rootpath)
+        extra_sys_path = set()
+        extra_sys_path |= set(found_files.iter_package_paths())
+        for filepath in found_files.iter_module_paths():
+            extra_sys_path.add(os.path.dirname(filepath))
+
+        # create a list of packages, but don't include packages which are
+        # subpackages of others as checks will be duplicated
+        packages = [p.split(os.path.sep) for p in found_files.packages]
+        packages.sort(key=lambda x: len(x))
+        check_paths = set()
+        for package in packages:
+            package_path = os.path.join(*package)
+            if len(package) == 1:
+                check_paths.add(package_path)
+                continue
+            for i in range(1, len(package)):
+                if os.path.join(*package[:-i]) in check_paths:
+                    break
+            else:
+                check_paths.add(package_path)
+
+        for filepath in found_files.modules:
+            package = os.path.dirname(filepath).split(os.path.sep)
+            for i in range(0, len(package)):
+                if os.path.join(*package[:i+1]) in check_paths:
+                    break
+            else:
+                check_paths.add(filepath)
+
+        check_paths = [os.path.abspath(os.path.join(found_files.rootpath, p)) for p in check_paths]
 
         # insert the target path into the system path to get correct behaviour
         self._orig_sys_path = sys.path

prospector/tools/pylint/linter.py

@@ -7,9 +7,8 @@
 
 class ProspectorLinter(PyLinter):  # pylint: disable=R0901,R0904
 
-    def __init__(self, ignore, rootpath, *args, **kwargs):
-        self._ignore = ignore
-        self._rootpath = rootpath
+    def __init__(self, found_files, *args, **kwargs):
+        self._files = found_files
 
         # set up the standard PyLint linter
         PyLinter.__init__(self, *args, **kwargs)
@@ -25,8 +24,6 @@ def expand_files(self, modules):
         expanded = PyLinter.expand_files(self, modules)
         filtered = []
         for module in expanded:
-            rel_path = os.path.relpath(module['path'], self._rootpath)
-            if any([m.search(rel_path) for m in self._ignore]):
-                continue
-            filtered.append(module)
+            if self._files.check_module(module['path']):
+                filtered.append(module)
         return filtered