"""Finders try to find right section for passed module name """ from __future__ import absolute_import, division, print_function, unicode_literals import inspect import os import os.path import re import sys import sysconfig from fnmatch import fnmatch from glob import glob from .pie_slice import PY2 from .utils import chdir, exists_case_sensitive try: from pipreqs import pipreqs except ImportError: pipreqs = None try: # pip>=10 from pip._internal.download import PipSession from pip._internal.req import parse_requirements except ImportError: try: from pip.download import PipSession from pip.req import parse_requirements except ImportError: parse_requirements = None try: from requirementslib import Pipfile except ImportError: Pipfile = None try: from functools import lru_cache except ImportError: from backports.functools_lru_cache import lru_cache KNOWN_SECTION_MAPPING = { 'STDLIB': 'STANDARD_LIBRARY', 'FUTURE': 'FUTURE_LIBRARY', 'FIRSTPARTY': 'FIRST_PARTY', 'THIRDPARTY': 'THIRD_PARTY', } class BaseFinder(object): def __init__(self, config, sections): self.config = config self.sections = sections class ForcedSeparateFinder(BaseFinder): def find(self, module_name): for forced_separate in self.config['forced_separate']: # Ensure all forced_separate patterns will match to end of string path_glob = forced_separate if not forced_separate.endswith('*'): path_glob = '%s*' % forced_separate if fnmatch(module_name, path_glob) or fnmatch(module_name, '.' + path_glob): return forced_separate class LocalFinder(BaseFinder): def find(self, module_name): if module_name.startswith("."): return self.sections.LOCALFOLDER class KnownPatternFinder(BaseFinder): def __init__(self, config, sections): super(KnownPatternFinder, self).__init__(config, sections) self.known_patterns = [] for placement in reversed(self.sections): known_placement = KNOWN_SECTION_MAPPING.get(placement, placement) config_key = 'known_{0}'.format(known_placement.lower()) known_patterns = self.config.get(config_key, []) known_patterns = [ pattern for known_pattern in known_patterns for pattern in self._parse_known_pattern(known_pattern) ] for known_pattern in known_patterns: regexp = '^' + known_pattern.replace('*', '.*').replace('?', '.?') + '$' self.known_patterns.append((re.compile(regexp), placement)) @staticmethod def _is_package(path): """ Evaluates if path is a python package """ if PY2: return os.path.exists(os.path.join(path, '__init__.py')) else: return os.path.isdir(path) def _parse_known_pattern(self, pattern): """ Expand pattern if identified as a directory and return found sub packages """ if pattern.endswith(os.path.sep): patterns = [ filename for filename in os.listdir(pattern) if self._is_package(os.path.join(pattern, filename)) ] else: patterns = [pattern] return patterns def find(self, module_name): # Try to find most specific placement instruction match (if any) parts = module_name.split('.') module_names_to_check = ('.'.join(parts[:first_k]) for first_k in range(len(parts), 0, -1)) for module_name_to_check in module_names_to_check: for pattern, placement in self.known_patterns: if pattern.match(module_name_to_check): return placement class PathFinder(BaseFinder): def __init__(self, config, sections): super(PathFinder, self).__init__(config, sections) # restore the original import path (i.e. not the path to bin/isort) self.paths = [os.getcwd()] # virtual env self.virtual_env = self.config.get('virtual_env') or os.environ.get('VIRTUAL_ENV') if self.virtual_env: self.virtual_env = os.path.realpath(self.virtual_env) self.virtual_env_src = False if self.virtual_env: self.virtual_env_src = '{0}/src/'.format(self.virtual_env) for path in glob('{0}/lib/python*/site-packages'.format(self.virtual_env)): if path not in self.paths: self.paths.append(path) for path in glob('{0}/lib/python*/*/site-packages'.format(self.virtual_env)): if path not in self.paths: self.paths.append(path) for path in glob('{0}/src/*'.format(self.virtual_env)): if os.path.isdir(path): self.paths.append(path) # conda self.conda_env = self.config.get('conda_env') or os.environ.get('CONDA_PREFIX') if self.conda_env: self.conda_env = os.path.realpath(self.conda_env) for path in glob('{0}/lib/python*/site-packages'.format(self.conda_env)): if path not in self.paths: self.paths.append(path) for path in glob('{0}/lib/python*/*/site-packages'.format(self.conda_env)): if path not in self.paths: self.paths.append(path) # handle case-insensitive paths on windows self.stdlib_lib_prefix = os.path.normcase(sysconfig.get_paths()['stdlib']) if self.stdlib_lib_prefix not in self.paths: self.paths.append(self.stdlib_lib_prefix) # handle compiled libraries self.ext_suffix = sysconfig.get_config_var("EXT_SUFFIX") or ".so" # add system paths for path in sys.path[1:]: if path not in self.paths: self.paths.append(path) def find(self, module_name): for prefix in self.paths: package_path = "/".join((prefix, module_name.split(".")[0])) is_module = (exists_case_sensitive(package_path + ".py") or exists_case_sensitive(package_path + ".so") or exists_case_sensitive(package_path + self.ext_suffix) or exists_case_sensitive(package_path + "/__init__.py")) is_package = exists_case_sensitive(package_path) and os.path.isdir(package_path) if is_module or is_package: if 'site-packages' in prefix: return self.sections.THIRDPARTY if 'dist-packages' in prefix: return self.sections.THIRDPARTY if self.virtual_env and self.virtual_env_src in prefix: return self.sections.THIRDPARTY if self.conda_env and self.conda_env in prefix: return self.sections.THIRDPARTY if os.path.normcase(prefix).startswith(self.stdlib_lib_prefix): return self.sections.STDLIB return self.config['default_section'] class ReqsBaseFinder(BaseFinder): def __init__(self, config, sections, path='.'): super(ReqsBaseFinder, self).__init__(config, sections) self.path = path if self.enabled: self.mapping = self._load_mapping() self.names = self._load_names() @staticmethod def _load_mapping(): """Return list of mappings `package_name -> module_name` Example: django-haystack -> haystack """ if not pipreqs: return path = os.path.dirname(inspect.getfile(pipreqs)) path = os.path.join(path, 'mapping') with open(path) as f: # pypi_name: import_name return dict(line.strip().split(":")[::-1] for line in f) def _load_names(self): """Return list of thirdparty modules from requirements """ names = [] for path in self._get_files(): for name in self._get_names(path): names.append(self._normalize_name(name)) return names @staticmethod def _get_parents(path): prev = '' while path != prev: prev = path yield path path = os.path.dirname(path) def _get_files(self): """Return paths to all requirements files """ path = os.path.abspath(self.path) if os.path.isfile(path): path = os.path.dirname(path) for path in self._get_parents(path): for file_path in self._get_files_from_dir(path): yield file_path def _normalize_name(self, name): """Convert package name to module name Examples: Django -> django django-haystack -> haystack Flask-RESTFul -> flask_restful """ if self.mapping: name = self.mapping.get(name, name) return name.lower().replace('-', '_') def find(self, module_name): # required lib not installed yet if not self.enabled: return module_name, _sep, _submodules = module_name.partition('.') module_name = module_name.lower() if not module_name: return for name in self.names: if module_name == name: return self.sections.THIRDPARTY class RequirementsFinder(ReqsBaseFinder): exts = ('.txt', '.in') enabled = bool(parse_requirements) def _get_files_from_dir(self, path): """Return paths to requirements files from passed dir. """ return RequirementsFinder._get_files_from_dir_cached(path) @classmethod @lru_cache(maxsize=16) def _get_files_from_dir_cached(cls, path): result = [] for fname in os.listdir(path): if 'requirements' not in fname: continue full_path = os.path.join(path, fname) # *requirements*/*.{txt,in} if os.path.isdir(full_path): for subfile_name in os.listdir(path): for ext in cls.exts: if subfile_name.endswith(ext): result.append(os.path.join(path, subfile_name)) continue # *requirements*.{txt,in} if os.path.isfile(full_path): for ext in cls.exts: if fname.endswith(ext): result.append(full_path) break return result def _get_names(self, path): """Load required packages from path to requirements file """ return RequirementsFinder._get_names_cached(path) @classmethod @lru_cache(maxsize=16) def _get_names_cached(cls, path): results = [] with chdir(os.path.dirname(path)): requirements = parse_requirements(path, session=PipSession()) for req in requirements: if req.name: results.append(req.name) return results class PipfileFinder(ReqsBaseFinder): enabled = bool(Pipfile) def _get_names(self, path): with chdir(path): project = Pipfile.load(path) for req in project.packages: yield req.name def _get_files_from_dir(self, path): if 'Pipfile' in os.listdir(path): yield path class DefaultFinder(BaseFinder): def find(self, module_name): return self.config['default_section'] class FindersManager(object): finders = ( ForcedSeparateFinder, LocalFinder, KnownPatternFinder, PathFinder, PipfileFinder, RequirementsFinder, DefaultFinder, ) def __init__(self, config, sections, finders=None): self.verbose = config.get('verbose', False) finders = self.finders if finders is None else finders self.finders = [] for finder in finders: try: self.finders.append(finder(config, sections)) except Exception as exception: # if one finder fails to instantiate isort can continue using the rest if self.verbose: print('{} encountered an error ({}) during instantiation and cannot be used'.format(finder.__name__, str(exception))) self.finders = tuple(self.finders) def find(self, module_name): for finder in self.finders: try: section = finder.find(module_name) except Exception as exception: # isort has to be able to keep trying to identify the correct import section even if one approach fails if self.verbose: print('{} encountered an error ({}) while trying to identify the {} module'.format(finder.__name__, str(exception), module_name)) if section is not None: return section