From 9296f561d91f847ad65279b41c458d41a67f7ff7 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 12 Nov 2018 22:27:38 -0500 Subject: [PATCH] Fix bugs in environment implementation - Fix virtualenv - Update pythonfinder Signed-off-by: Dan Ryan --- pipenv/core.py | 5 +- pipenv/environment.py | 33 +++++----- pipenv/project.py | 5 +- pipenv/vendor/pythonfinder/environment.py | 4 ++ pipenv/vendor/pythonfinder/models/asdf.py | 9 +++ pipenv/vendor/pythonfinder/models/path.py | 69 +++++++++++++++------ pipenv/vendor/pythonfinder/models/pyenv.py | 7 ++- pipenv/vendor/pythonfinder/models/python.py | 2 +- pipenv/vendor/pythonfinder/utils.py | 6 +- 9 files changed, 96 insertions(+), 44 deletions(-) create mode 100644 pipenv/vendor/pythonfinder/models/asdf.py diff --git a/pipenv/core.py b/pipenv/core.py index ee1a7df8..8504553c 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -926,7 +926,8 @@ def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None): prefix=project.get_location_for_virtualenv(), is_venv=True, sources=sources, - pipfile=project.parsed_pipfile + pipfile=project.parsed_pipfile, + project=project ) project._environment.add_dist("pipenv") # Say where the virtualenv is. @@ -1626,7 +1627,7 @@ def do_outdated(pypi_mirror=None): outdated_packages = { canonicalize_name(pkg.project_name): package_info (pkg.project_name, pkg.parsed_version, pkg.latest_version) - for pkg in project.get_outdated_packages() + for pkg in project.environment.get_outdated_packages() } for result in installed_packages: dep = Requirement.from_line(str(result.as_requirement())) diff --git a/pipenv/environment.py b/pipenv/environment.py index 9315447c..8548c38f 100644 --- a/pipenv/environment.py +++ b/pipenv/environment.py @@ -22,7 +22,8 @@ BASE_WORKING_SET = pkg_resources.WorkingSet(sys.path) class Environment(object): - def __init__(self, prefix=None, is_venv=False, base_working_set=None, pipfile=None, sources=None): + def __init__(self, prefix=None, is_venv=False, base_working_set=None, pipfile=None, + sources=None, project=None): super(Environment, self).__init__() self._modules = {'pkg_resources': pkg_resources, 'pipenv': pipenv} self.base_working_set = base_working_set if base_working_set else BASE_WORKING_SET @@ -30,10 +31,17 @@ class Environment(object): self.is_venv = not prefix == os.path.normcase(os.path.normpath(sys.prefix)) if not sources: sources = [] + self.project = project + if project and not sources: + sources = project.sources self.sources = sources + if project and not pipfile: + pipfile = project.pipfile + self.pipfile = pipfile self.extra_dists = [] prefix = prefix if prefix else sys.prefix self.prefix = vistir.compat.Path(prefix) + self.sys_paths = get_paths() def safe_import(self, name): """Helper utility for reimporting previously imported modules while inside the env""" @@ -73,7 +81,7 @@ class Environment(object): deps.add(dist) try: reqs = dist.requires() - except AttributeError: + except (AttributeError, OSError): # The METADATA file can't be found return deps for req in reqs: dist = working_set.find(req) @@ -187,12 +195,6 @@ class Environment(object): path = json.loads(path.strip()) return path - @cached_property - def system_paths(self): - paths = {} - paths = get_paths() - return paths - @cached_property def sys_prefix(self): """The prefix run inside the context of the environment @@ -271,7 +273,8 @@ class Environment(object): packages = [pkg for pkg in workingset if self.dist_is_in_project(pkg)] return packages - def get_finder(self): + @contextlib.contextmanager + def get_finder(self, pre=False): from .vendor.pip_shims import Command, cmdoptions, index_group, PackageFinder from .environments import PIPENV_CACHE_DIR index_urls = [source.get("url") for source in self.sources] @@ -286,10 +289,10 @@ class Environment(object): cmd_opts = pip_command.cmd_opts pip_command.parser.insert_option_group(0, index_opts) pip_command.parser.insert_option_group(0, cmd_opts) - pip_args = self._modules["pipenv"].utils.prepare_pip_source_args(self.sources, []) + pip_args = self._modules["pipenv"].utils.prepare_pip_source_args(self.sources) pip_options, _ = pip_command.parser.parse_args(pip_args) pip_options.cache_dir = PIPENV_CACHE_DIR - pip_options.pre = self.pipfile.get("pre", False) + pip_options.pre = self.pipfile.get("pre", pre) with pip_command._build_session(pip_options) as session: finder = PackageFinder( find_links=pip_options.find_links, @@ -300,7 +303,7 @@ class Environment(object): ) yield finder - def get_package_info(self): + def get_package_info(self, pre=False): dependency_links = [] packages = self.get_installed_packages() # This code is borrowed from pip's current implementation @@ -314,7 +317,7 @@ class Environment(object): for dist in packages: typ = 'unknown' all_candidates = finder.find_all_candidates(dist.key) - if not finder.pip_options.pre: + if not self.pipfile.get("pre", finder.allow_all_prereleases): # Remove prereleases all_candidates = [ candidate for candidate in all_candidates @@ -334,9 +337,9 @@ class Environment(object): dist.latest_filetype = typ yield dist - def get_outdated_packages(self): + def get_outdated_packages(self, pre=False): return [ - pkg for pkg in self.get_package_info() + pkg for pkg in self.get_package_info(pre=pre) if pkg.latest_version._version > pkg.parsed_version._version ] diff --git a/pipenv/project.py b/pipenv/project.py index 0eafff8e..7857b25a 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -352,13 +352,14 @@ class Project(object): is_venv = prefix == sys.prefix sources = self.sources if self.sources else [DEFAULT_SOURCE,] self._environment = Environment( - prefix=prefix, is_venv=is_venv, sources=sources, pipfile=self.parsed_pipfile + prefix=prefix, is_venv=is_venv, sources=sources, pipfile=self.parsed_pipfile, + project=self ) self._environment.add_dist("pipenv") return self._environment def get_outdated_packages(self): - return self.environment.get_outdated_packages() + return self.environment.get_outdated_packages(pre=self.pipfile.get("pre", False)) @classmethod def _sanitize(cls, name): diff --git a/pipenv/vendor/pythonfinder/environment.py b/pipenv/vendor/pythonfinder/environment.py index 27a5b3fc..ec4a760f 100644 --- a/pipenv/vendor/pythonfinder/environment.py +++ b/pipenv/vendor/pythonfinder/environment.py @@ -7,9 +7,13 @@ import sys PYENV_INSTALLED = bool(os.environ.get("PYENV_SHELL")) or bool( os.environ.get("PYENV_ROOT") ) +ASDF_INSTALLED = bool(os.environ.get("ASDF_DATA_DIR")) PYENV_ROOT = os.path.expanduser( os.path.expandvars(os.environ.get("PYENV_ROOT", "~/.pyenv")) ) +ASDF_DATA_DIR = os.path.expanduser( + os.path.expandvars(os.environ.get("ASDF_DATA_DIR", "~/.asdf")) +) IS_64BIT_OS = None SYSTEM_ARCH = platform.architecture()[0] diff --git a/pipenv/vendor/pythonfinder/models/asdf.py b/pipenv/vendor/pythonfinder/models/asdf.py new file mode 100644 index 00000000..3ba6e4fa --- /dev/null +++ b/pipenv/vendor/pythonfinder/models/asdf.py @@ -0,0 +1,9 @@ +# -*- coding=utf-8 -*- +import attr + +from .pyenv import PyenvFinder + + +@attr.s +class AsdfFinder(PyenvFinder): + version_root = attr.ib(default="installs/python/*") diff --git a/pipenv/vendor/pythonfinder/models/path.py b/pipenv/vendor/pythonfinder/models/path.py index 3d01e7cf..9c96e5f8 100644 --- a/pipenv/vendor/pythonfinder/models/path.py +++ b/pipenv/vendor/pythonfinder/models/path.py @@ -17,7 +17,7 @@ from cached_property import cached_property from vistir.compat import Path, fs_str from .mixins import BasePath -from ..environment import PYENV_INSTALLED, PYENV_ROOT +from ..environment import PYENV_INSTALLED, PYENV_ROOT, ASDF_INSTALLED, ASDF_DATA_DIR from ..exceptions import InvalidPythonVersion from ..utils import ( ensure_path, @@ -40,6 +40,7 @@ class SystemPath(object): python_version_dict = attr.ib(default=attr.Factory(defaultdict)) only_python = attr.ib(default=False) pyenv_finder = attr.ib(default=None, validator=optional_instance_of("PyenvPath")) + asdf_finder = attr.ib(default=None) system = attr.ib(default=False) _version_dict = attr.ib(default=attr.Factory(defaultdict)) ignore_unsupported = attr.ib(default=False) @@ -105,6 +106,8 @@ class SystemPath(object): self._setup_windows() if PYENV_INSTALLED: self._setup_pyenv() + if ASDF_INSTALLED: + self._setup_asdf() venv = os.environ.get("VIRTUAL_ENV") if os.name == "nt": bin_dir = "Scripts" @@ -124,32 +127,62 @@ class SystemPath(object): path=syspath_bin, is_root=True, only_python=False ) - def _setup_pyenv(self): - from .pyenv import PyenvFinder - - last_pyenv = next( - (p for p in reversed(self.path_order) if PYENV_ROOT.lower() in p.lower()), + def _get_last_instance(self, path): + last_instance = next(iter( + (p for p in reversed(self.path_order) if path.lower() in p.lower())), None, ) try: - pyenv_index = self.path_order.index(last_pyenv) + path_index = self.path_order.index(last_instance) except ValueError: return + return path_index + + def _slice_in_paths(self, start_idx, paths): + before_path = self.path_order[: start_idx + 1] + after_path = self.path_order[start_idx + 2 :] + self.path_order = ( + before_path + [p.as_posix() for p in paths] + after_path + ) + + def _remove_path(self, path): + path_copy = reversed(self.path_order[:]) + new_order = [] + target = os.path.normcase(os.path.normpath(os.path.abspath(path))) + path_map = { + os.path.normcase(os.path.normpath(os.path.abspath(pth))): pth + for pth in self.paths.keys() + } + if target in path_map: + del self.paths[path_map.get(target)] + for current_path in path_copy: + normalized = os.path.normcase(os.path.normpath(os.path.abspath(current_path))) + if normalized != target: + new_order.append(normalized) + new_order = reversed(new_order) + self.path_order = new_order + + def _setup_asdf(self): + from .asdf import AsdfFinder + asdf_index = self._get_last_instance(ASDF_DATA_DIR) + self.asdf_finder = AsdfFinder.create(root=ASDF_DATA_DIR, ignore_unsupported=True) + root_paths = [p for p in self.asdf_finder.roots] + self._slice_in_paths(asdf_index, root_paths) + self.paths.update(self.asdf_finder.roots) + self._register_finder("asdf", self.asdf_finder) + + def _setup_pyenv(self): + from .pyenv import PyenvFinder + + pyenv_index = self._get_last_instance(PYENV_ROOT) self.pyenv_finder = PyenvFinder.create( root=PYENV_ROOT, ignore_unsupported=self.ignore_unsupported ) root_paths = [p for p in self.pyenv_finder.roots] - before_path = self.path_order[: pyenv_index + 1] - after_path = self.path_order[pyenv_index + 2 :] - self.path_order = ( - before_path + [p.as_posix() for p in root_paths] + after_path - ) - pyenv_shim_path = os.path.join(PYENV_ROOT, "shims") - if pyenv_shim_path in self.path_order: - self.path_order.remove(pyenv_shim_path) + self._slice_in_paths(pyenv_index, root_paths) + self.paths.update(self.pyenv_finder.roots) - if pyenv_shim_path in self.paths: - del self.paths[pyenv_shim_path] + self._remove_path(os.path.join(PYENV_ROOT, "shims")) self._register_finder("pyenv", self.pyenv_finder) def _setup_windows(self): @@ -396,7 +429,7 @@ class SystemPath(object): ) -@attr.s +@attr.s(slots=True) class PathEntry(BasePath): path = attr.ib(default=None, validator=optional_instance_of(Path)) _children = attr.ib(default=attr.Factory(dict)) diff --git a/pipenv/vendor/pythonfinder/models/pyenv.py b/pipenv/vendor/pythonfinder/models/pyenv.py index ac7f8588..cf85f57a 100644 --- a/pipenv/vendor/pythonfinder/models/pyenv.py +++ b/pipenv/vendor/pythonfinder/models/pyenv.py @@ -26,7 +26,7 @@ from .python import PythonVersion logger = logging.getLogger(__name__) -@attr.s +@attr.s(slots=True) class PyenvFinder(BaseFinder, BasePath): root = attr.ib(default=None, validator=optional_instance_of(Path)) #: ignore_unsupported should come before versions, because its value is used @@ -34,6 +34,7 @@ class PyenvFinder(BaseFinder, BasePath): ignore_unsupported = attr.ib(default=True) paths = attr.ib(default=attr.Factory(list)) roots = attr.ib(default=attr.Factory(defaultdict)) + version_root = attr.ib(default="versions/*") versions = attr.ib() pythons = attr.ib() @@ -50,7 +51,7 @@ class PyenvFinder(BaseFinder, BasePath): version_order_lines = version_order_file.read_text(encoding="utf-8").splitlines() version_paths = [ - p for p in self.root.glob("versions/*") + p for p in self.root.glob(self.version_root) if not (p.parent.name == "envs" or p.name == "envs") ] versions = {v.name: v for v in version_paths} @@ -74,7 +75,7 @@ class PyenvFinder(BaseFinder, BasePath): @versions.default def get_versions(self): versions = defaultdict() - bin_ = sysconfig._INSTALL_SCHEMES['posix_prefix']["scripts"] + bin_ = "{base}/bin" for p in self.get_version_order(): bin_dir = Path(bin_.format(base=p.as_posix())) version_path = None diff --git a/pipenv/vendor/pythonfinder/models/python.py b/pipenv/vendor/pythonfinder/models/python.py index 24d520b6..583dc6b3 100644 --- a/pipenv/vendor/pythonfinder/models/python.py +++ b/pipenv/vendor/pythonfinder/models/python.py @@ -21,7 +21,7 @@ from ..utils import ( ) -@attr.s +@attr.s(slots=True) class PythonVersion(object): major = attr.ib(default=0) minor = attr.ib(default=None) diff --git a/pipenv/vendor/pythonfinder/utils.py b/pipenv/vendor/pythonfinder/utils.py index 42a63e54..881cdb2e 100644 --- a/pipenv/vendor/pythonfinder/utils.py +++ b/pipenv/vendor/pythonfinder/utils.py @@ -54,7 +54,7 @@ def get_python_version(path): version_cmd = [path, "-c", "import sys; print(sys.version.split()[0])"] try: c = vistir.misc.run(version_cmd, block=True, nospin=True, return_object=True, - combine_stderr=False) + combine_stderr=False) except OSError: raise InvalidPythonVersion("%s is not a valid python path" % path) if not c.out: @@ -92,7 +92,7 @@ def looks_like_python(name): @lru_cache(maxsize=1024) def path_is_python(path): - return path_is_executable(path) and looks_like_python(path.name) + return path_is_known_executable(path) and looks_like_python(path.name) @lru_cache(maxsize=1024) @@ -117,7 +117,7 @@ def _filter_none(k, v): return False -@lru_cache(maxsize=128) +@lru_cache(maxsize=1024) def filter_pythons(path): """Return all valid pythons in a given path""" if not isinstance(path, vistir.compat.Path):