kennethreitz/pipenv
1
0
Fork 0
mirror of https://github.com/kennethreitz/pipenv.git synced 2026-06-05 22:50:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into shell-fallback-improvement

Browse Source
This commit is contained in:
Tzu-ping Chung
2018-07-30 14:30:06 +08:00
committed by GitHub
parent 7758d745a8 5a55c88773
commit 9ec4ff13d6
33 changed files with 896 additions and 364 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/advanced.rst
+1 -1
View File
@@ -242,11 +242,11 @@ different products which integrate with Pipenv projects:
- `Emacs <https://github.com/pwalsh/pipenv.el>`_ (Editor Integration)
- `Fish Shell <https://github.com/fisherman/pipenv>`_ (Automatic ``$ pipenv shell``!)
- `VS Code <https://code.visualstudio.com/docs/python/environments>`_ (Editor Integration)
- `PyCharm <https://www.jetbrains.com/pycharm/download/>`_ (Editor Integration)
Works in progress:
- `Sublime Text <https://github.com/kennethreitz/pipenv-sublime>`_ (Editor Integration)
- `PyCharm <https://www.jetbrains.com/pycharm/download/>`_ (Editor Integration)
- Mysterious upcoming Google Cloud product (Cloud Hosting)
news/2434.bugfix
+1
View File
@@ -0,0 +1 @@
Fixed the ability of pipenv to parse ``dependency_links`` from ``setup.py`` when ``PIP_PROCESS_DEPENDENCY_LINKS`` is enabled.
news/2582.bugfix
+1
View File
@@ -0,0 +1 @@
Fixed multiple issues with finding the correct system python locations.
news/2582.feature
+4
View File
@@ -0,0 +1,4 @@
Greatly enhanced python discovery functionality:
- Added pep514 (windows launcher/finder) support for python discovery.
- Introduced architecture discovery for python installations which support different architectures.
news/2582.vendor
+1
View File
@@ -0,0 +1 @@
Update ``pythonfinder`` to major release ``1.0.0`` for integration.
news/2607.bugfix
+2
View File
@@ -0,0 +1,2 @@
Catch JSON decoding error to prevent exception when the lock file is of
invalid format.
news/2643.bugfix
+1
View File
@@ -0,0 +1 @@
Dependency links to private repositories defined via ``ssh://`` schemes will now install correctly and skip hashing as long as ``PIP_PROCESS_DEPENDENCY_LINKS=1``.
news/2643.feature
+1
View File
@@ -0,0 +1 @@
Enhanced resolution of editable and VCS dependencies.
pipenv/core.py
+51 -126
View File
@@ -6,7 +6,6 @@ import sys
import shutil
import time
import tempfile
from glob import glob
import json as simplejson
import click
import click_completion
@@ -51,8 +50,6 @@ from .environments import (
PIPENV_SKIP_VALIDATION,
PIPENV_HIDE_EMOJIS,
PIPENV_INSTALL_TIMEOUT,
PYENV_ROOT,
PYENV_INSTALLED,
PIPENV_YES,
PIPENV_DONT_LOAD_ENV,
PIPENV_DEFAULT_PYTHON_VERSION,
@@ -317,76 +314,38 @@ def ensure_pipfile(validate=True, skip_requirements=False, system=False):
project.write_toml(p)
def find_python_from_py(python):
"""Find a Python executable from on Windows.
def find_a_system_python(line):
"""Find a Python installation from a given line.
Ask py.exe for its opinion.
This tries to parse the line in various of ways:
* Looks like an absolute path? Use it directly.
* Looks like a py.exe call? Use py.exe to get the executable.
* Starts with "py" something? Looks like a python command. Try to find it
in PATH, and use it directly.
* Search for "python" and "pythonX.Y" executables in PATH to find a match.
* Nothing fits, return None.
"""
py = system_which("py")
if not py:
if not line:
return None
version_args = ["-{0}".format(python[0])]
if len(python) >= 2:
version_args.append("-{0}.{1}".format(python[0], python[2]))
import subprocess
for ver_arg in reversed(version_args):
try:
python_exe = subprocess.check_output(
[py, ver_arg, "-c", "import sys; print(sys.executable)"]
)
except subprocess.CalledProcessError:
continue
if not isinstance(python_exe, str):
python_exe = python_exe.decode(sys.getdefaultencoding())
python_exe = python_exe.strip()
version = python_version(python_exe)
if (version or "").startswith(python):
return python_exe
def find_python_in_path(python):
"""Find a Python executable from a version number.
This uses the PATH environment variable to locate an appropriate Python.
"""
possibilities = ["python", "python{0}".format(python[0])]
if len(python) >= 2:
possibilities.extend(
[
"python{0}{1}".format(python[0], python[2]),
"python{0}.{1}".format(python[0], python[2]),
"python{0}.{1}m".format(python[0], python[2]),
]
)
# Reverse the list, so we find specific ones first.
possibilities = reversed(possibilities)
for possibility in possibilities:
# Windows compatibility.
if os.name == "nt":
possibility = "{0}.exe".format(possibility)
pythons = system_which(possibility, mult=True)
for p in pythons:
version = python_version(p)
if (version or "").startswith(python):
return p
def find_a_system_python(python):
"""Finds a system python, given a version (e.g. 2 / 2.7 / 3.6.2), or a full path."""
if python.startswith("py"):
return system_which(python)
elif os.path.isabs(python):
return python
python_from_py = find_python_from_py(python)
if python_from_py:
return python_from_py
return find_python_in_path(python)
if os.path.isabs(line):
return line
from .vendor.pythonfinder import Finder
finder = Finder(system=False, global_search=True)
if ((line.startswith("py ") or line.startswith("py.exe "))
and os.name == 'nt'):
line = line.split(" ", 1)[1].lstrip("-")
elif line.startswith("py"):
python_entry = finder.which(line)
if python_entry:
return python_entry.path.as_posix()
return None
python_entry = finder.find_python_version(line)
if not python_entry:
python_entry = finder.which("python{0}".format(line))
if python_entry:
return python_entry.path.as_posix()
return None
def ensure_python(three=None, python=None):
@@ -409,35 +368,7 @@ def ensure_python(three=None, python=None):
)
sys.exit(1)
def activate_pyenv():
from notpip._vendor.packaging.version import parse as parse_version
"""Adds all pyenv installations to the PATH."""
if PYENV_INSTALLED:
if PYENV_ROOT:
pyenv_paths = {}
for found in glob("{0}{1}versions{1}*".format(PYENV_ROOT, os.sep)):
pyenv_paths[os.path.split(found)[1]] = "{0}{1}bin".format(
found, os.sep
)
for version_str, pyenv_path in pyenv_paths.items():
version = parse_version(version_str)
if version.is_prerelease and pyenv_paths.get(version.base_version):
continue
add_to_path(pyenv_path)
else:
click.echo(
"{0}: PYENV_ROOT is not set. New python paths will "
"probably not be exported properly after installation."
"".format(crayons.red("Warning", bold=True)),
err=True,
)
global USING_DEFAULT_PYTHON
# Add pyenv paths to PATH.
activate_pyenv()
path_to_python = None
USING_DEFAULT_PYTHON = three is None and not python
# Find out which python is desired.
if not python:
@@ -446,8 +377,7 @@ def ensure_python(three=None, python=None):
python = project.required_python_version
if not python:
python = PIPENV_DEFAULT_PYTHON_VERSION
if python:
path_to_python = find_a_system_python(python)
path_to_python = find_a_system_python(python)
if not path_to_python and python is not None:
# We need to install Python.
click.echo(
@@ -459,6 +389,7 @@ def ensure_python(three=None, python=None):
err=True,
)
# Pyenv is installed
from .vendor.pythonfinder.environment import PYENV_INSTALLED
if not PYENV_INSTALLED:
abort()
else:
@@ -501,8 +432,6 @@ def ensure_python(three=None, python=None):
click.echo(crayons.blue(e.err), err=True)
# Print the results, in a beautiful blue…
click.echo(crayons.blue(c.out), err=True)
# Add new paths to PATH.
activate_pyenv()
# Find the newly installed Python, hopefully.
version = str(version)
path_to_python = find_a_system_python(version)
@@ -915,21 +844,17 @@ def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None):
# Actually create the virtualenv.
with spinner():
try:
c = delegator.run(cmd, block=False, timeout=PIPENV_TIMEOUT, env=pip_config)
except OSError:
click.echo(
"{0}: it looks like {1} is not in your {2}. "
"We cannot continue until this is resolved."
"".format(
crayons.red("Warning", bold=True),
crayons.red(cmd[0]),
crayons.normal("PATH", bold=True),
),
err=True,
)
sys.exit(1)
click.echo(crayons.blue(c.out), err=True)
c = delegator.run(
cmd, block=False, timeout=PIPENV_TIMEOUT, env=pip_config,
)
c.block()
click.echo(crayons.blue("{0}".format(c.out)), err=True)
if c.return_code != 0:
click.echo(crayons.blue("{0}".format(c.err)), err=True)
click.echo(u"{0}: Failed to create virtual environment.".format(
crayons.red("Warning", bold=True),
), err=True)
sys.exit(1)
# Associate project directory with the environment.
# This mimics Pew's "setproject".
@@ -1258,19 +1183,19 @@ def do_init(
err=True,
)
else:
click.echo(
crayons.red(
u"Pipfile.lock ({0}) out of date, updating to ({1})…".format(
old_hash[-6:], new_hash[-6:]
),
bold=True,
),
err=True,
)
if old_hash:
msg = u"Pipfile.lock ({1}) out of date, updating to ({0})…"
else:
msg = u"Pipfile.lock is corrupted, replaced with ({0})…"
click.echo(crayons.red(
msg.format(old_hash[-6:], new_hash[-6:]),
bold=True,
), err=True)
do_lock(
system=system,
pre=pre,
keep_outdated=keep_outdated,
verbose=verbose,
write=True,
pypi_mirror=pypi_mirror,
)
pipenv/environments.py
-7
View File
@@ -213,12 +213,5 @@ PIPENV_SHELL = (
os.environ.get("COMSPEC")
)
# Internal, to tell if pyenv is installed.
PYENV_ROOT = os.environ.get("PYENV_ROOT", os.path.expanduser("~/.pyenv"))
PYENV_INSTALLED = (
bool(os.environ.get("PYENV_SHELL")) or
bool(os.environ.get("PYENV_ROOT"))
)
# Internal, to tell whether the command line session is interactive.
SESSION_IS_INTERACTIVE = bool(os.isatty(sys.stdout.fileno()))
pipenv/help.py
+13 -19
View File
@@ -1,12 +1,13 @@
# coding: utf-8
import os
import pprint
import sys
import pipenv
from pprint import pprint
from .__version__ import __version__
from .core import project, system_which, find_python_in_path, python_version
from .core import project
from .pep508checker import lookup
from .vendor import pythonfinder
def print_utf(line):
@@ -19,32 +20,25 @@ def print_utf(line):
def get_pipenv_diagnostics():
print("<details><summary>$ pipenv --support</summary>")
print("")
print("Pipenv version: `{0!r}`".format(__version__))
print("Pipenv version: `{0!r}`".format(pipenv.__version__))
print("")
print("Pipenv location: `{0!r}`".format(os.path.dirname(pipenv.__file__)))
print("")
print("Python location: `{0!r}`".format(sys.executable))
print("")
print("Other Python installations in `PATH`:")
print("Python installations found:")
print("")
for python_v in ("2.5", "2.6", "2.7", "3.4", "3.5", "3.6", "3.7"):
found = find_python_in_path(python_v)
if found:
print(" - `{0}`: `{1}`".format(python_v, found))
found = system_which("python{0}".format(python_v), mult=True)
if found:
for f in found:
print(" - `{0}`: `{1}`".format(python_v, f))
print("")
for p in ("python", "python2", "python3", "py"):
found = system_which(p, mult=True)
for f in found:
print(" - `{0}`: `{1}`".format(python_version(f), f))
finder = pythonfinder.Finder(system=False, global_search=True)
python_paths = finder.find_all_python_versions()
for python in python_paths:
print(" - `{}`: `{}`".format(python.py_version.version, python.path))
print("")
print("PEP 508 Information:")
print("")
print("```")
pprint(lookup)
pprint.pprint(lookup)
print("```")
print("")
print("System environment variables:")
pipenv/patched/piptools/repositories/pypi.py
+27 -13
View File
@@ -1,7 +1,7 @@
# coding: utf-8
from __future__ import (absolute_import, division, print_function,
unicode_literals)
import copy
import hashlib
import os
import sys
@@ -21,18 +21,16 @@ from .._compat import (
SafeFileCache,
)
from pipenv.patched.notpip._vendor.packaging.requirements import InvalidRequirement, Requirement
from pipenv.patched.notpip._vendor.packaging.version import Version, InvalidVersion, parse as parse_version
from pipenv.patched.notpip._vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier, Specifier
from pipenv.patched.notpip._vendor.packaging.markers import Marker, Op, Value, Variable
from pipenv.patched.notpip._vendor.pyparsing import ParseException
from pipenv.patched.notpip._vendor.packaging.requirements import Requirement
from pipenv.patched.notpip._vendor.packaging.specifiers import SpecifierSet, Specifier
from pipenv.patched.notpip._vendor.packaging.markers import Op, Value, Variable
from pipenv.patched.notpip._internal.exceptions import InstallationError
from pipenv.patched.notpip._internal.vcs import VcsSupport
from ..cache import CACHE_DIR
from pipenv.environments import PIPENV_CACHE_DIR
from ..exceptions import NoCandidateFound
from ..utils import (fs_str, is_pinned_requirement, lookup_table, as_tuple, key_from_req,
make_install_requirement, format_requirement, dedup, clean_requires_python)
from ..utils import (fs_str, is_pinned_requirement, lookup_table,
make_install_requirement, clean_requires_python)
from .base import BaseRepository
@@ -64,15 +62,20 @@ class HashCache(SafeFileCache):
def get_hash(self, location):
# if there is no location hash (i.e., md5 / sha256 / etc) we on't want to store it
hash_value = None
can_hash = location.hash
vcs = VcsSupport()
orig_scheme = location.scheme
new_location = copy.deepcopy(location)
if orig_scheme in vcs.all_schemes:
new_location.url = new_location.url.split("+", 1)[-1]
can_hash = new_location.hash
if can_hash:
# hash url WITH fragment
hash_value = self.get(location.url)
hash_value = self.get(new_location.url)
if not hash_value:
hash_value = self._get_file_hash(location)
hash_value = self._get_file_hash(new_location)
hash_value = hash_value.encode('utf8')
if can_hash:
self.set(location.url, hash_value)
self.set(new_location.url, hash_value)
return hash_value.decode('utf8')
def _get_file_hash(self, location):
@@ -276,6 +279,13 @@ class PyPIRepository(BaseRepository):
setup_requires = {}
dist = None
if ireq.editable:
try:
from pipenv.utils import chdir
with chdir(ireq.setup_py_dir):
from setuptools.dist import distutils
distutils.core.run_setup(ireq.setup_py)
except (ImportError, InstallationError, TypeError, AttributeError):
pass
try:
dist = ireq.get_dist()
except InstallationError:
@@ -425,6 +435,10 @@ class PyPIRepository(BaseRepository):
if ireq.editable:
return set()
vcs = VcsSupport()
if ireq.link and ireq.link.scheme in vcs.all_schemes and 'ssh' in ireq.link.scheme:
return set()
if not is_pinned_requirement(ireq):
raise TypeError(
"Expected pinned requirement, got {}".format(ireq))
pipenv/project.py
+5 -1
View File
@@ -815,7 +815,11 @@ class Project(object):
if not os.path.exists(self.lockfile_location):
return
lockfile = self.load_lockfile(expand_env_vars=False)
try:
lockfile = self.load_lockfile(expand_env_vars=False)
except ValueError:
# Lockfile corrupted
return ""
if "_meta" in lockfile and hasattr(lockfile, "keys"):
return lockfile["_meta"].get("hash", {}).get("sha256")
# Lockfile exists but has no hash at all
pipenv/utils.py
+16
View File
@@ -1361,3 +1361,19 @@ def is_virtual_environment(path):
if python_like.is_file() and os.access(str(python_like), os.X_OK):
return True
return False
@contextmanager
def chdir(path):
"""Context manager to change working directories."""
from ._compat import Path
if not path:
return
prev_cwd = Path.cwd().as_posix()
if isinstance(path, Path):
path = path.as_posix()
os.chdir(str(path))
try:
yield
finally:
os.chdir(prev_cwd)
pipenv/vendor/README.md
Vendored
+15 -4
View File
@@ -2,10 +2,21 @@
These packages are copied as-is from upstream to reduce Pipenv dependencies.
They should always be kept synced with upstream. DO NOT MODIFY DIRECTLY! If
you need to patch anything, move the package to `patched`.
you need to patch anything, move the package to `patched` and generate a
patch for it using `git diff -p <dependency_root_dir>`. This patch belongs
in `./pipenv/tasks/vendoring/patches/patched/<packagename.patchdesc>.patch`.
Known vendored versions:
To add a vendored dependency or to update a single dependency, use the
vendoring scripts:
```
pipenv run inv vendoring.update --package="pkgname==versionnum"
```
- python-dotenv: 0.8.2
This will automatically pin the package in `./pipenv/vendor/vendor.txt`
or it will update the pin if the package is already present, and it will
then update the package and download any necessary licenses (if available).
Note that this will not download any dependencies, you must add those each
individually.
When updating, update the corresponding LICENSE files as well.
When updating, ensure that the corresponding LICENSE files are still
up-to-date.
pipenv/vendor/cached-property.LICENSE
Vendored
+12
View File
@@ -0,0 +1,12 @@
Copyright (c) 2015, Daniel Greenfeld
All rights reserved.
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
* Neither the name of cached-property nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
pipenv/vendor/cached_property.py
Vendored
+152
View File
@@ -0,0 +1,152 @@
# -*- coding: utf-8 -*-
__author__ = "Daniel Greenfeld"
__email__ = "pydanny@gmail.com"
__version__ = "1.4.3"
__license__ = "BSD"
from time import time
import threading
try:
import asyncio
except (ImportError, SyntaxError):
asyncio = None
class cached_property(object):
"""
A property that is only computed once per instance and then replaces itself
with an ordinary attribute. Deleting the attribute resets the property.
Source: https://github.com/bottlepy/bottle/commit/fa7733e075da0d790d809aa3d2f53071897e6f76
""" # noqa
def __init__(self, func):
self.__doc__ = getattr(func, "__doc__")
self.func = func
def __get__(self, obj, cls):
if obj is None:
return self
if asyncio and asyncio.iscoroutinefunction(self.func):
return self._wrap_in_coroutine(obj)
value = obj.__dict__[self.func.__name__] = self.func(obj)
return value
def _wrap_in_coroutine(self, obj):
@asyncio.coroutine
def wrapper():
future = asyncio.ensure_future(self.func(obj))
obj.__dict__[self.func.__name__] = future
return future
return wrapper()
class threaded_cached_property(object):
"""
A cached_property version for use in environments where multiple threads
might concurrently try to access the property.
"""
def __init__(self, func):
self.__doc__ = getattr(func, "__doc__")
self.func = func
self.lock = threading.RLock()
def __get__(self, obj, cls):
if obj is None:
return self
obj_dict = obj.__dict__
name = self.func.__name__
with self.lock:
try:
# check if the value was computed before the lock was acquired
return obj_dict[name]
except KeyError:
# if not, do the calculation and release the lock
return obj_dict.setdefault(name, self.func(obj))
class cached_property_with_ttl(object):
"""
A property that is only computed once per instance and then replaces itself
with an ordinary attribute. Setting the ttl to a number expresses how long
the property will last before being timed out.
"""
def __init__(self, ttl=None):
if callable(ttl):
func = ttl
ttl = None
else:
func = None
self.ttl = ttl
self._prepare_func(func)
def __call__(self, func):
self._prepare_func(func)
return self
def __get__(self, obj, cls):
if obj is None:
return self
now = time()
obj_dict = obj.__dict__
name = self.__name__
try:
value, last_updated = obj_dict[name]
except KeyError:
pass
else:
ttl_expired = self.ttl and self.ttl < now - last_updated
if not ttl_expired:
return value
value = self.func(obj)
obj_dict[name] = (value, now)
return value
def __delete__(self, obj):
obj.__dict__.pop(self.__name__, None)
def __set__(self, obj, value):
obj.__dict__[self.__name__] = (value, time())
def _prepare_func(self, func):
self.func = func
if func:
self.__doc__ = func.__doc__
self.__name__ = func.__name__
self.__module__ = func.__module__
# Aliases to make cached_property_with_ttl easier to use
cached_property_ttl = cached_property_with_ttl
timed_cached_property = cached_property_with_ttl
class threaded_cached_property_with_ttl(cached_property_with_ttl):
"""
A cached_property version for use in environments where multiple threads
might concurrently try to access the property.
"""
def __init__(self, ttl=None):
super(threaded_cached_property_with_ttl, self).__init__(ttl)
self.lock = threading.RLock()
def __get__(self, obj, cls):
with self.lock:
return super(threaded_cached_property_with_ttl, self).__get__(obj, cls)
# Alias to make threaded_cached_property_with_ttl easier to use
threaded_cached_property_ttl = threaded_cached_property_with_ttl
timed_threaded_cached_property = threaded_cached_property_with_ttl
pipenv/vendor/pythonfinder/__init__.py
Vendored
+1 -1
View File
@@ -1,6 +1,6 @@
from __future__ import print_function, absolute_import
__version__ = "0.1.4.dev0"
__version__ = "1.0.0"
__all__ = ["Finder", "WindowsFinder", "SystemPath", "InvalidPythonVersion"]
from .pythonfinder import Finder
pipenv/vendor/pythonfinder/_vendor/Makefile
Vendored
-14
View File
@@ -1,14 +0,0 @@
# Taken from pip: https://github.com/pypa/pip/blob/95bcf8c5f6394298035a7332c441868f3b0169f4/src/pip/_vendor/Makefile
all: clean vendor
clean:
@# Delete vendored items
find . -maxdepth 1 -mindepth 1 -type d -exec rm -rf {} \;
vendor:
@# Install vendored libraries
pip install -t . -r vendor.txt
@# Cleanup .egg-info directories
rm -rf *.egg-info
rm -rf *.dist-info
pipenv/vendor/pythonfinder/_vendor/vendor.txt
Vendored
-1
View File
@@ -1 +0,0 @@
-e git+https://github.com/zooba/pep514tools.git@320e48745660b696e2dcaee888fc2e516b435e48#egg=pep514tools
pipenv/vendor/pythonfinder/exceptions.py
Vendored
+1
View File
@@ -4,4 +4,5 @@ from __future__ import print_function, absolute_import
class InvalidPythonVersion(Exception):
"""Raised when parsing an invalid python version"""
pass
pipenv/vendor/pythonfinder/models/__init__.py
Vendored
+67 -12
View File
@@ -3,7 +3,8 @@ from __future__ import print_function, absolute_import
import abc
import operator
import six
from ..utils import KNOWN_EXTS
from itertools import chain
from ..utils import KNOWN_EXTS, unnest
@six.add_metaclass(abc.ABCMeta)
@@ -39,37 +40,91 @@ class BasePath(object):
for ext in KNOWN_EXTS
]
children = self.children
found = next((children[(self.path / child).as_posix()] for child in valid_names if (self.path / child).as_posix() in children), None)
found = next(
(
children[(self.path / child).as_posix()]
for child in valid_names
if (self.path / child).as_posix() in children
),
None,
)
return found
def find_python_version(self, major, minor=None, patch=None, pre=None, dev=None):
def find_all_python_versions(
self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None
):
"""Search for a specific python version on the path. Return all copies
:param major: Major python version to search for.
:type major: int
:param int minor: Minor python version to search for, defaults to None
:param int patch: Patch python version to search for, defaults to None
:param bool pre: Search for prereleases (default None) - prioritize releases if None
:param bool dev: Search for devreleases (default None) - prioritize releases if None
:param str arch: Architecture to include, e.g. '64bit', defaults to None
:return: A list of :class:`~pythonfinder.models.PathEntry` instances matching the version requested.
:rtype: List[:class:`~pythonfinder.models.PathEntry`]
"""
call_method = (
"find_all_python_versions" if self.is_dir else "find_python_version"
)
sub_finder = operator.methodcaller(
call_method, major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch
)
if not self.is_dir:
return sub_finder(self)
path_filter = filter(None, (sub_finder(p) for p in self.children.values()))
version_sort = operator.attrgetter("as_python.version_sort")
return [c for c in sorted(path_filter, key=version_sort, reverse=True)]
def find_python_version(
self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None
):
"""Search or self for the specified Python version and return the first match.
:param major: Major version number.
:type major: int
:param minor: Minor python version, defaults to None
:param minor: int, optional
:param patch: Patch python version, defaults to None
:param patch: int, optional
:param int minor: Minor python version to search for, defaults to None
:param int patch: Patch python version to search for, defaults to None
:param bool pre: Search for prereleases (default None) - prioritize releases if None
:param bool dev: Search for devreleases (default None) - prioritize releases if None
:param str arch: Architecture to include, e.g. '64bit', defaults to None
:returns: A :class:`~pythonfinder.models.PathEntry` instance matching the version requested.
"""
version_matcher = operator.methodcaller(
"matches", major, minor=minor, patch=patch, pre=pre, dev=dev
"matches",
major=major,
minor=minor,
patch=patch,
pre=pre,
dev=dev,
arch=arch,
)
is_py = operator.attrgetter("is_python")
py_version = operator.attrgetter("as_python")
if not self.is_dir:
if self.is_python and self.as_python.matches(major, minor=minor, patch=patch, pre=pre, dev=dev):
if self.is_python and self.as_python and version_matcher(self.as_python):
return self
return
finder = ((child, child.as_python) for child in self.children.values() if child.is_python and child.as_python)
finder = (
(child, child.as_python)
for child in unnest(self.pythons.values())
if child.as_python
)
py_filter = filter(
None, filter(lambda child: version_matcher(child[1]), finder)
)
version_sort = operator.attrgetter("version")
version_sort = operator.attrgetter("version_sort")
return next(
(c[0] for c in sorted(py_filter, key=lambda child: child[1].version, reverse=True)), None
(
c[0]
for c in sorted(
py_filter, key=lambda child: child[1].version_sort, reverse=True
)
),
None,
)
pipenv/vendor/pythonfinder/models/path.py
Vendored
+168 -67
View File
@@ -1,10 +1,13 @@
# -*- coding=utf-8 -*-
from __future__ import print_function, absolute_import
import attr
import copy
import operator
import os
import sys
from collections import defaultdict
from cached_property import cached_property
from itertools import chain
from . import BasePath
from .python import PythonVersion
from ..environment import PYENV_INSTALLED, PYENV_ROOT
@@ -13,9 +16,10 @@ from ..utils import (
optional_instance_of,
filter_pythons,
path_is_known_executable,
is_python_name,
looks_like_python,
ensure_path,
fs_str
fs_str,
unnest,
)
try:
@@ -26,37 +30,69 @@ except ImportError:
@attr.s
class SystemPath(object):
global_search = attr.ib(default=True)
paths = attr.ib(default=attr.Factory(defaultdict))
_executables = attr.ib(default=attr.Factory(list))
_python_executables = attr.ib(default=attr.Factory(list))
path_order = attr.ib(default=attr.Factory(list))
python_version_dict = attr.ib()
python_version_dict = attr.ib(default=attr.Factory(defaultdict))
only_python = attr.ib(default=False)
pyenv_finder = attr.ib(default=None, validator=optional_instance_of("PyenvPath"))
system = attr.ib(default=False)
_version_dict = attr.ib(default=attr.Factory(defaultdict))
@property
__finders = attr.ib(default=attr.Factory(dict))
def _register_finder(self, finder_name, finder):
if finder_name not in self.__finders:
self.__finders[finder_name] = finder
@cached_property
def executables(self):
if not self._executables:
self._executables = [p for p in self.paths.values() if p.is_executable]
return self._executables
self.executables = [
p
for p in chain(*(child.children.values() for child in self.paths.values()))
if p.is_executable
]
return self.executables
@property
@cached_property
def python_executables(self):
if not self._python_executables:
self._python_executables = [p for p in self.paths.values() if p.is_python]
python_executables = {}
for child in self.paths.values():
if child.pythons:
python_executables.update(dict(child.pythons))
for finder_name, finder in self.__finders.items():
if finder.pythons:
python_executables.update(dict(finder.pythons))
self._python_executables = python_executables
return self._python_executables
@python_version_dict.default
def get_python_version_dict(self):
version_dict = defaultdict(list)
for p in self.python_executables:
try:
version_object = PythonVersion.from_path(p)
except (ValueError, InvalidPythonVersion):
@cached_property
def version_dict(self):
self._version_dict = defaultdict(list)
for finder_name, finder in self.__finders.items():
for version, entry in finder.versions.items():
if finder_name == "windows":
if entry not in self._version_dict[version]:
self._version_dict[version].append(entry)
continue
if isinstance(entry, VersionPath):
for path in entry.paths.values():
if path not in self._version_dict[version] and path.is_python:
self._version_dict[version].append(path)
continue
continue
elif entry not in self._version_dict[version] and entry.is_python:
self._version_dict[version].append(entry)
for p, entry in self.python_executables.items():
version = entry.as_python
if not version:
continue
version_dict[version_object.version_tuple].append(version_object)
return version_dict
version = version.version_tuple
if version and entry not in self._version_dict[version]:
self._version_dict[version].append(entry)
return self._version_dict
def __attrs_post_init__(self):
#: slice in pyenv
@@ -66,24 +102,22 @@ class SystemPath(object):
self._setup_windows()
if PYENV_INSTALLED:
self._setup_pyenv()
venv = os.environ.get('VIRTUAL_ENV')
if os.name == 'nt':
bin_dir = 'Scripts'
venv = os.environ.get("VIRTUAL_ENV")
if os.name == "nt":
bin_dir = "Scripts"
else:
bin_dir = 'bin'
if venv:
p = Path(venv)
bin_dir = "bin"
if venv and (self.system or self.global_search):
p = ensure_path(venv)
self.path_order = [(p / bin_dir).as_posix()] + self.path_order
self.paths[p] = PathEntry.create(
path=p, is_root=True, only_python=False
)
self.paths[p] = PathEntry.create(path=p, is_root=True, only_python=False)
if self.system:
syspath = Path(sys.executable)
syspath_bin = syspath.parent
if syspath_bin.name != bin_dir and syspath_bin.joinpath(bin_dir).exists():
syspath_bin = syspath_bin / bin_dir
self.path_order = [syspath_bin.as_posix()] + self.path_order
self.paths[syspath_bin.as_posix()] = PathEntry.create(
self.paths[syspath_bin] = PathEntry.create(
path=syspath_bin, is_root=True, only_python=False
)
@@ -94,7 +128,10 @@ class SystemPath(object):
(p for p in reversed(self.path_order) if PYENV_ROOT.lower() in p.lower()),
None,
)
pyenv_index = self.path_order.index(last_pyenv)
try:
pyenv_index = self.path_order.index(last_pyenv)
except ValueError:
return
self.pyenv_finder = PyenvFinder.create(root=PYENV_ROOT)
# paths = (v.paths.values() for v in self.pyenv_finder.versions.values())
root_paths = (
@@ -106,6 +143,7 @@ class SystemPath(object):
before_path + [p.path.as_posix() for p in root_paths] + after_path
)
self.paths.update({p.path: p for p in root_paths})
self._register_finder("pyenv", self.pyenv_finder)
def _setup_windows(self):
from .windows import WindowsFinder
@@ -115,15 +153,17 @@ class SystemPath(object):
path_addition = [p.path.as_posix() for p in root_paths]
self.path_order = self.path_order[:] + path_addition
self.paths.update({p.path: p for p in root_paths})
self._register_finder("windows", self.windows_finder)
def get_path(self, path):
path = Path(path)
path = ensure_path(path)
_path = self.paths.get(path.as_posix())
if not _path and path.as_posix() in self.path_order:
self.paths[path.as_posix()] = PathEntry.create(
path=path.resolve(), is_root=True, only_python=self.only_python
_path = PathEntry.create(
path=path.absolute(), is_root=True, only_python=self.only_python
)
return self.paths.get(path.as_posix())
self.paths[path.as_posix()] = _path
return _path
def find_all(self, executable):
"""Search the path for an executable. Return all copies.
@@ -147,60 +187,91 @@ class SystemPath(object):
filtered = filter(None, (sub_which(self.get_path(k)) for k in self.path_order))
return next((f for f in filtered), None)
def find_all_python_versions(self, major, minor=None, patch=None, pre=None, dev=None):
def find_all_python_versions(
self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None
):
"""Search for a specific python version on the path. Return all copies
:param major: Major python version to search for.
:type major: int
:param minor: Minor python version to search for, defaults to None
:param minor: int, optional
:param path: Patch python version to search for, defaults to None
:param path: int, optional
:param int minor: Minor python version to search for, defaults to None
:param int patch: Patch python version to search for, defaults to None
:param bool pre: Search for prereleases (default None) - prioritize releases if None
:param bool dev: Search for devreleases (default None) - prioritize releases if None
:param str arch: Architecture to include, e.g. '64bit', defaults to None
:return: A list of :class:`~pythonfinder.models.PathEntry` instances matching the version requested.
:rtype: List[:class:`~pythonfinder.models.PathEntry`]
"""
sub_finder = operator.methodcaller(
"find_python_version", major, minor=minor, patch=patch, pre=pre, dev=dev
"find_all_python_versions",
major,
minor=minor,
patch=patch,
pre=pre,
dev=dev,
arch=arch,
)
if os.name == "nt" and self.windows_finder:
windows_finder_version = sub_finder(self.windows_finder)
if windows_finder_version:
return windows_finder_version
paths = (self.get_path(k) for k in self.path_order)
path_filter = filter(None, (sub_finder(p) for p in paths if p is not None))
version_sort = operator.attrgetter("as_python.version")
path_filter = filter(
None, unnest((sub_finder(p) for p in paths if p is not None))
)
version_sort = operator.attrgetter("as_python.version_sort")
return [c for c in sorted(path_filter, key=version_sort, reverse=True)]
def find_python_version(self, major, minor=None, patch=None, pre=None, dev=None):
def find_python_version(
self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None
):
"""Search for a specific python version on the path.
:param major: Major python version to search for.
:type major: int
:param minor: Minor python version to search for, defaults to None
:param minor: int, optional
:param path: Patch python version to search for, defaults to None
:param path: int, optional
:param int minor: Minor python version to search for, defaults to None
:param int patch: Patch python version to search for, defaults to None
:param bool pre: Search for prereleases (default None) - prioritize releases if None
:param bool dev: Search for devreleases (default None) - prioritize releases if None
:param str arch: Architecture to include, e.g. '64bit', defaults to None
:return: A :class:`~pythonfinder.models.PathEntry` instance matching the version requested.
:rtype: :class:`~pythonfinder.models.PathEntry`
"""
sub_finder = operator.methodcaller(
"find_python_version", major, minor=minor, patch=patch, pre=pre, dev=dev
"find_python_version",
major,
minor=minor,
patch=patch,
pre=pre,
dev=dev,
arch=arch,
)
if major and minor and patch:
_tuple_pre = pre if pre is not None else False
_tuple_dev = dev if dev is not None else False
version_tuple = (major, minor_, patch, _tuple_pre, _tuple_dev)
version_tuple_pre = (major, minor, patch, True, False)
if os.name == "nt" and self.windows_finder:
windows_finder_version = sub_finder(self.windows_finder)
if windows_finder_version:
return windows_finder_version
paths = (self.get_path(k) for k in self.path_order)
path_filter = filter(None, (sub_finder(p) for p in paths if p is not None))
version_sort = operator.attrgetter("as_python.version")
return next(
version_sort = operator.attrgetter("as_python.version_sort")
ver = next(
(c for c in sorted(path_filter, key=version_sort, reverse=True)), None
)
if ver:
if ver.as_python.version_tuple[:5] in self.python_version_dict:
self.python_version_dict[ver.as_python.version_tuple[:5]].append(ver)
else:
self.python_version_dict[ver.as_python.version_tuple[:5]] = [ver]
return ver
@classmethod
def create(cls, path=None, system=False, only_python=False):
def create(cls, path=None, system=False, only_python=False, global_search=True):
"""Create a new :class:`pythonfinder.models.SystemPath` instance.
:param path: Search path to prepend when searching, defaults to None
@@ -214,7 +285,9 @@ class SystemPath(object):
"""
path_entries = defaultdict(PathEntry)
paths = os.environ.get("PATH").split(os.pathsep)
paths = []
if global_search:
paths = os.environ.get("PATH").split(os.pathsep)
if path:
paths = [path] + paths
_path_objects = [ensure_path(p.strip('"')) for p in paths]
@@ -227,7 +300,13 @@ class SystemPath(object):
for p in _path_objects
}
)
return cls(paths=path_entries, path_order=paths, only_python=only_python, system=system)
return cls(
paths=path_entries,
path_order=paths,
only_python=only_python,
system=system,
global_search=global_search,
)
@attr.s
@@ -237,10 +316,10 @@ class PathEntry(BasePath):
is_root = attr.ib(default=True)
only_python = attr.ib(default=False)
py_version = attr.ib(default=None)
pythons = attr.ib(default=None)
pythons = attr.ib()
def __str__(self):
return fs_str('{0}'.format(self.path.as_posix()))
return fs_str("{0}".format(self.path.as_posix()))
def _filter_children(self):
if self.only_python:
@@ -249,21 +328,38 @@ class PathEntry(BasePath):
children = self.path.iterdir()
return children
@property
@cached_property
def children(self):
if not self._children and self.is_dir and self.is_root:
self._children = {
child.as_posix(): PathEntry(path=child, is_root=False)
child.as_posix(): PathEntry.create(path=child, is_root=False)
for child in self._filter_children()
}
elif not self.is_dir:
self._children = {self.path.as_posix(): self}
return self._children
@property
@pythons.default
def get_pythons(self):
pythons = defaultdict()
if self.is_dir:
for path, entry in self.children.items():
_path = ensure_path(entry.path)
if entry.is_python:
pythons[_path.as_posix()] = entry
else:
if self.is_python:
_path = ensure_path(self.path)
pythons[_path.as_posix()] = copy.deepcopy(self)
return pythons
@cached_property
def as_python(self):
if not self.is_dir and self.is_python:
if not self.py_version:
try:
from .python import PythonVersion
self.py_version = PythonVersion.from_path(self.path)
except (ValueError, InvalidPythonVersion):
self.py_version = None
@@ -286,9 +382,10 @@ class PathEntry(BasePath):
"""
target = ensure_path(path)
_new = cls(
path=target, is_root=is_root, only_python=only_python, pythons=pythons
)
creation_args = {"path": target, "is_root": is_root, "only_python": only_python}
if pythons:
creation_args["pythons"] = pythons
_new = cls(**creation_args)
if pythons and only_python:
children = {}
for pth, python in pythons.items():
@@ -299,22 +396,26 @@ class PathEntry(BasePath):
_new._children = children
return _new
@property
@cached_property
def name(self):
return self.path.name
@property
@cached_property
def is_dir(self):
return self.path.is_dir()
try:
ret_val = self.path.is_dir()
except OSError:
ret_val = False
return ret_val
@property
@cached_property
def is_executable(self):
return path_is_known_executable(self.path)
@property
@cached_property
def is_python(self):
return self.is_executable and (
self.py_version or is_python_name(self.path.name)
self.py_version or looks_like_python(self.path.name)
)
pipenv/vendor/pythonfinder/models/pyenv.py
Vendored
+14 -1
View File
@@ -18,6 +18,7 @@ except ImportError:
class PyenvFinder(BaseFinder):
root = attr.ib(default=None, validator=optional_instance_of(Path))
versions = attr.ib()
pythons = attr.ib()
@versions.default
def get_versions(self):
@@ -31,9 +32,21 @@ class PyenvFinder(BaseFinder):
version.get("is_prerelease"),
version.get("is_devrelease"),
)
versions[version_tuple] = VersionPath.create(path=p.resolve(), only_python=True)
versions[version_tuple] = VersionPath.create(
path=p.resolve(), only_python=True
)
return versions
@pythons.default
def get_pythons(self):
pythons = defaultdict()
for v in self.versions.values():
for p in v.paths.values():
_path = ensure_path(p.path)
if p.is_python:
pythons[_path] = p
return pythons
@classmethod
def create(cls, root):
root = ensure_path(root)
pipenv/vendor/pythonfinder/models/python.py
Vendored
+58 -6
View File
@@ -2,6 +2,7 @@
from __future__ import print_function, absolute_import
import attr
import copy
from collections import defaultdict
import platform
from packaging.version import parse as parse_version, Version
from ..environment import SYSTEM_ARCH
@@ -18,7 +19,7 @@ except ImportError:
class PythonVersion(object):
major = attr.ib(default=0)
minor = attr.ib(default=None)
patch = attr.ib(default=None)
patch = attr.ib(default=0)
is_prerelease = attr.ib(default=False)
is_postrelease = attr.ib(default=False)
is_devrelease = attr.ib(default=False)
@@ -27,6 +28,24 @@ class PythonVersion(object):
comes_from = attr.ib(default=None)
executable = attr.ib(default=None)
@property
def version_sort(self):
"""version_sort tuple for sorting against other instances of the same class.
Returns a tuple of the python version but includes a point for non-dev,
and a point for non-prerelease versions. So released versions will have 2 points
for this value. E.g. `(3, 6, 6, 2)` is a release, `(3, 6, 6, 1)` is a prerelease,
`(3, 6, 6, 0)` is a dev release, and `(3, 6, 6, 3)` is a postrelease.
"""
release_sort = 2
if self.is_postrelease:
release_sort = 3
elif self.is_prerelease:
release_sort = 1
elif self.is_devrelease:
release_sort = 0
return (self.major, self.minor, self.patch if self.patch else 0, release_sort)
@property
def version_tuple(self):
"""Provides a version tuple for using as a dictionary key.
@@ -43,13 +62,18 @@ class PythonVersion(object):
self.is_devrelease,
)
def matches(self, major, minor=None, patch=None, pre=False, dev=False):
def matches(
self, major=None, minor=None, patch=None, pre=False, dev=False, arch=None
):
if arch and arch.isdigit():
arch = "{0}bit".format(arch)
return (
self.major == major
(major is None or self.major == major)
and (minor is None or self.minor == minor)
and (patch is None or self.patch == patch)
and (pre is None or self.is_prerelease == pre)
and (dev is None or self.is_devrelease == dev)
and (arch is None or self.architecture == arch)
)
def as_major(self):
@@ -77,7 +101,7 @@ class PythonVersion(object):
"""
try:
version = parse_version(version)
version = parse_version(str(version))
except TypeError:
raise ValueError("Unable to parse version: %s" % version)
if not version or not version.release:
@@ -119,7 +143,7 @@ class PythonVersion(object):
from .path import PathEntry
if not isinstance(path, PathEntry):
path = PathEntry(path)
path = PathEntry.create(path, is_root=False, only_python=True)
if not path.is_python:
raise ValueError("Not a valid python path: %s" % path.path)
return
@@ -154,7 +178,7 @@ class PythonVersion(object):
creation_dict.update(
{
"architecture": getattr(
launcher_entry, "sys_architecture", SYSTEM_ARCH
launcher_entry.info, "sys_architecture", SYSTEM_ARCH
),
"executable": exe_path,
}
@@ -167,4 +191,32 @@ class PythonVersion(object):
@classmethod
def create(cls, **kwargs):
if "architecture" in kwargs:
if kwargs["architecture"].isdigit():
kwargs["architecture"] = "{0}bit".format(kwargs["architecture"])
return cls(**kwargs)
@attr.s
class VersionMap(object):
versions = attr.ib(default=attr.Factory(defaultdict(list)))
def add_entry(self, entry):
version = entry.as_python
if version:
entries = versions[version.version_tuple]
paths = {p.path for p in self.versions.get(version.version_tuple, [])}
if entry.path not in paths:
self.versions[version.version_tuple].append(entry)
def merge(self, target):
for version, entries in target.versions.items():
if version not in self.versions:
self.versions[version] = entries
else:
current_entries = {p.path for p in self.versions.get(version)}
new_entries = {p.path for p in entries}
new_entries -= current_entries
self.versions[version].append(
[e for e in entries if e.path in new_entries]
)
pipenv/vendor/pythonfinder/models/windows.py
Vendored
+33 -5
View File
@@ -5,7 +5,7 @@ import operator
from collections import defaultdict
from . import BaseFinder
from .path import PathEntry
from .python import PythonVersion
from .python import PythonVersion, VersionMap
from ..exceptions import InvalidPythonVersion
from ..utils import ensure_path
@@ -15,17 +15,37 @@ class WindowsFinder(BaseFinder):
paths = attr.ib(default=attr.Factory(list))
version_list = attr.ib(default=attr.Factory(list))
versions = attr.ib()
pythons = attr.ib()
def find_python_version(self, major, minor=None, patch=None, pre=None, dev=None):
def find_all_python_versions(
self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None
):
version_matcher = operator.methodcaller(
"matches", major, minor=minor, patch=patch, pre=pre, dev=dev
"matches",
major=major,
minor=minor,
patch=patch,
pre=pre,
dev=dev,
arch=arch,
)
py_filter = filter(
None, filter(lambda c: version_matcher(c), self.version_list)
)
version_sort = operator.attrgetter("version")
version_sort = operator.attrgetter("version_sort")
return [c.comes_from for c in sorted(py_filter, key=version_sort, reverse=True)]
def find_python_version(
self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None
):
return next(
(c.comes_from for c in sorted(py_filter, key=version_sort, reverse=True)), None
(
v
for v in self.find_all_python_versions(
major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch
)
),
None,
)
@versions.default
@@ -52,6 +72,14 @@ class WindowsFinder(BaseFinder):
self.paths.append(base_dir)
return versions
@pythons.default
def get_pythons(self):
pythons = defaultdict()
for version in self.version_list:
_path = ensure_path(version.comes_from.path)
pythons[_path.as_posix()] = version.comes_from
return pythons
@classmethod
def create(cls):
return cls()
pipenv/vendor/pythonfinder/pythonfinder.py
Vendored
+78 -25
View File
@@ -2,12 +2,31 @@
from __future__ import print_function, absolute_import
import os
import six
import operator
from .models import SystemPath
class Finder(object):
def __init__(self, path=None, system=False):
def __init__(self, path=None, system=False, global_search=True):
"""Finder A cross-platform Finder for locating python and other executables.
Searches for python and other specified binaries starting in `path`, if supplied,
but searching the bin path of `sys.executable` if `system=True`, and then
searching in the `os.environ['PATH']` if `global_search=True`. When `global_search`
is `False`, this search operation is restricted to the allowed locations of
`path` and `system`.
:param path: A bin-directory search location, defaults to None
:param path: str, optional
:param system: Whether to include the bin-dir of `sys.executable`, defaults to False
:param system: bool, optional
:param global_search: Whether to search the global path from os.environ, defaults to True
:param global_search: bool, optional
:returns: a :class:`~pythonfinder.pythonfinder.Finder` object.
"""
self.path_prepend = path
self.global_search = global_search
self.system = system
self._system_path = None
self._windows_finder = None
@@ -16,7 +35,9 @@ class Finder(object):
def system_path(self):
if not self._system_path:
self._system_path = SystemPath.create(
path=self.path_prepend, system=self.system
path=self.path_prepend,
system=self.system,
global_search=self.global_search,
)
return self._system_path
@@ -31,38 +52,70 @@ class Finder(object):
def which(self, exe):
return self.system_path.which(exe)
def find_python_version(self, major, minor=None, patch=None, pre=None, dev=None):
def find_python_version(
self, major, minor=None, patch=None, pre=None, dev=None, arch=None
):
from .models import PythonVersion
if (
major
and not minor
and not patch
and not pre
and not dev
and isinstance(major, six.string_types)
isinstance(major, six.string_types)
and pre is None
and minor is None
and dev is None
and patch is None
):
from .models import PythonVersion
version_dict = {}
if "." in major:
version_dict = PythonVersion.parse(major)
elif len(major) == 1:
version_dict = {
'major': int(major),
'minor': None,
'patch': None,
'is_prerelease': False,
'is_devrelease': False
}
if arch is None and "-" in major:
major, arch = major.rsplit("-", 1)
if not arch.isdigit():
major = "{0}-{1}".format(major, arch)
else:
arch = "{0}bit".format(arch)
version_dict = PythonVersion.parse(major)
major = version_dict.get("major", major)
minor = version_dict.get("minor", minor)
patch = version_dict.get("patch", patch)
pre = version_dict.get("is_prerelease", pre)
dev = version_dict.get("is_devrelease", dev)
pre = version_dict.get("is_prerelease", pre) if pre is None else pre
dev = version_dict.get("is_devrelease", dev) if dev is None else dev
arch = version_dict.get("architecture", arch) if arch is None else arch
if os.name == "nt":
match = self.windows_finder.find_python_version(
major, minor=minor, patch=patch, pre=pre, dev=dev
major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch
)
if match:
return match
return self.system_path.find_python_version(
major, minor=minor, patch=patch, pre=pre, dev=dev
major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch
)
def find_all_python_versions(
self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None
):
version_sort = operator.attrgetter("as_python.version_sort")
python_version_dict = getattr(self.system_path, "python_version_dict")
if python_version_dict:
paths = filter(
None,
[
path
for version in python_version_dict.values()
for path in version
if path.as_python
],
)
paths = sorted(paths, key=version_sort, reverse=True)
return paths
versions = self.system_path.find_all_python_versions(
major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch
)
if not isinstance(versions, list):
versions = [versions]
paths = sorted(versions, key=version_sort, reverse=True)
path_map = {}
for path in paths:
try:
resolved_path = path.path.resolve()
except OSError:
resolved_path = path.path.absolute()
if not path_map.get(resolved_path.as_posix()):
path_map[resolved_path.as_posix()] = path
return list(path_map.values())
pipenv/vendor/pythonfinder/utils.py
Vendored
+17 -8
View File
@@ -8,6 +8,7 @@ import subprocess
import sys
from fnmatch import fnmatch
from .exceptions import InvalidPythonVersion
from itertools import chain
try:
from pathlib import Path
@@ -31,10 +32,7 @@ def _run(cmd):
"""
encoding = locale.getdefaultlocale()[1] or "utf-8"
c = subprocess.Popen(
cmd,
env=os.environ.copy(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cmd, env=os.environ.copy(), stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
out, err = c.communicate()
return out.decode(encoding).strip(), err.decode(encoding).strip()
@@ -72,7 +70,7 @@ def path_is_known_executable(path):
)
def is_python_name(name):
def looks_like_python(name):
rules = ["*python", "*python?", "*python?.?", "*python?.?m"]
match_rules = []
for rule in rules:
@@ -88,7 +86,7 @@ def is_python_name(name):
def path_is_python(path):
return path_is_executable(path) and is_python_name(path.name)
return path_is_executable(path) and looks_like_python(path.name)
def ensure_path(path):
@@ -101,8 +99,13 @@ def ensure_path(path):
"""
if isinstance(path, Path):
return Path(os.path.expandvars(path.as_posix()))
return Path(os.path.expandvars(path))
path = path.as_posix()
path = Path(os.path.expandvars(path))
try:
path = path.resolve()
except OSError:
path = path.absolute()
return path
def _filter_none(k, v):
@@ -132,3 +135,9 @@ def fs_str(string):
_fs_encoding = sys.getfilesystemencoding() or sys.getdefaultencoding()
def unnest(item):
if isinstance(next((i for i in item), None), (list, tuple)):
return chain(*filter(None, item))
return chain(filter(None, item))
pipenv/vendor/vendor.txt
Vendored
+2 -1
View File
@@ -21,7 +21,7 @@ git+https://github.com/naiquevin/pipdeptree.git@ee5eaf86ed0f49ea97601475e048d81e
pipreqs==0.4.9
docopt==0.6.2
yarg==0.1.9
pythonfinder
pythonfinder==1.0.0
requests==2.19.1
chardet==3.0.4
idna==2.7
@@ -39,3 +39,4 @@ six==1.11.0
semver==2.8.0
shutilwhich==1.1.0
toml==0.9.4
cached-property==1.4.3
run-tests.sh
+10 -1
View File
@@ -15,8 +15,17 @@ if [[ ! -z "$TEST_SUITE" ]]; then
echo "Using TEST_SUITE=$TEST_SUITE"
fi
export PATH="$HOME/.local/bin:$PATH"
HOME=$(readlink -f ~/)
if [[ -z "$HOME" ]]; then
if [[ "$USER" == "root" ]]; then
HOME="/root"
fi
fi
if [[ ! -z "$HOME" ]]; then
export PATH="${HOME}/.local/bin:${PATH}"
fi
# pip uninstall -y pipenv
echo "Path: $PATH"
echo "Installing Pipenv…"
pip install -e "$(pwd)" --upgrade
pipenv install --deploy --dev
tasks/vendoring/__init__.py
+70 -26
View File
@@ -280,19 +280,47 @@ def write_backport_imports(ctx, vendor_dir):
backport_init.write_text('\n'.join(init_py_lines) + '\n')
def vendor(ctx, vendor_dir, rewrite=True):
log('Reinstalling vendored libraries')
is_patched = vendor_dir.name == 'patched'
requirements_file = vendor_dir.name
def _ensure_package_in_requirements(ctx, requirements_file, package):
requirement = None
log('using requirements file: %s' % requirements_file)
req_file_lines = [l for l in requirements_file.read_text().splitlines()]
if package:
match = [r for r in req_file_lines if r.strip().lower().startswith(package)]
matched_req = None
if match:
for m in match:
specifiers = [m.index(s) for s in ['>', '<', '=', '~'] if s in m]
if m.lower() == package or (specifiers and m[:min(specifiers)].lower() == package):
matched_req = "{0}".format(m)
requirement = matched_req
log("Matched req: %r" % matched_req)
if not matched_req:
req_file_lines.append("{0}".format(package))
log("Writing requirements file: %s" % requirements_file)
requirements_file.write_text('\n'.join(req_file_lines))
requirement = "{0}".format(package)
return requirement
def install(ctx, vendor_dir, package=None):
requirements_file = vendor_dir / "{0}.txt".format(vendor_dir.name)
requirement = "-r {0}".format(requirements_file.as_posix())
log('Using requirements file: %s' % requirement)
if package:
requirement = _ensure_package_in_requirements(ctx, requirements_file, package)
# We use --no-deps because we want to ensure that all of our dependencies
# are added to vendor.txt, this includes all dependencies recursively up
# the chain.
ctx.run(
'pip install -t {0} -r {0}/{1}.txt --no-compile --no-deps'.format(
str(vendor_dir),
requirements_file,
'pip install -t {0} --no-compile --no-deps --upgrade {1}'.format(
vendor_dir.as_posix(),
requirement,
)
)
def post_install_cleanup(ctx, vendor_dir):
remove_all(vendor_dir.glob('*.dist-info'))
remove_all(vendor_dir.glob('*.egg-info'))
@@ -300,6 +328,13 @@ def vendor(ctx, vendor_dir, rewrite=True):
drop_dir(vendor_dir / 'bin')
drop_dir(vendor_dir / 'tests')
def vendor(ctx, vendor_dir, package=None, rewrite=True):
log('Reinstalling vendored libraries')
is_patched = vendor_dir.name == 'patched'
install(ctx, vendor_dir, package=package)
log('Running post-install cleanup...')
post_install_cleanup(ctx, vendor_dir)
# Detect the vendored packages/modules
vendored_libs = detect_vendored_libs(_get_vendor_dir(ctx))
patched_libs = detect_vendored_libs(_get_patched_dir(ctx))
@@ -320,25 +355,26 @@ def vendor(ctx, vendor_dir, rewrite=True):
log('Renaming specified libs...')
for item in vendor_dir.iterdir():
if item.is_dir():
if rewrite:
if rewrite and not package or (package and item.name.lower() in package):
log('Rewriting imports for %s...' % item)
rewrite_imports(item, vendored_libs, vendor_dir)
rename_if_needed(ctx, vendor_dir, item)
elif item.name not in FILE_WHITE_LIST:
if rewrite:
if rewrite and not package or (package and item.stem.lower() in package):
rewrite_file_imports(item, vendored_libs, vendor_dir)
write_backport_imports(ctx, vendor_dir)
log('Applying post-patches...')
patches = patch_dir.glob('*.patch' if not is_patched else '_post*.patch')
for patch in patches:
apply_patch(ctx, patch)
if is_patched:
piptools_vendor = vendor_dir / 'piptools' / '_vendored'
if piptools_vendor.exists():
drop_dir(piptools_vendor)
msgpack = vendor_dir / 'notpip' / '_vendor' / 'msgpack'
if msgpack.exists():
remove_all(msgpack.glob('*.so'))
if not package:
log('Applying post-patches...')
patches = patch_dir.glob('*.patch' if not is_patched else '_post*.patch')
for patch in patches:
apply_patch(ctx, patch)
if is_patched:
piptools_vendor = vendor_dir / 'piptools' / '_vendored'
if piptools_vendor.exists():
drop_dir(piptools_vendor)
msgpack = vendor_dir / 'notpip' / '_vendor' / 'msgpack'
if msgpack.exists():
remove_all(msgpack.glob('*.so'))
@invoke.task
@@ -371,16 +407,19 @@ def rewrite_all_imports(ctx):
@invoke.task
def download_licenses(ctx, vendor_dir, requirements_file='vendor.txt'):
def download_licenses(ctx, vendor_dir=None, requirements_file='vendor.txt', package=None):
log('Downloading licenses')
if not vendor_dir:
vendor_dir = _get_vendor_dir(ctx)
requirements_file = vendor_dir / requirements_file
requirement = "-r {0}".format(requirements_file.as_posix())
if package:
requirement = _ensure_package_in_requirements(ctx, requirements_file, package)
tmp_dir = vendor_dir / '__tmp__'
ctx.run(
'pip download -r {0}/{1} --no-binary :all: --no-deps -d {2}'.format(
str(vendor_dir),
requirements_file,
str(tmp_dir),
'pip download --no-binary :all: --no-deps -d {0} {1}'.format(
tmp_dir.as_posix(),
requirement,
)
)
for sdist in tmp_dir.iterdir():
@@ -503,10 +542,15 @@ def generate_patch(ctx, package_path, patch_description, base='HEAD'):
@invoke.task(name=TASK_NAME)
def main(ctx):
def main(ctx, package=None):
vendor_dir = _get_vendor_dir(ctx)
patched_dir = _get_patched_dir(ctx)
log('Using vendor dir: %s' % vendor_dir)
if package:
vendor(ctx, vendor_dir, package=package)
download_licenses(ctx, vendor_dir, package=package)
log("Vendored %s" % package)
return
clean_vendor(ctx, vendor_dir)
clean_vendor(ctx, patched_dir)
vendor(ctx, vendor_dir)
tasks/vendoring/patches/patched/piptools.patch
+50 -25
View File
@@ -19,18 +19,22 @@ index 4e6174c..75f9b49 100644
# NOTE
# We used to store the cache dir under ~/.pip-tools, which is not the
diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py
index 1c4b943..c922be1 100644
index 1c4b943..91902dc 100644
--- a/pipenv/patched/piptools/repositories/pypi.py
+++ b/pipenv/patched/piptools/repositories/pypi.py
@@ -4,6 +4,7 @@ from __future__ import (absolute_import, division, print_function,
@@ -1,9 +1,10 @@
# coding: utf-8
from __future__ import (absolute_import, division, print_function,
unicode_literals)
-
+import copy
import hashlib
import os
+import sys
from contextlib import contextmanager
from shutil import rmtree
@@ -15,13 +16,24 @@ from .._compat import (
@@ -15,13 +16,22 @@ from .._compat import (
Wheel,
FAVORITE_HASH,
TemporaryDirectory,
@@ -40,25 +44,23 @@ index 1c4b943..c922be1 100644
+ SafeFileCache,
)
+from pip._vendor.packaging.requirements import InvalidRequirement, Requirement
+from pip._vendor.packaging.version import Version, InvalidVersion, parse as parse_version
+from pip._vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier, Specifier
+from pip._vendor.packaging.markers import Marker, Op, Value, Variable
+from pip._vendor.pyparsing import ParseException
-from ..cache import CACHE_DIR
+from pip._vendor.packaging.requirements import Requirement
+from pip._vendor.packaging.specifiers import SpecifierSet, Specifier
+from pip._vendor.packaging.markers import Op, Value, Variable
+from pip._internal.exceptions import InstallationError
+from pip._internal.vcs import VcsSupport
+
from ..cache import CACHE_DIR
+from pipenv.environments import PIPENV_CACHE_DIR
from ..exceptions import NoCandidateFound
-from ..utils import (fs_str, is_pinned_requirement, lookup_table,
from ..utils import (fs_str, is_pinned_requirement, lookup_table,
- make_install_requirement)
+from ..utils import (fs_str, is_pinned_requirement, lookup_table, as_tuple, key_from_req,
+ make_install_requirement, format_requirement, dedup, clean_requires_python)
+ make_install_requirement, clean_requires_python)
+
from .base import BaseRepository
@@ -37,6 +49,40 @@ except ImportError:
@@ -37,6 +47,45 @@ except ImportError:
from pip.wheel import WheelCache
@@ -77,15 +79,20 @@ index 1c4b943..c922be1 100644
+ def get_hash(self, location):
+ # if there is no location hash (i.e., md5 / sha256 / etc) we on't want to store it
+ hash_value = None
+ can_hash = location.hash
+ vcs = VcsSupport()
+ orig_scheme = location.scheme
+ new_location = copy.deepcopy(location)
+ if orig_scheme in vcs.all_schemes:
+ new_location.url = new_location.url.split("+", 1)[-1]
+ can_hash = new_location.hash
+ if can_hash:
+ # hash url WITH fragment
+ hash_value = self.get(location.url)
+ hash_value = self.get(new_location.url)
+ if not hash_value:
+ hash_value = self._get_file_hash(location)
+ hash_value = self._get_file_hash(new_location)
+ hash_value = hash_value.encode('utf8')
+ if can_hash:
+ self.set(location.url, hash_value)
+ self.set(new_location.url, hash_value)
+ return hash_value.decode('utf8')
+
+ def _get_file_hash(self, location):
@@ -99,7 +106,7 @@ index 1c4b943..c922be1 100644
class PyPIRepository(BaseRepository):
DEFAULT_INDEX_URL = PyPI.simple_url
@@ -46,10 +92,11 @@ class PyPIRepository(BaseRepository):
@@ -46,10 +95,11 @@ class PyPIRepository(BaseRepository):
config), but any other PyPI mirror can be used if index_urls is
changed/configured on the Finder.
"""
@@ -113,7 +120,7 @@ index 1c4b943..c922be1 100644
index_urls = [pip_options.index_url] + pip_options.extra_index_urls
if pip_options.no_index:
@@ -74,11 +121,15 @@ class PyPIRepository(BaseRepository):
@@ -74,11 +124,15 @@ class PyPIRepository(BaseRepository):
# of all secondary dependencies for the given requirement, so we
# only have to go to disk once for each requirement
self._dependencies_cache = {}
@@ -131,7 +138,7 @@ index 1c4b943..c922be1 100644
def freshen_build_caches(self):
"""
@@ -114,10 +165,14 @@ class PyPIRepository(BaseRepository):
@@ -114,10 +168,14 @@ class PyPIRepository(BaseRepository):
if ireq.editable:
return ireq # return itself as the best match
@@ -148,7 +155,7 @@ index 1c4b943..c922be1 100644
# Reuses pip's internal candidate sort key to sort
matching_candidates = [candidates_by_version[ver] for ver in matching_versions]
@@ -126,11 +181,71 @@ class PyPIRepository(BaseRepository):
@@ -126,11 +184,71 @@ class PyPIRepository(BaseRepository):
best_candidate = max(matching_candidates, key=self.finder._candidate_sort_key)
# Turn the candidate into a pinned InstallRequirement
@@ -223,7 +230,7 @@ index 1c4b943..c922be1 100644
"""
Given a pinned or an editable InstallRequirement, returns a set of
dependencies (also InstallRequirements, but not necessarily pinned).
@@ -155,20 +270,40 @@ class PyPIRepository(BaseRepository):
@@ -155,20 +273,47 @@ class PyPIRepository(BaseRepository):
os.makedirs(download_dir)
if not os.path.isdir(self._wheel_download_dir):
os.makedirs(self._wheel_download_dir)
@@ -235,6 +242,13 @@ index 1c4b943..c922be1 100644
+ dist = None
+ if ireq.editable:
+ try:
+ from pipenv.utils import chdir
+ with chdir(ireq.setup_py_dir):
+ from setuptools.dist import distutils
+ distutils.core.run_setup(ireq.setup_py)
+ except (ImportError, InstallationError, TypeError, AttributeError):
+ pass
+ try:
+ dist = ireq.get_dist()
+ except InstallationError:
+ ireq.run_egg_info()
@@ -268,7 +282,7 @@ index 1c4b943..c922be1 100644
)
except TypeError:
# Pip >= 10 (new resolver!)
@@ -188,17 +323,97 @@ class PyPIRepository(BaseRepository):
@@ -188,17 +333,97 @@ class PyPIRepository(BaseRepository):
finder=self.finder,
session=self.session,
upgrade_strategy="to-satisfy-only",
@@ -369,7 +383,18 @@ index 1c4b943..c922be1 100644
return set(self._dependencies_cache[ireq])
def get_hashes(self, ireq):
@@ -217,24 +432,22 @@ class PyPIRepository(BaseRepository):
@@ -210,6 +435,10 @@ class PyPIRepository(BaseRepository):
if ireq.editable:
return set()
+ vcs = VcsSupport()
+ if ireq.link and ireq.link.scheme in vcs.all_schemes and 'ssh' in ireq.link.scheme:
+ return set()
+
if not is_pinned_requirement(ireq):
raise TypeError(
"Expected pinned requirement, got {}".format(ireq))
@@ -217,24 +446,22 @@ class PyPIRepository(BaseRepository):
# We need to get all of the candidates that match our current version
# pin, these will represent all of the files that could possibly
# satisfy this constraint.
tests/integration/test_lock.py
+24
View File
@@ -385,3 +385,27 @@ django = "*"
django_version = '==2.0.6' if py_version.startswith('3') else '==1.11.13'
assert py_version == '2.7.14'
assert p.lockfile['default']['django']['version'] == django_version
@pytest.mark.lock
@pytest.mark.install
def test_lockfile_corrupted(PipenvInstance):
with PipenvInstance() as p:
with open(p.lockfile_path, 'w') as f:
f.write('{corrupted}')
c = p.pipenv('install')
assert c.return_code == 0
assert 'Pipfile.lock is corrupted' in c.err
assert p.lockfile['_meta']
@pytest.mark.lock
@pytest.mark.install
def test_lockfile_with_empty_dict(PipenvInstance):
with PipenvInstance() as p:
with open(p.lockfile_path, 'w') as f:
f.write('{}')
c = p.pipenv('install')
assert c.return_code == 0
assert 'Pipfile.lock is corrupted' in c.err
assert p.lockfile['_meta']