From 9560aa60aaea107ab80b265160b33dd7e5675aca Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 19 Nov 2018 01:09:32 -0500 Subject: [PATCH 1/5] Update requirementslib and pythonfinder - Fixes #3242 - Fixes #3232 - Fixes #3229 - Fixes #3234 Signed-off-by: Dan Ryan --- news/3254.bugfix.rst | 1 + pipenv/vendor/pythonfinder/environment.py | 2 +- pipenv/vendor/pythonfinder/models/path.py | 11 +++++- pipenv/vendor/pythonfinder/models/python.py | 38 +++++++++++++------ pipenv/vendor/pythonfinder/utils.py | 18 ++++++++- .../vendor/requirementslib/models/lockfile.py | 17 ++++++--- .../vendor/requirementslib/models/pipfile.py | 2 +- .../requirementslib/models/setup_info.py | 7 ++-- 8 files changed, 69 insertions(+), 27 deletions(-) create mode 100644 news/3254.bugfix.rst diff --git a/news/3254.bugfix.rst b/news/3254.bugfix.rst new file mode 100644 index 00000000..11b18201 --- /dev/null +++ b/news/3254.bugfix.rst @@ -0,0 +1 @@ +Updated ``requirementslib`` and ``pythonfinder`` for multiple bugfixes. diff --git a/pipenv/vendor/pythonfinder/environment.py b/pipenv/vendor/pythonfinder/environment.py index ec4a760f..0ea09109 100644 --- a/pipenv/vendor/pythonfinder/environment.py +++ b/pipenv/vendor/pythonfinder/environment.py @@ -7,7 +7,7 @@ import sys PYENV_INSTALLED = bool(os.environ.get("PYENV_SHELL")) or bool( os.environ.get("PYENV_ROOT") ) -ASDF_INSTALLED = bool(os.environ.get("ASDF_DATA_DIR")) +ASDF_INSTALLED = bool(os.environ.get("ASDF_DIR")) PYENV_ROOT = os.path.expanduser( os.path.expandvars(os.environ.get("PYENV_ROOT", "~/.pyenv")) ) diff --git a/pipenv/vendor/pythonfinder/models/path.py b/pipenv/vendor/pythonfinder/models/path.py index 7fae3312..42608cde 100644 --- a/pipenv/vendor/pythonfinder/models/path.py +++ b/pipenv/vendor/pythonfinder/models/path.py @@ -484,7 +484,14 @@ class PathEntry(BasePath): for child in self._filter_children(): if any(shim in normalize_path(str(child)) for shim in SHIM_PATHS): continue - yield (child.as_posix(), PathEntry.create(path=child, **pass_args)) + if self.only_python: + try: + entry = PathEntry.create(path=child, **pass_args) + except (InvalidPythonVersion, ValueError): + continue + else: + entry = PathEntry.create(path=child, **pass_args) + yield (child.as_posix(), entry) return @cached_property @@ -508,7 +515,7 @@ class PathEntry(BasePath): if self.is_python: try: py_version = PythonVersion.from_path(path=self, name=self.name) - except InvalidPythonVersion: + except (InvalidPythonVersion, ValueError): py_version = None except Exception: if not IGNORE_UNSUPPORTED: diff --git a/pipenv/vendor/pythonfinder/models/python.py b/pipenv/vendor/pythonfinder/models/python.py index eac856c5..d7de3e05 100644 --- a/pipenv/vendor/pythonfinder/models/python.py +++ b/pipenv/vendor/pythonfinder/models/python.py @@ -25,6 +25,7 @@ from ..utils import ( is_in_path, parse_pyenv_version_order, parse_asdf_version_order, + parse_python_version, ) logger = logging.getLogger(__name__) @@ -361,19 +362,32 @@ class PythonVersion(object): try: version = parse_version(str(version)) except TypeError: - raise ValueError("Unable to parse version: %s" % version) - if not version or not version.release: - raise ValueError("Not a valid python version: %r" % version) - return - if len(version.release) >= 3: - major, minor, patch = version.release[:3] - elif len(version.release) == 2: - major, minor = version.release - patch = None + try: + version_dict = parse_python_version(str(version)) + except Exception: + raise ValueError("Unable to parse version: %s" % version) + else: + if not version_dict: + raise ValueError("Not a valid python version: %r" % version) + major = int(version_dict.get("major")) + minor = int(version_dict.get("minor")) + patch = version_dict.get("patch") + if patch: + patch = int(patch) + version = ".".join([v for v in [major, minor, patch] if v is not None]) + version = parse_version(version) else: - major = version.release[0] - minor = None - patch = None + if not version or not version.release: + raise ValueError("Not a valid python version: %r" % version) + if len(version.release) >= 3: + major, minor, patch = version.release[:3] + elif len(version.release) == 2: + major, minor = version.release + patch = None + else: + major = version.release[0] + minor = None + patch = None return { "major": major, "minor": minor, diff --git a/pipenv/vendor/pythonfinder/utils.py b/pipenv/vendor/pythonfinder/utils.py index 2debd80e..f8ec1972 100644 --- a/pipenv/vendor/pythonfinder/utils.py +++ b/pipenv/vendor/pythonfinder/utils.py @@ -8,6 +8,7 @@ from fnmatch import fnmatch import attr import io +import re import six import vistir @@ -24,6 +25,9 @@ except ImportError: from backports.functools_lru_cache import lru_cache +version_re = re.compile(r"(?P[0-9]+)\.(?P[0-9]+)\.?(?P(?<=\.)[0-9]+)") + + PYTHON_IMPLEMENTATIONS = ( "python", "ironpython", "jython", "pypy", "anaconda", "miniconda", "stackless", "activepython", "micropython" @@ -46,13 +50,13 @@ for rule in RULES: ) -@lru_cache(maxsize=128) +@lru_cache(maxsize=1024) def get_python_version(path): """Get python version string using subprocess from a given path.""" version_cmd = [path, "-c", "import sys; print(sys.version.split()[0])"] try: c = vistir.misc.run(version_cmd, block=True, nospin=True, return_object=True, - combine_stderr=False) + combine_stderr=False, write_to_stdout=False) except OSError: raise InvalidPythonVersion("%s is not a valid python path" % path) if not c.out: @@ -60,6 +64,14 @@ def get_python_version(path): return c.out.strip() +@lru_cache(maxsize=1024) +def parse_python_version(version_str): + m = version_re.match(version_str) + if not m: + raise InvalidPythonVersion("%s is not a python version" % version_str) + return m.groupdict() + + def optional_instance_of(cls): return attr.validators.optional(attr.validators.instance_of(cls)) @@ -151,6 +163,7 @@ def parse_pyenv_version_order(filename="version"): contents = fh.read() version_order = [v for v in contents.splitlines()] return version_order + return [] def parse_asdf_version_order(filename=".tool-versions"): @@ -165,6 +178,7 @@ def parse_asdf_version_order(filename=".tool-versions"): python_key, _, versions = python_section.partition(" ") if versions: return versions.split() + return [] # TODO: Reimplement in vistir diff --git a/pipenv/vendor/requirementslib/models/lockfile.py b/pipenv/vendor/requirementslib/models/lockfile.py index 9d19edaf..54b2761c 100644 --- a/pipenv/vendor/requirementslib/models/lockfile.py +++ b/pipenv/vendor/requirementslib/models/lockfile.py @@ -160,7 +160,7 @@ class Lockfile(object): path = os.curdir path = Path(path).absolute() project_path = path if path.is_dir() else path.parent - lockfile_path = project_path / "Pipfile.lock" + lockfile_path = path if path.is_file() else project_path / "Pipfile.lock" if not project_path.exists(): raise OSError("Project does not exist: %s" % project_path.as_posix()) elif not lockfile_path.exists() and not create: @@ -168,7 +168,12 @@ class Lockfile(object): projectfile = cls.read_projectfile(lockfile_path.as_posix()) if not lockfile_path.exists(): if not data: - lf = cls.lockfile_from_pipfile(project_path.joinpath("Pipfile")) + path_str = lockfile_path.as_posix() + if path_str[-5:] == ".lock": + pipfile = Path(path_str[:-5]) + else: + pipfile = project_path.joinpath("Pipfile") + lf = cls.lockfile_from_pipfile(pipfile) else: lf = plette.lockfiles.Lockfile(data) projectfile.model = lf @@ -212,7 +217,7 @@ class Lockfile(object): def load(cls, path, create=True): """Create a new lockfile instance. - :param project_path: Path to project root + :param project_path: Path to project root or lockfile :type project_path: str or :class:`pathlib.Path` :param str lockfile_name: Name of the lockfile in the project root directory :param pipfile_path: Path to the project pipfile @@ -225,9 +230,9 @@ class Lockfile(object): projectfile = cls.load_projectfile(path, create=create) except JSONDecodeError: path = os.path.abspath(path) - if not os.path.isdir(path): - path = os.path.dirname(path) - path = Path(os.path.join(path, "Pipfile.lock")) + path = Path( + os.path.join(path, "Pipfile.lock") if os.path.isdir(path) else path + ) formatted_path = path.as_posix() backup_path = "%s.bak" % formatted_path LockfileCorruptException.show(formatted_path, backup_path=backup_path) diff --git a/pipenv/vendor/requirementslib/models/pipfile.py b/pipenv/vendor/requirementslib/models/pipfile.py index 0f6de6bf..e3d353d9 100644 --- a/pipenv/vendor/requirementslib/models/pipfile.py +++ b/pipenv/vendor/requirementslib/models/pipfile.py @@ -187,7 +187,7 @@ class Pipfile(object): raise RuntimeError("Must pass a path to classmethod 'Pipfile.load'") if not isinstance(path, Path): path = Path(path).absolute() - pipfile_path = path if path.name == "Pipfile" else path.joinpath("Pipfile") + pipfile_path = path if path.is_file() else path.joinpath("Pipfile") project_path = pipfile_path.parent if not project_path.exists(): raise FileNotFoundError("%s is not a valid project path!" % path) diff --git a/pipenv/vendor/requirementslib/models/setup_info.py b/pipenv/vendor/requirementslib/models/setup_info.py index 006ee609..01e7ce67 100644 --- a/pipenv/vendor/requirementslib/models/setup_info.py +++ b/pipenv/vendor/requirementslib/models/setup_info.py @@ -103,7 +103,7 @@ def iter_egginfos(path, pkg_name=None): if not entry.name.endswith("egg-info"): for dir_entry in iter_egginfos(entry.path, pkg_name=pkg_name): yield dir_entry - elif pkg_name is None or entry.name.startswith(pkg_name): + elif pkg_name is None or entry.name.startswith(pkg_name.replace("-", "_")): yield entry @@ -224,7 +224,7 @@ class SetupInfo(object): target_cwd = self.setup_py.parent.as_posix() with cd(target_cwd), _suppress_distutils_logs(): script_name = self.setup_py.as_posix() - args = ["egg_info", "--egg-base", self.base_dir] + args = ["egg_info", self.base_dir] g = {"__file__": script_name, "__name__": "__main__"} local_dict = {} if sys.version_info < (3, 5): @@ -247,7 +247,8 @@ class SetupInfo(object): except NameError: python = os.environ.get('PIP_PYTHON_PATH', sys.executable) out, _ = run([python, "setup.py"] + args, cwd=target_cwd, block=True, - combine_stderr=False, return_object=False, nospin=True) + combine_stderr=False, return_object=False, nospin=True, + write_to_stdout=False) finally: _setup_stop_after = None sys.argv = save_argv From fad64653cc884072500836b04e225c23ec1a3bc9 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 19 Nov 2018 01:13:00 -0500 Subject: [PATCH 2/5] Remove local tests Signed-off-by: Dan Ryan --- run_local_tests.sh | 57 ---------------------------------------------- 1 file changed, 57 deletions(-) delete mode 100755 run_local_tests.sh diff --git a/run_local_tests.sh b/run_local_tests.sh deleted file mode 100755 index d3ecb3d6..00000000 --- a/run_local_tests.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash -owner=$USER -IFS=' ' read -a groups <<< $(groups) -group=${groups[0]} -python=$(python -c "import sys; print(sys.executable)") -python2=$(python2.7 -c "import sys; print(sys.executable)") -#python3=$(pyenv which python3.7) -ramdisk="/mnt/ramdisk" -venv_path="$ramdisk/.venv" -venv_bin="$venv_path/bin" -# pip="$venv_bin/pip" -# pipenv="$venv_bin/pipenv" -py="$venv_bin/python" -pip="$py -m pip" -pipenv="$py -m pipenv" -venv2_path="$ramdisk/.venv2" -# pipenv2="$venv2_bin/pipenv" -# pip2="$venv2_bin/pip" -venv2_bin="$venv2_path/bin" -py2="$venv2_bin/python" -pip2="$py2 -m pip" -pipenv2="$py2 -m pipenv" -#venv3_path="$ramdisk/.venv3" -#venv3_bin="$venv3_path/bin" -#pip3="$venv3_bin/pip" -#pipenv3="$venv3_bin/pipenv" - - -export RAM_DISK="/mnt/ramdisk/" - -[ ! -e $venv_path ] && sudo mount -t ramfs -o size=2g ramfs $ramdisk \ - && sudo chown -R $owner:$group $ramdisk - -[ ! -e $venv_path ] && python -m virtualenv --python=$python $venv_path \ - && VIRTUAL_ENV="$venv_path" $pip install -e . \ - && VIRTUAL_ENV="$venv_path" $pipenv run pip install -e . \ - && VIRTUAL_ENV="$venv_path" $pipenv install --dev - -[ ! -e $venv2_path ] && python -m virtualenv --python=$python2 $venv2_path \ - && VIRTUAL_ENV="$venv2_path" $pip2 install pathlib2 -e . \ - && VIRTUAL_ENV="$venv2_path" $pipenv2 run pip install -e . \ - && VIRTUAL_ENV="$venv2_path" $pipenv2 install --dev - -#[ ! -e $venv3_path ] && python -m virtualenv --python=$python3 $venv3_path \ - #&& $pip3 install -e . \ - #&& $pipenv3 run pip install -e . \ - #&& $pipenv3 install --dev - - -#export PYPI_VENDOR_DIR="$(pwd)/tests/pypi/" -PIP_PROCESS_DEPENDENCY_LINKS=1 && VIRTUAL_ENV="$venv2_path" $pipenv run pytest tests/ $@ 2>&1 -PIP_PROCESS_DEPENDENCY_LINKS=1 && VIRTUAL_ENV="$venv2_path" $pipenv2 run pytest tests/ $@ 2>&1 -#$pipenv run pytest -v -n 4 --ignore=pipenv/vendor --ignore=pipenv/patched --ignore=build --ignore=tests/pypi --ignore=tests/pytest-pypi/pypi -p tests.pytest-pypi.pytest_pypi tests/ $@ 2>&1 -#$pipenv2 run pytest -v -n 4 --ignore=pipenv/vendor --ignore=pipenv/patched --ignore=build --ignore=tests/pypi --ignore=tests/pytest-pypi/pypi -p tests.pytest-pypi.pytest_pypi tests/ $@ 2>&1 -#$pipenv3 run pytest -v -n 4 --ignore=pipenv/vendor --ignore=pipenv/patched tests/ $@ 2>&1 -# && /mnt/ramdisk/.venv/bin/pip install dist/pipenv-11.10.1.dev1-py2.py3-none-any.whl \ -# && /mnt/ramdisk/.venv/bin/pipenv run pip install dist/pipenv-11.10.1.dev1-py2.py3-none-any.whl \ From de2cd76af74e104544455447db8ef6ce2aa74989 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 19 Nov 2018 01:14:10 -0500 Subject: [PATCH 3/5] fix manifest Signed-off-by: Dan Ryan --- MANIFEST.in | 1 + pipenv/vendor/vistir/misc.py | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 3c8eb1d4..d5e7d3d8 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -15,6 +15,7 @@ include pipenv/vendor/pipreqs/stdlib pipenv/vendor/pipreqs/mapping include pipenv/vendor/*.txt pipenv/vendor/pexpect/bashrc.sh include pipenv/vendor/Makefile include pipenv/pipenv.1 +include .gitmodules exclude .editorconfig .travis.yml .env appveyor.yml tox.ini pytest.ini exclude Pipfile* CHANGELOG.draft.rst exclude docker-compose.yml Dockerfile diff --git a/pipenv/vendor/vistir/misc.py b/pipenv/vendor/vistir/misc.py index 110766b4..f7ed26b4 100644 --- a/pipenv/vendor/vistir/misc.py +++ b/pipenv/vendor/vistir/misc.py @@ -138,7 +138,6 @@ def _spawn_subprocess(script, env=None, block=True, cwd=None, combine_stderr=Tru return subprocess.Popen(script.cmdify(), **options) - def _create_subprocess( cmd, env=None, @@ -153,7 +152,7 @@ def _create_subprocess( write_to_stdout=True ): if not env: - env = {} + env = os.environ.copy() try: c = _spawn_subprocess(cmd, env=env, block=block, cwd=cwd, combine_stderr=combine_stderr) From fca047e4a1ccacc21f284d65ec91d954f16a3ce1 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 19 Nov 2018 09:19:32 -0500 Subject: [PATCH 4/5] Exclude gitmodules Signed-off-by: Dan Ryan --- MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 30d340ec..5e012535 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -15,7 +15,7 @@ include pipenv/vendor/pipreqs/stdlib pipenv/vendor/pipreqs/mapping include pipenv/vendor/*.txt pipenv/vendor/pexpect/bashrc.sh include pipenv/vendor/Makefile include pipenv/pipenv.1 -inlcude .gitmodules +exclude .gitmodules exclude .editorconfig .travis.yml .env appveyor.yml tox.ini pytest.ini exclude Pipfile* CHANGELOG.draft.rst exclude docker-compose.yml Dockerfile From d15d17b514e491d1844d5f5e32d98985609db316 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 19 Nov 2018 16:00:50 -0500 Subject: [PATCH 5/5] Fix setup_info merge Signed-off-by: Dan Ryan --- pipenv/vendor/requirementslib/models/setup_info.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pipenv/vendor/requirementslib/models/setup_info.py b/pipenv/vendor/requirementslib/models/setup_info.py index 01e7ce67..cd3c3477 100644 --- a/pipenv/vendor/requirementslib/models/setup_info.py +++ b/pipenv/vendor/requirementslib/models/setup_info.py @@ -223,16 +223,16 @@ class SetupInfo(object): if self.setup_py is not None and self.setup_py.exists(): target_cwd = self.setup_py.parent.as_posix() with cd(target_cwd), _suppress_distutils_logs(): + # This is for you, Hynek + # see https://github.com/hynek/environ_config/blob/69b1c8a/setup.py script_name = self.setup_py.as_posix() - args = ["egg_info", self.base_dir] + args = ["egg_info"] g = {"__file__": script_name, "__name__": "__main__"} local_dict = {} if sys.version_info < (3, 5): save_argv = sys.argv else: save_argv = sys.argv.copy() - # This is for you, Hynek - # see https://github.com/hynek/environ_config/blob/69b1c8a/setup.py try: global _setup_distribution, _setup_stop_after _setup_stop_after = "run"