From d838d7b4becfcba0faf6d8a75710d2c020a67f55 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Fri, 29 Mar 2019 00:33:18 -0400 Subject: [PATCH] Update locking to respect pep517 and isolation - Fix verbosity and logging - Clean minor fixes Signed-off-by: Dan Ryan --- pipenv/core.py | 18 ++-- pipenv/patched/piptools/repositories/pypi.py | 15 ++-- pipenv/utils.py | 83 ++++++++++++++----- .../vendoring/patches/patched/piptools.patch | 44 +++++----- 4 files changed, 102 insertions(+), 58 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index e3c61618..eaefbb5d 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1410,10 +1410,11 @@ def pip_install( line = "{0}&subdirectory={1}".format(line, repo.subdirectory) else: line = requirement.as_line(**line_kwargs) - click.echo( - "Writing requirement line to temporary file: {0!r}".format(line), - err=True - ) + if environments.is_verbose(): + click.echo( + "Writing requirement line to temporary file: {0!r}".format(line), + err=True + ) f.write(vistir.misc.to_bytes(line)) r = f.name f.close() @@ -1430,10 +1431,11 @@ def pip_install( ignore_hashes = True if not requirement.hashes else ignore_hashes line = requirement.as_line(include_hashes=not ignore_hashes) line = "{0} {1}".format(line, " ".join(src)) - click.echo( - "Writing requirement line to temporary file: {0!r}".format(line), - err=True - ) + if environments.is_verbose(): + click.echo( + "Writing requirement line to temporary file: {0!r}".format(line), + err=True + ) f.write(vistir.misc.to_bytes(line)) r = f.name f.close() diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py index 8dd369ac..84279bf1 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -248,8 +248,12 @@ class PyPIRepository(BaseRepository): def resolve_reqs(self, download_dir, ireq, wheel_cache): results = None - ireq.isolated = False + ireq.isolated = self.build_isolation ireq._wheel_cache = wheel_cache + if ireq and not ireq.link: + ireq.populate_link(self.finder, False, False) + if ireq.link and not ireq.link.is_wheel: + ireq.ensure_has_source_dir(self.source_dir) try: from pipenv.patched.notpip._internal.operations.prepare import RequirementPreparer except ImportError: @@ -273,20 +277,21 @@ class PyPIRepository(BaseRepository): 'download_dir': download_dir, 'wheel_download_dir': self._wheel_download_dir, 'progress_bar': 'off', - 'build_isolation': False, + 'build_isolation': self.build_isolation, } resolver_kwargs = { 'finder': self.finder, 'session': self.session, 'upgrade_strategy': "to-satisfy-only", - 'force_reinstall': True, + 'force_reinstall': False, 'ignore_dependencies': False, 'ignore_requires_python': True, 'ignore_installed': True, 'ignore_compatibility': False, - 'isolated': False, + 'isolated': True, 'wheel_cache': wheel_cache, - 'use_user_site': False + 'use_user_site': False, + 'use_pep517': True } resolver = None preparer = None diff --git a/pipenv/utils.py b/pipenv/utils.py index 252ed30f..21993514 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -326,15 +326,19 @@ class Resolver(object): @staticmethod @lru_cache() def _get_pip_command(): - from .vendor.pip_shims.shims import Command + from .vendor.pip_shims.shims import Command, cmdoptions class PipCommand(Command): """Needed for pip-tools.""" name = "PipCommand" - from pipenv.patched.piptools.scripts.compile import get_pip_command - return get_pip_command() + from pipenv.patched.piptools.pip import get_pip_command + pip_cmd = get_pip_command() + pip_cmd.parser.add_option(cmdoptions.no_use_pep517()) + pip_cmd.parser.add_option(cmdoptions.use_pep517()) + pip_cmd.parser.add_option(cmdoptions.no_build_isolation()) + return pip_cmd @classmethod def get_metadata( @@ -476,16 +480,29 @@ class Resolver(object): self._pip_command = self._get_pip_command() return self._pip_command - def prepare_pip_args(self): + def prepare_pip_args(self, use_pep517=True, build_isolation=True): pip_args = [] if self.sources: pip_args = prepare_pip_source_args(self.sources, pip_args) + if not use_pep517: + pip_args.append("--no-use-pep517") + if not build_isolation: + pip_args.append("--no-build-isolation") + pip_args.extend(["--cache-dir", environments.PIPENV_CACHE_DIR]) return pip_args @property def pip_args(self): + use_pep517 = False if ( + os.environ.get("PIP_NO_USE_PEP517", None) is not None + ) else (True if os.environ.get("PIP_USE_PEP517", None) is not None else None) + build_isolation = False if ( + os.environ.get("PIP_NO_BUILD_ISOLATION", None) is not None + ) else (True if os.environ.get("PIP_BUILD_ISOLATION", None) is not None else None) if self._pip_args is None: - self._pip_args = self.prepare_pip_args() + self._pip_args = self.prepare_pip_args( + use_pep517=use_pep517, build_isolation=build_isolation + ) return self._pip_args def prepare_constraint_file(self): @@ -497,8 +514,13 @@ class Resolver(object): dir=self.req_dir, delete=False, ) + skip_args = ("build-isolation", "use-pep517", "cache-dir") + args_to_add = [ + arg for arg in self.pip_args + if not any(bad_arg in arg for bad_arg in skip_args) + ] if self.sources: - requirementstxt_sources = " ".join(self.pip_args) if self.pip_args else "" + requirementstxt_sources = " ".join(args_to_add) if args_to_add else "" requirementstxt_sources = requirementstxt_sources.replace(" --", "\n--") constraints_file.write(u"{0}\n".format(requirementstxt_sources)) constraints = self.initial_constraints @@ -535,7 +557,8 @@ class Resolver(object): if self._repository is None: from pipenv.patched.piptools.repositories.pypi import PyPIRepository self._repository = PyPIRepository( - pip_options=self.pip_options, use_json=False, session=self.session + pip_options=self.pip_options, use_json=False, session=self.session, + build_isolation=self.pip_options.build_isolation ) return self._repository @@ -574,22 +597,24 @@ class Resolver(object): from pipenv.patched.piptools.exceptions import NoCandidateFound from pipenv.patched.piptools.cache import CorruptCacheError from .exceptions import CacheError, ResolutionFailure - try: - results = self.resolver.resolve(max_rounds=environments.PIPENV_MAX_ROUNDS) - except CorruptCacheError as e: - if environments.PIPENV_IS_CI or self.clear: - if self._retry_attempts < 3: - self.get_resolver(clear=True, pre=self.pre) - self._retry_attempts += 1 - self.resolve() + with temp_environ(): + os.environ["PIP_NO_USE_PEP517"] = str("") + try: + results = self.resolver.resolve(max_rounds=environments.PIPENV_MAX_ROUNDS) + except CorruptCacheError as e: + if environments.PIPENV_IS_CI or self.clear: + if self._retry_attempts < 3: + self.get_resolver(clear=True, pre=self.pre) + self._retry_attempts += 1 + self.resolve() + else: + raise CacheError(e.path) + except (NoCandidateFound, DistributionNotFound, HTTPError) as e: + raise ResolutionFailure(message=str(e)) else: - raise CacheError(e.path) - except (NoCandidateFound, DistributionNotFound, HTTPError) as e: - raise ResolutionFailure(message=str(e)) - else: - self.results = results - self.resolved_tree.update(results) - return self.resolved_tree + self.results = results + self.resolved_tree.update(results) + return self.resolved_tree @classmethod def prepend_hash_types(cls, checksums): @@ -726,6 +751,10 @@ def actually_resolve_deps( ): from pipenv.vendor.vistir.path import create_tracked_tempdir from pipenv.vendor.requirementslib.models.requirements import Requirement + import pipenv.patched.piptools.logging + + if environments.is_verbose(): + pipenv.patched.piptools.logging.log.verbose = True if not req_dir: req_dir = create_tracked_tempdir(suffix="-requirements", prefix="pipenv-") @@ -857,6 +886,9 @@ def resolve(cmd, sp): click_echo(c.out.strip(), err=True) click_echo(c.err.strip(), err=True) sys.exit(c.return_code) + if environments.is_verbose(): + for ln in c.err.strip(): + sp.hide_and_write(ln) return c @@ -1009,7 +1041,12 @@ def venv_resolve_deps( sp.write(decode_for_output("Resolving dependencies...")) c = resolve(cmd, sp) results = c.out.strip() - sp.green.ok(environments.PIPENV_SPINNER_OK_TEXT.format("Success!")) + if c.ok: + sp.green.ok(environments.PIPENV_SPINNER_OK_TEXT.format("Success!")) + else: + sp.red.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format("Locking Failed!")) + click_echo("Output: {0}".format(c.out.strip()), err=True) + click_echo("Error: {0}".format(c.err.strip()), err=True) try: with open(target_file.name, "r") as fh: results = json.load(fh) diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index 3d3cb99b..ee64d2c6 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -162,7 +162,7 @@ index e54ae08..75b8208 100644 +from packaging.specifiers import SpecifierSet, Specifier + +os.environ["PIP_SHIMS_BASE_MODULE"] = str("pipenv.patched.notpip") -+from pip_shims.shims import VcsSupport, WheelCache, InstallationError ++from pip_shims.shims import VcsSupport, WheelCache, InstallationError, pip_version +from pip_shims.shims import Resolver as PipResolver + + @@ -255,7 +255,7 @@ index e54ae08..75b8208 100644 # pip 19.0 has removed process_dependency_links from the PackageFinder constructor - if pkg_resources.parse_version(pip.__version__) < pkg_resources.parse_version('19.0'): -+ if pkg_resources.parse_version(pip_shims.shims.pip_version) < pkg_resources.parse_version('19.0'): ++ if pkg_resources.parse_version(pip_version) < pkg_resources.parse_version('19.0'): finder_kwargs["process_dependency_links"] = pip_options.process_dependency_links self.finder = PackageFinder(**finder_kwargs) @@ -287,7 +287,7 @@ index e54ae08..75b8208 100644 # Reuses pip's internal candidate sort key to sort matching_candidates = [candidates_by_version[ver] for ver in matching_versions] -@@ -135,14 +187,71 @@ class PyPIRepository(BaseRepository): +@@ -135,14 +187,75 @@ class PyPIRepository(BaseRepository): # Turn the candidate into a pinned InstallRequirement return make_install_requirement( @@ -353,15 +353,19 @@ index e54ae08..75b8208 100644 + def resolve_reqs(self, download_dir, ireq, wheel_cache): results = None -+ ireq.isolated = False ++ ireq.isolated = self.build_isolation + ireq._wheel_cache = wheel_cache ++ if ireq and not ireq.link: ++ ireq.populate_link(self.finder, False, False) ++ if ireq.link and not ireq.link.is_wheel: ++ ireq.ensure_has_source_dir(self.source_dir) try: from pip._internal.operations.prepare import RequirementPreparer - from pip._internal.resolve import Resolver as PipResolver except ImportError: # Pip 9 and below reqset = RequirementSet( -@@ -151,9 +260,11 @@ class PyPIRepository(BaseRepository): +@@ -151,9 +264,11 @@ class PyPIRepository(BaseRepository): download_dir=download_dir, wheel_download_dir=self._wheel_download_dir, session=self.session, @@ -374,27 +378,24 @@ index e54ae08..75b8208 100644 else: # pip >= 10 preparer_kwargs = { -@@ -162,7 +273,7 @@ class PyPIRepository(BaseRepository): - 'download_dir': download_dir, - 'wheel_download_dir': self._wheel_download_dir, - 'progress_bar': 'off', -- 'build_isolation': self.build_isolation, -+ 'build_isolation': False, - } - resolver_kwargs = { - 'finder': self.finder, -@@ -170,8 +281,9 @@ class PyPIRepository(BaseRepository): +@@ -170,11 +285,13 @@ class PyPIRepository(BaseRepository): 'upgrade_strategy': "to-satisfy-only", 'force_reinstall': False, 'ignore_dependencies': False, - 'ignore_requires_python': False, + 'ignore_requires_python': True, 'ignore_installed': True, +- 'isolated': False, + 'ignore_compatibility': False, - 'isolated': False, ++ 'isolated': True, 'wheel_cache': wheel_cache, - 'use_user_site': False -@@ -186,15 +298,22 @@ class PyPIRepository(BaseRepository): +- 'use_user_site': False ++ 'use_user_site': False, ++ 'use_pep517': True + } + resolver = None + preparer = None +@@ -186,15 +303,21 @@ class PyPIRepository(BaseRepository): resolver_kwargs['preparer'] = preparer reqset = RequirementSet() ireq.is_direct = True @@ -412,7 +413,6 @@ index e54ae08..75b8208 100644 + cleanup_fn() + except OSError: + pass -+ + results = set(results) if results else set() + return results, ireq @@ -421,7 +421,7 @@ index e54ae08..75b8208 100644 """ Given a pinned or an editable InstallRequirement, returns a set of dependencies (also InstallRequirements, but not necessarily pinned). -@@ -223,7 +342,8 @@ class PyPIRepository(BaseRepository): +@@ -223,7 +346,8 @@ class PyPIRepository(BaseRepository): wheel_cache = WheelCache(CACHE_DIR, self.pip_options.format_control) prev_tracker = os.environ.get('PIP_REQ_TRACKER') try: @@ -431,7 +431,7 @@ index e54ae08..75b8208 100644 finally: if 'PIP_REQ_TRACKER' in os.environ: if prev_tracker: -@@ -245,6 +365,10 @@ class PyPIRepository(BaseRepository): +@@ -245,6 +369,10 @@ class PyPIRepository(BaseRepository): if ireq.editable: return set() @@ -442,7 +442,7 @@ index e54ae08..75b8208 100644 if not is_pinned_requirement(ireq): raise TypeError( "Expected pinned requirement, got {}".format(ireq)) -@@ -252,24 +376,16 @@ class PyPIRepository(BaseRepository): +@@ -252,24 +380,16 @@ class PyPIRepository(BaseRepository): # We need to get all of the candidates that match our current version # pin, these will represent all of the files that could possibly # satisfy this constraint.