From 18530dfb46bbe6aef9dc23ce8774ddca6990db65 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 10 Jun 2018 14:19:36 -0400 Subject: [PATCH] Fix VCS resolution - Iterate over VCS dependencies from pipfile instead of iterating over those found in `pip freeze` output - Only pass editable dependencies to pip-tools for resolution - Normalize names and ensure that we update lockfile entries accordingly Signed-off-by: Dan Ryan --- pipenv/core.py | 12 +++++++---- pipenv/utils.py | 57 +++++++++++++++++++------------------------------ 2 files changed, 30 insertions(+), 39 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 36c745ca..cdc1ff75 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1090,7 +1090,7 @@ def do_lock( lockfile[settings['lockfile_key']].update(dep_lockfile) # Add refs for VCS installs. # TODO: be smarter about this. - vcs_lines, vcs_lockfile = get_vcs_deps( + vcs_reqs, vcs_lockfile = get_vcs_deps( project, pip_freeze, which=which, @@ -1100,6 +1100,7 @@ def do_lock( allow_global=system, dev=settings['dev'] ) + vcs_lines = [req.as_line() for req in vcs_reqs if req.editable] vcs_results = venv_resolve_deps( vcs_lines, which=which, @@ -1111,13 +1112,16 @@ def do_lock( pypi_mirror=pypi_mirror, ) for dep in vcs_results: + normalized = pep423_name(dep['name']) if not hasattr(dep, 'keys') or not hasattr(dep['name'], 'keys'): continue - is_top_level = dep['name'] in vcs_lockfile - pipfile_entry = vcs_lockfile[dep['name']] if is_top_level else None - dep_lockfile = clean_resolved_dep(dep, is_top_level=is_top_level, pipfile_entry=pipfile_entry) + is_top_level = dep['name'] in vcs_lockfile or normalized in vcs_lockfile + lockfile_key = next(k for k in [dep['name'], normalized] if k in vcs_lockfile) + lockfile_entry = vcs_lockfile[lockfile_key] if is_top_level else None + dep_lockfile = clean_resolved_dep(dep, is_top_level=is_top_level, pipfile_entry=lockfile_entry) vcs_lockfile.update(dep_lockfile) lockfile[settings['lockfile_key']].update(vcs_lockfile) + # Support for --keep-outdated… if keep_outdated: for section_name, section in ( diff --git a/pipenv/utils.py b/pipenv/utils.py index 9159ffe1..3eb26158 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -1135,7 +1135,11 @@ def extract_uri_from_vcs_dep(dep): return None -def install_or_update_vcs(vcs_obj, src_dir, name, rev=None): +def resolve_ref(vcs_obj, target_dir, ref): + return vcs_obj.get_revision_sha(target_dir, ref) + + +def obtain_vcs_req(vcs_obj, src_dir, name, rev=None): target_dir = os.path.join(src_dir, name) target_rev = vcs_obj.make_rev_options(rev) if not os.path.exists(target_dir): @@ -1160,7 +1164,7 @@ def get_vcs_deps( from ._compat import TemporaryDirectory section = "vcs_dev_packages" if dev else "vcs_packages" - lines = [] + reqs = [] lockfile = {} try: packages = getattr(project, section) @@ -1175,45 +1179,27 @@ def get_vcs_deps( ) src_dir.mkdir(mode=0o775, exist_ok=True) vcs_registry = VcsSupport - vcs_uri_map = { - extract_uri_from_vcs_dep(v): {"name": k, "ref": v.get("ref")} - for k, v in packages.items() - } - for line in pip_freeze.strip().split("\n"): - # if the line doesn't match a vcs dependency in the Pipfile, - # ignore it - _vcs_match = first(_uri for _uri in vcs_uri_map.keys() if _uri in line) - if not _vcs_match: - continue - - pipfile_name = vcs_uri_map[_vcs_match]["name"] - pipfile_rev = vcs_uri_map[_vcs_match]["ref"] - pipfile_req = Requirement.from_pipfile(pipfile_name, packages[pipfile_name]) - names = {pipfile_name} - backend = vcs_registry()._registry.get(pipfile_req.vcs) - # TODO: Why doesn't pip freeze list 'git+git://' formatted urls? - if line.startswith("-e ") and not "{0}+".format(pipfile_req.vcs) in line: - line = line.replace("-e ", "-e {0}+".format(pipfile_req.vcs)) - installed = Requirement.from_line(line) - __vcs = backend(url=installed.req.uri) - - names.add(installed.normalized_name) + vcs_registry = VcsSupport + for pkg_name, pkg_pipfile in packages.items(): + requirement = Requirement.from_pipfile(pkg_name, pkg_pipfile) + backend = vcs_registry()._registry.get(requirement.vcs) + __vcs = backend(url=requirement.req.vcs_uri) locked_rev = None - for _name in names: - locked_rev = install_or_update_vcs( - __vcs, src_dir.as_posix(), _name, rev=pipfile_rev - ) - if installed.is_vcs: - installed.req.ref = locked_rev - lockfile[pipfile_name] = installed.pipfile_entry[1] - lines.append(line) - return lines, lockfile + name = requirement.normalized_name + locked_rev = obtain_vcs_req( + __vcs, src_dir.as_posix(), name, rev=pkg_pipfile.get("ref") + ) + if requirement.is_vcs: + requirement.req.ref = locked_rev + lockfile[name] = requirement.pipfile_entry[1] + reqs.append(requirement) + return reqs, lockfile def clean_resolved_dep(dep, is_top_level=False, pipfile_entry=None): from notpip._vendor.distlib.markers import DEFAULT_CONTEXT as marker_context allowed_marker_keys = ['markers'] + [k for k in marker_context.keys()] - name = dep['name'] + name = pep423_name(dep['name']) # We use this to determine if there are any markers on top level packages # So we can make sure those win out during resolution if the packages reoccur dep_keys = [k for k in getattr(pipfile_entry, 'keys', list)()] if is_top_level else [] @@ -1224,6 +1210,7 @@ def clean_resolved_dep(dep, is_top_level=False, pipfile_entry=None): if key in dep: lockfile[key] = dep[key] # In case we lock a uri or a file when the user supplied a path + # remove the uri or file keys from the entry and keep the path if pipfile_entry and any(k in pipfile_entry for k in ['file', 'path']): fs_key = next((k for k in ['path', 'file'] if k in pipfile_entry), None) lockfile_key = next((k for k in ['uri', 'file', 'path'] if k in lockfile), None)