From 5bd3e2dcfa6d26480ed2cd0fa8ad1bbb959d92af Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Thu, 12 Apr 2018 00:16:22 -0400 Subject: [PATCH] Implement new requirement parser - Leverage functionality where possible to avoid rework Signed-off-by: Dan Ryan --- pipenv/core.py | 32 ++++-------- pipenv/project.py | 20 +++----- pipenv/requirements.py | 50 ++++++++++-------- pipenv/utils.py | 113 +++-------------------------------------- 4 files changed, 54 insertions(+), 161 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 5c5783f8..d295708c 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -27,7 +27,6 @@ from .cmdparse import ScriptEmptyError from .project import Project, SourceNotFound from .requirements import PipenvRequirement from .utils import ( - convert_deps_from_pip, convert_deps_to_pip, is_required_version, proper_case, @@ -36,14 +35,12 @@ from .utils import ( merge_deps, venv_resolve_deps, escape_grouped_arguments, - is_vcs, python_version, find_windows_executable, prepare_pip_source_args, temp_environ, is_valid_url, download_file, - get_requirement, is_pinned, is_star, rmtree, @@ -976,7 +973,7 @@ def get_downloads_info(names_map, section): p = project.parsed_pipfile for fname in os.listdir(project.download_location): # Get name from filename mapping. - name = list(convert_deps_from_pip(names_map[fname]))[0] + name = PipenvRequirement.from_line(names_map[fname]).name # Get the version info from the filenames. version = parse_download_fname(fname, name) # Get the hash of each file. @@ -1243,14 +1240,12 @@ def do_purge(bare=False, downloads=False, allow_global=False, verbose=False): actually_installed = [] for package in installed: try: - dep = convert_deps_from_pip(package) + dep = PipenvRequirement.from_line(package) except AssertionError: dep = None - if dep and not is_vcs(dep): - dep = [k for k in dep.keys()][0] - # TODO: make this smarter later. - if not dep.startswith('-e ') and not dep.startswith('git+'): - actually_installed.append(dep) + if dep and not dep.is_vcs and not dep.editable: + dep = dep.name + actually_installed.append(dep) if not bare: click.echo( u'Found {0} installed package(s), purging…'.format( @@ -1712,7 +1707,8 @@ def do_outdated(): ) results = filter(bool, results) for result in results: - packages.update(convert_deps_from_pip(result)) + dep = PipenvRequirement.from_line(result) + packages.update(dep.as_pipfile()) updated_packages = {} lockfile = do_lock(write=False) for section in ('develop', 'default'): @@ -1936,9 +1932,8 @@ def do_install( if selective_upgrade: for i, package_name in enumerate(package_names[:]): section = project.packages if not dev else project.dev_packages - package = convert_deps_from_pip(package_name) - package__name = list(package.keys())[0] - package__val = list(package.values())[0] + package = PipenvRequirement.from_line(package_name) + package__name, package__val = package.pipfile_entry try: if not is_star(section[package__name]) and is_star( package__val @@ -1978,17 +1973,12 @@ def do_install( ) # Warn if --editable wasn't passed. try: - converted = convert_deps_from_pip(package_name) + converted = PipenvRequirement.from_line(package_name) except ValueError as e: click.echo('{0}: {1}'.format(crayons.red('WARNING'), e)) requirements_directory.cleanup() sys.exit(1) - key = [k for k in converted.keys()][0] - if is_vcs(key) or is_vcs(converted[key]) and not converted[ - key - ].get( - 'editable' - ): + if converted.is_vcs and not converted.editable: click.echo( '{0}: You installed a VCS dependency in non–editable mode. ' 'This will work fine, but sub-dependencies will not be resolved by {1}.' diff --git a/pipenv/project.py b/pipenv/project.py index 43ac92d0..f8fcfe17 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -20,6 +20,7 @@ except ImportError: from pathlib2 import Path from .cmdparse import Script +from .requirements import PipenvRequirement from .utils import ( atomic_open_for_write, mkdir_p, @@ -27,7 +28,6 @@ from .utils import ( proper_case, find_requirements, is_editable, - is_file, is_vcs, cleanup_toml, is_installable_file, @@ -35,6 +35,7 @@ from .utils import ( normalize_drive, python_version, safe_expandvars, + is_star, ) from .environments import ( PIPENV_MAX_DEPTH, @@ -45,6 +46,7 @@ from .environments import ( PIPENV_PYTHON, PIPENV_DEFAULT_PYTHON_VERSION, ) +from .vendor.first import first def _normalized(p): @@ -727,24 +729,18 @@ class Project(object): # Read and append Pipfile. p = self.parsed_pipfile # Don't re-capitalize file URLs or VCSs. - converted = convert_deps_from_pip(package_name) - converted = converted[first(k for k in converted.keys())] - if not ( - is_file(package_name) or is_vcs(converted) or 'path' in converted - ): - package_name = pep423_name(package_name) + package = PipenvRequirement.from_line(package_name) + converted = first(package.as_pipfile().values()) key = 'dev-packages' if dev else 'packages' # Set empty group if it doesn't exist yet. if key not in p: p[key] = {} - package = convert_deps_from_pip(package_name) - package_name = first(k for k in package.keys()) - name = self.get_package_name_in_pipfile(package_name, dev) - if name and converted == '*': + name = self.get_package_name_in_pipfile(package.name, dev) + if name and is_star(converted): # Skip for wildcard version return # Add the package to the group. - p[key][name or package_name] = package[package_name] + p[key][name or package.normalized_name] = converted # Write Pipfile. self.write_toml(p) diff --git a/pipenv/requirements.py b/pipenv/requirements.py index 3120b1dd..616b7af4 100644 --- a/pipenv/requirements.py +++ b/pipenv/requirements.py @@ -2,23 +2,20 @@ from __future__ import absolute_import import abc import sys -from pipenv import PIPENV_VENDOR, PIPENV_PATCHED - -sys.path.insert(0, PIPENV_VENDOR) -sys.path.insert(0, PIPENV_PATCHED) import hashlib import os import requirements import six -from attr import attrs, attrib, Factory, validators +from .vendor.attr import attrs, attrib, Factory, validators +from .vendor import attr from pip9.index import Link from pip9.download import path_to_url from pip9.req.req_install import _strip_extras from pip9._vendor.distlib.markers import Evaluator from pip9._vendor.packaging.markers import Marker, InvalidMarker from pip9._vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier -from pipenv.utils import SCHEME_LIST, VCS_LIST, is_installable_file, is_vcs, multi_split, get_converted_relative_path, is_star, is_valid_url -from first import first +from .utils import SCHEME_LIST, VCS_LIST, is_installable_file, is_vcs, multi_split, get_converted_relative_path, is_star, is_valid_url, pep423_name +from .vendor.first import first try: from pathlib import Path @@ -366,19 +363,6 @@ class FileRequirement(BaseRequirement): } return cls(**arg_dict) - @req.default - def get_requirement(self): - base = '{0}'.format(self.link) - req = first(requirements.parse(base)) - if self.editable: - req.editable = True - if self.link and self.link.scheme.startswith('file') and self.path: - req.path = self.path - req.local_file = True - req.uri = None - req.link = self.link - return req - @property def line_part(self): seed = self.path or self.link.url or self.uri @@ -414,8 +398,6 @@ class VCSRequirement(FileRequirement): vcs = attrib(validator=validators.optional(_validate_vcs), default=None) # : vcs reference name (branch / commit / tag) ref = attrib(default=None) - #: path to hit - without any of the VCS prefixes (like git+ / http+ / etc) - uri = attrib(default=None) subdirectory = attrib(default=None) name = attrib() link = attrib() @@ -488,6 +470,7 @@ class VCSRequirement(FileRequirement): @classmethod def from_line(cls, line, editable=None): + path = None if line.startswith('-e '): editable = True line = line.split(' ', 1)[1] @@ -599,6 +582,24 @@ class PipenvRequirement(object): return _specs_to_string(self.req.req.specs) return + @property + def is_vcs(self): + return isinstance(self, VCSRequirement) + + @property + def is_file_or_url(self): + return isinstance(self, FileRequirement) + + @property + def is_named(self): + return isinstance(self, NamedRequirement) + + @property + def normalized_name(self): + if not self.is_vcs and not self.is_file_or_url: + return pep423_name(self.name) + return self.name + @classmethod def from_line(cls, line): hashes = None @@ -616,6 +617,7 @@ class PipenvRequirement(object): r = FileRequirement.from_line(line) elif is_vcs(stripped_line): r = VCSRequirement.from_line(line) + vcs = r.vcs else: name = multi_split(stripped_line, '!=<>~')[0] if not extras: @@ -703,6 +705,10 @@ class PipenvRequirement(object): base_dict = base_dict.get('version') return {name: base_dict} + @property + def pipfile_entry(self): + return self.as_pipfile().copy().popitem() + def _extras_to_string(extras): """Turn a list of extras into a string""" diff --git a/pipenv/utils.py b/pipenv/utils.py index 614b333c..f8cf9192 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -82,97 +82,6 @@ def _get_requests_session(): return requests_session -def get_requirement(dep): - from .patched.notpip._internal.req.req_install import _strip_extras, Wheel - from .patched.notpip._internal.index import Link - from .vendor import requirements - """Pre-clean requirement strings passed to the requirements parser. - - Ensures that we can accept both local and relative paths, file and VCS URIs, - remote URIs, and package names, and that we pass only valid requirement strings - to the requirements parser. Performs necessary modifications to requirements - object if the user input was a local relative path. - - :param str dep: A requirement line - :returns: :class:`requirements.Requirement` object - """ - path = None - uri = None - cleaned_uri = None - editable = False - dep_link = None - # check for editable dep / vcs dep - if dep.startswith('-e '): - editable = True - # Use the user supplied path as the written dependency - dep = dep.split(' ', 1)[1] - # Split out markers if they are present - similar to how pip does it - # See notpip.req.req_install.InstallRequirement.from_line - if not any(dep.startswith(uri_prefix) for uri_prefix in SCHEME_LIST): - marker_sep = ';' - else: - marker_sep = '; ' - if marker_sep in dep: - dep, markers = dep.split(marker_sep, 1) - markers = markers.strip() - if not markers: - markers = None - else: - markers = None - # Strip extras from the requirement so we can make a properly parseable req - dep, extras = _strip_extras(dep) - # Only operate on local, existing, non-URI formatted paths which are installable - if is_installable_file(dep): - dep_path = Path(dep) - dep_link = Link(dep_path.absolute().as_uri()) - if dep_path.is_absolute() or dep_path.as_posix() == '.': - path = dep_path.as_posix() - else: - path = get_converted_relative_path(dep) - dep = dep_link.egg_fragment if dep_link.egg_fragment else dep_link.url_without_fragment - elif is_vcs(dep): - # Generate a Link object for parsing egg fragments - dep_link = Link(dep) - # Save the original path to store in the pipfile - uri = dep_link.url - # Construct the requirement using proper git+ssh:// replaced uris or names if available - cleaned_uri = clean_git_uri(dep) - dep = cleaned_uri - if editable: - dep = '-e {0}'.format(dep) - req = [r for r in requirements.parse(dep)][0] - # if all we built was the requirement name and still need everything else - if req.name and not any([req.uri, req.path]): - if dep_link: - if dep_link.scheme.startswith('file') and path and not req.path: - req.path = path - req.local_file = True - req.uri = None - else: - req.uri = dep_link.url_without_fragment - # If the result is a local file with a URI and we have a local path, unset the URI - # and set the path instead -- note that local files may have 'path' set by accident - elif req.local_file and path and not req.vcs: - req.path = path - req.uri = None - if dep_link and dep_link.is_wheel and not req.name: - req.name = os.path.basename(Wheel(dep_link.path).name) - elif req.vcs and req.uri and cleaned_uri and cleaned_uri != uri: - req.uri = strip_ssh_from_git_uri(req.uri) - req.line = strip_ssh_from_git_uri(req.line) - req.editable = editable - if markers: - req.markers = markers - if extras: - # Bizarrely this is also what pip does... - req.extras = [ - r for r in requirements.parse('fakepkg{0}'.format(extras)) - ][ - 0 - ].extras - return req - - def cleanup_toml(tml): toml = tml.split('\n') new_toml = [] @@ -582,12 +491,6 @@ def multi_split(s, split): return [i for i in s.split('|') if len(i) > 0] -def convert_deps_from_pip(dep): - """"Converts a pip-formatted dependency to a Pipfile-formatted one.""" - req = PipenvRequirement.from_line(dep) - return req.as_pipfile() - - def is_star(val): return isinstance(val, six.string_types) and val == '*' @@ -601,6 +504,7 @@ def is_pinned(val): def convert_deps_to_pip(deps, project=None, r=True, include_index=False): """"Converts a Pipfile-formatted dependency to a pip-formatted one.""" from ._compat import NamedTemporaryFile + from .requirements import PipenvRequirement dependencies = [] for dep_name, dep in deps.items(): indexes = project.sources if hasattr(project, 'sources') else None @@ -1265,19 +1169,16 @@ def get_vcs_deps(project, pip_freeze=None, which=None, verbose=False, clear=Fals backend = vcs_registry._registry[first(b for b in vcs_registry if b == backend_name)] __vcs = backend(url=_pip_uri) - installed = convert_deps_from_pip(line) - if not hasattr(installed, 'keys'): - pass - lock_name = first(installed.keys()) - names.add(lock_name) + installed = PipenvRequirement.from_line(line) + names.add(installed.normalized_name) locked_rev = None for _name in names: locked_rev = install_or_update_vcs(__vcs, src_dir, _name, rev=pipfile_rev) - if is_vcs(installed[lock_name]): - installed[lock_name]['ref'] = locked_rev - lockfiles.append({pipfile_name: installed[lock_name]}) + if installed.is_vcs: + installed.req.ref = locked_rev + lockfiles.append({pipfile_name: installed.pipfile_entry[1]}) pipfile_srcdir = os.path.join(src_dir, pipfile_name) - lockfile_srcdir = os.path.join(src_dir, lock_name) + lockfile_srcdir = os.path.join(src_dir, installed.normalized_name) lines.append(line) if os.path.exists(pipfile_srcdir): lockfiles.extend(venv_resolve_deps(['-e {0}'.format(pipfile_srcdir)], which=which, verbose=verbose, project=project, clear=clear, pre=pre, allow_global=allow_global))