kennethreitz/pipenv
1
0
Fork 0
mirror of https://github.com/kennethreitz/pipenv.git synced 2026-06-05 22:50:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implement new requirement parser

Browse Source 
- Leverage functionality where possible to avoid rework

Signed-off-by: Dan Ryan <dan@danryan.co>
This commit is contained in:
Dan Ryan
2018-04-12 00:16:22 -04:00
parent cf1220ddeb
commit 5bd3e2dcfa
4 changed files with 54 additions and 161 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pipenv/core.py
+11 -21
View File
@@ -27,7 +27,6 @@ from .cmdparse import ScriptEmptyError
from .project import Project, SourceNotFound
from .requirements import PipenvRequirement
from .utils import (
convert_deps_from_pip,
convert_deps_to_pip,
is_required_version,
proper_case,
@@ -36,14 +35,12 @@ from .utils import (
merge_deps,
venv_resolve_deps,
escape_grouped_arguments,
is_vcs,
python_version,
find_windows_executable,
prepare_pip_source_args,
temp_environ,
is_valid_url,
download_file,
get_requirement,
is_pinned,
is_star,
rmtree,
@@ -976,7 +973,7 @@ def get_downloads_info(names_map, section):
p = project.parsed_pipfile
for fname in os.listdir(project.download_location):
# Get name from filename mapping.
name = list(convert_deps_from_pip(names_map[fname]))[0]
name = PipenvRequirement.from_line(names_map[fname]).name
# Get the version info from the filenames.
version = parse_download_fname(fname, name)
# Get the hash of each file.
@@ -1243,14 +1240,12 @@ def do_purge(bare=False, downloads=False, allow_global=False, verbose=False):
actually_installed = []
for package in installed:
try:
dep = convert_deps_from_pip(package)
dep = PipenvRequirement.from_line(package)
except AssertionError:
dep = None
if dep and not is_vcs(dep):
dep = [k for k in dep.keys()][0]
# TODO: make this smarter later.
if not dep.startswith('-e ') and not dep.startswith('git+'):
actually_installed.append(dep)
if dep and not dep.is_vcs and not dep.editable:
dep = dep.name
actually_installed.append(dep)
if not bare:
click.echo(
u'Found {0} installed package(s), purging…'.format(
@@ -1712,7 +1707,8 @@ def do_outdated():
)
results = filter(bool, results)
for result in results:
packages.update(convert_deps_from_pip(result))
dep = PipenvRequirement.from_line(result)
packages.update(dep.as_pipfile())
updated_packages = {}
lockfile = do_lock(write=False)
for section in ('develop', 'default'):
@@ -1936,9 +1932,8 @@ def do_install(
if selective_upgrade:
for i, package_name in enumerate(package_names[:]):
section = project.packages if not dev else project.dev_packages
package = convert_deps_from_pip(package_name)
package__name = list(package.keys())[0]
package__val = list(package.values())[0]
package = PipenvRequirement.from_line(package_name)
package__name, package__val = package.pipfile_entry
try:
if not is_star(section[package__name]) and is_star(
package__val
@@ -1978,17 +1973,12 @@ def do_install(
)
# Warn if --editable wasn't passed.
try:
converted = convert_deps_from_pip(package_name)
converted = PipenvRequirement.from_line(package_name)
except ValueError as e:
click.echo('{0}: {1}'.format(crayons.red('WARNING'), e))
requirements_directory.cleanup()
sys.exit(1)
key = [k for k in converted.keys()][0]
if is_vcs(key) or is_vcs(converted[key]) and not converted[
key
].get(
'editable'
):
if converted.is_vcs and not converted.editable:
click.echo(
'{0}: You installed a VCS dependency in non–editable mode. '
'This will work fine, but sub-dependencies will not be resolved by {1}.'
pipenv/project.py
+8 -12
View File
@@ -20,6 +20,7 @@ except ImportError:
from pathlib2 import Path
from .cmdparse import Script
from .requirements import PipenvRequirement
from .utils import (
atomic_open_for_write,
mkdir_p,
@@ -27,7 +28,6 @@ from .utils import (
proper_case,
find_requirements,
is_editable,
is_file,
is_vcs,
cleanup_toml,
is_installable_file,
@@ -35,6 +35,7 @@ from .utils import (
normalize_drive,
python_version,
safe_expandvars,
is_star,
)
from .environments import (
PIPENV_MAX_DEPTH,
@@ -45,6 +46,7 @@ from .environments import (
PIPENV_PYTHON,
PIPENV_DEFAULT_PYTHON_VERSION,
)
from .vendor.first import first
def _normalized(p):
@@ -727,24 +729,18 @@ class Project(object):
# Read and append Pipfile.
p = self.parsed_pipfile
# Don't re-capitalize file URLs or VCSs.
converted = convert_deps_from_pip(package_name)
converted = converted[first(k for k in converted.keys())]
if not (
is_file(package_name) or is_vcs(converted) or 'path' in converted
):
package_name = pep423_name(package_name)
package = PipenvRequirement.from_line(package_name)
converted = first(package.as_pipfile().values())
key = 'dev-packages' if dev else 'packages'
# Set empty group if it doesn't exist yet.
if key not in p:
p[key] = {}
package = convert_deps_from_pip(package_name)
package_name = first(k for k in package.keys())
name = self.get_package_name_in_pipfile(package_name, dev)
if name and converted == '*':
name = self.get_package_name_in_pipfile(package.name, dev)
if name and is_star(converted):
# Skip for wildcard version
return
# Add the package to the group.
p[key][name or package_name] = package[package_name]
p[key][name or package.normalized_name] = converted
# Write Pipfile.
self.write_toml(p)
pipenv/requirements.py
+28 -22
View File
@@ -2,23 +2,20 @@
from __future__ import absolute_import
import abc
import sys
from pipenv import PIPENV_VENDOR, PIPENV_PATCHED
sys.path.insert(0, PIPENV_VENDOR)
sys.path.insert(0, PIPENV_PATCHED)
import hashlib
import os
import requirements
import six
from attr import attrs, attrib, Factory, validators
from .vendor.attr import attrs, attrib, Factory, validators
from .vendor import attr
from pip9.index import Link
from pip9.download import path_to_url
from pip9.req.req_install import _strip_extras
from pip9._vendor.distlib.markers import Evaluator
from pip9._vendor.packaging.markers import Marker, InvalidMarker
from pip9._vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier
from pipenv.utils import SCHEME_LIST, VCS_LIST, is_installable_file, is_vcs, multi_split, get_converted_relative_path, is_star, is_valid_url
from first import first
from .utils import SCHEME_LIST, VCS_LIST, is_installable_file, is_vcs, multi_split, get_converted_relative_path, is_star, is_valid_url, pep423_name
from .vendor.first import first
try:
from pathlib import Path
@@ -366,19 +363,6 @@ class FileRequirement(BaseRequirement):
}
return cls(**arg_dict)
@req.default
def get_requirement(self):
base = '{0}'.format(self.link)
req = first(requirements.parse(base))
if self.editable:
req.editable = True
if self.link and self.link.scheme.startswith('file') and self.path:
req.path = self.path
req.local_file = True
req.uri = None
req.link = self.link
return req
@property
def line_part(self):
seed = self.path or self.link.url or self.uri
@@ -414,8 +398,6 @@ class VCSRequirement(FileRequirement):
vcs = attrib(validator=validators.optional(_validate_vcs), default=None)
# : vcs reference name (branch / commit / tag)
ref = attrib(default=None)
#: path to hit - without any of the VCS prefixes (like git+ / http+ / etc)
uri = attrib(default=None)
subdirectory = attrib(default=None)
name = attrib()
link = attrib()
@@ -488,6 +470,7 @@ class VCSRequirement(FileRequirement):
@classmethod
def from_line(cls, line, editable=None):
path = None
if line.startswith('-e '):
editable = True
line = line.split(' ', 1)[1]
@@ -599,6 +582,24 @@ class PipenvRequirement(object):
return _specs_to_string(self.req.req.specs)
return
@property
def is_vcs(self):
return isinstance(self, VCSRequirement)
@property
def is_file_or_url(self):
return isinstance(self, FileRequirement)
@property
def is_named(self):
return isinstance(self, NamedRequirement)
@property
def normalized_name(self):
if not self.is_vcs and not self.is_file_or_url:
return pep423_name(self.name)
return self.name
@classmethod
def from_line(cls, line):
hashes = None
@@ -616,6 +617,7 @@ class PipenvRequirement(object):
r = FileRequirement.from_line(line)
elif is_vcs(stripped_line):
r = VCSRequirement.from_line(line)
vcs = r.vcs
else:
name = multi_split(stripped_line, '!=<>~')[0]
if not extras:
@@ -703,6 +705,10 @@ class PipenvRequirement(object):
base_dict = base_dict.get('version')
return {name: base_dict}
@property
def pipfile_entry(self):
return self.as_pipfile().copy().popitem()
def _extras_to_string(extras):
"""Turn a list of extras into a string"""
pipenv/utils.py
+7 -106
View File
@@ -82,97 +82,6 @@ def _get_requests_session():
return requests_session
def get_requirement(dep):
from .patched.notpip._internal.req.req_install import _strip_extras, Wheel
from .patched.notpip._internal.index import Link
from .vendor import requirements
"""Pre-clean requirement strings passed to the requirements parser.
Ensures that we can accept both local and relative paths, file and VCS URIs,
remote URIs, and package names, and that we pass only valid requirement strings
to the requirements parser. Performs necessary modifications to requirements
object if the user input was a local relative path.
:param str dep: A requirement line
:returns: :class:`requirements.Requirement` object
"""
path = None
uri = None
cleaned_uri = None
editable = False
dep_link = None
# check for editable dep / vcs dep
if dep.startswith('-e '):
editable = True
# Use the user supplied path as the written dependency
dep = dep.split(' ', 1)[1]
# Split out markers if they are present - similar to how pip does it
# See notpip.req.req_install.InstallRequirement.from_line
if not any(dep.startswith(uri_prefix) for uri_prefix in SCHEME_LIST):
marker_sep = ';'
else:
marker_sep = '; '
if marker_sep in dep:
dep, markers = dep.split(marker_sep, 1)
markers = markers.strip()
if not markers:
markers = None
else:
markers = None
# Strip extras from the requirement so we can make a properly parseable req
dep, extras = _strip_extras(dep)
# Only operate on local, existing, non-URI formatted paths which are installable
if is_installable_file(dep):
dep_path = Path(dep)
dep_link = Link(dep_path.absolute().as_uri())
if dep_path.is_absolute() or dep_path.as_posix() == '.':
path = dep_path.as_posix()
else:
path = get_converted_relative_path(dep)
dep = dep_link.egg_fragment if dep_link.egg_fragment else dep_link.url_without_fragment
elif is_vcs(dep):
# Generate a Link object for parsing egg fragments
dep_link = Link(dep)
# Save the original path to store in the pipfile
uri = dep_link.url
# Construct the requirement using proper git+ssh:// replaced uris or names if available
cleaned_uri = clean_git_uri(dep)
dep = cleaned_uri
if editable:
dep = '-e {0}'.format(dep)
req = [r for r in requirements.parse(dep)][0]
# if all we built was the requirement name and still need everything else
if req.name and not any([req.uri, req.path]):
if dep_link:
if dep_link.scheme.startswith('file') and path and not req.path:
req.path = path
req.local_file = True
req.uri = None
else:
req.uri = dep_link.url_without_fragment
# If the result is a local file with a URI and we have a local path, unset the URI
# and set the path instead -- note that local files may have 'path' set by accident
elif req.local_file and path and not req.vcs:
req.path = path
req.uri = None
if dep_link and dep_link.is_wheel and not req.name:
req.name = os.path.basename(Wheel(dep_link.path).name)
elif req.vcs and req.uri and cleaned_uri and cleaned_uri != uri:
req.uri = strip_ssh_from_git_uri(req.uri)
req.line = strip_ssh_from_git_uri(req.line)
req.editable = editable
if markers:
req.markers = markers
if extras:
# Bizarrely this is also what pip does...
req.extras = [
r for r in requirements.parse('fakepkg{0}'.format(extras))
][
0
].extras
return req
def cleanup_toml(tml):
toml = tml.split('\n')
new_toml = []
@@ -582,12 +491,6 @@ def multi_split(s, split):
return [i for i in s.split('|') if len(i) > 0]
def convert_deps_from_pip(dep):
""""Converts a pip-formatted dependency to a Pipfile-formatted one."""
req = PipenvRequirement.from_line(dep)
return req.as_pipfile()
def is_star(val):
return isinstance(val, six.string_types) and val == '*'
@@ -601,6 +504,7 @@ def is_pinned(val):
def convert_deps_to_pip(deps, project=None, r=True, include_index=False):
""""Converts a Pipfile-formatted dependency to a pip-formatted one."""
from ._compat import NamedTemporaryFile
from .requirements import PipenvRequirement
dependencies = []
for dep_name, dep in deps.items():
indexes = project.sources if hasattr(project, 'sources') else None
@@ -1265,19 +1169,16 @@ def get_vcs_deps(project, pip_freeze=None, which=None, verbose=False, clear=Fals
backend = vcs_registry._registry[first(b for b in vcs_registry if b == backend_name)]
__vcs = backend(url=_pip_uri)
installed = convert_deps_from_pip(line)
if not hasattr(installed, 'keys'):
pass
lock_name = first(installed.keys())
names.add(lock_name)
installed = PipenvRequirement.from_line(line)
names.add(installed.normalized_name)
locked_rev = None
for _name in names:
locked_rev = install_or_update_vcs(__vcs, src_dir, _name, rev=pipfile_rev)
if is_vcs(installed[lock_name]):
installed[lock_name]['ref'] = locked_rev
lockfiles.append({pipfile_name: installed[lock_name]})
if installed.is_vcs:
installed.req.ref = locked_rev
lockfiles.append({pipfile_name: installed.pipfile_entry[1]})
pipfile_srcdir = os.path.join(src_dir, pipfile_name)
lockfile_srcdir = os.path.join(src_dir, lock_name)
lockfile_srcdir = os.path.join(src_dir, installed.normalized_name)
lines.append(line)
if os.path.exists(pipfile_srcdir):
lockfiles.extend(venv_resolve_deps(['-e {0}'.format(pipfile_srcdir)], which=which, verbose=verbose, project=project, clear=clear, pre=pre, allow_global=allow_global))