diff --git a/pipenv/core.py b/pipenv/core.py index 47294258..e56f53b7 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1,5 +1,5 @@ # -*- coding=utf-8 -*- - +from __future__ import absolute_import, print_function import json as simplejson import logging import os @@ -1288,7 +1288,7 @@ def pip_install( piplogger.setLevel(logging.INFO) if requirement: click.echo( - crayons.normal("Installing {0!r}".format(requirement.name), bold=True), + crayons.normal("Pip Installing {0!r}".format(requirement.name), bold=True), err=True, ) # Create files for hash mode. @@ -1303,9 +1303,11 @@ def pip_install( f.write(vistir.misc.to_bytes(requirement.as_line())) r = f.name f.close() - # Install dependencies when a package is a VCS dependency. + if requirement and requirement.vcs: - no_deps = False + # Install dependencies when a package is a non-editable VCS dependency. + if not requirement.editable: + no_deps = False # Don't specify a source directory when using --system. if not allow_global and ("PIP_SRC" not in os.environ): src.extend(["--src", "{0}".format(project.virtualenv_src_location)]) @@ -1361,9 +1363,9 @@ def pip_install( if "PIP_SRC" in os.environ: src_dir = os.environ["PIP_SRC"] src = ["--src", os.environ["PIP_SRC"]] - else: - src_dir = "{0}".format(project.virtualenv_src_location) - os.environ["PIP_SRC"] = project.virtualenv_src_location + # else: + # src_dir = "{0}".format(project.virtualenv_src_location) + # os.environ["PIP_SRC"] = project.virtualenv_src_location if not requirement.editable: no_deps = False # if not requirement.req.is_local: @@ -1451,8 +1453,8 @@ def pip_install( pip_command.extend(prepare_pip_source_args(sources)) if not ignore_hashes: pip_command.append("--require-hashes") - pip_command.append("--no-build-isolation") if not use_pep517: + pip_command.append("--no-build-isolation") pip_command.append("--no-use-pep517") if environments.is_verbose(): click.echo("$ {0}".format(pip_command), err=True) @@ -1932,7 +1934,7 @@ def do_install( # This is for if the user passed in dependencies, then we want to make sure we else: - from .vendor.requirementslib import Requirement + from .vendor.requirementslib.models.requirements import Requirement # make a tuple of (display_name, entry) pkg_list = packages + ["-e {0}".format(pkg) for pkg in editable_packages] diff --git a/pipenv/project.py b/pipenv/project.py index edbcff87..a2afd6b2 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -681,7 +681,7 @@ class Project(object): ) name = self.name if self.name is not None else "Pipfile" - config_parser = ConfigOptionParser(name=self.name) + config_parser = ConfigOptionParser(name=name) config_parser.add_option_group(make_option_group(index_group, config_parser)) install = config_parser.option_groups[0] indexes = ( diff --git a/pipenv/resolver.py b/pipenv/resolver.py index ed474fb4..c50e7ecb 100644 --- a/pipenv/resolver.py +++ b/pipenv/resolver.py @@ -7,12 +7,48 @@ import sys os.environ["PIP_PYTHON_PATH"] = str(sys.executable) -def _patch_path(): +def find_site_path(pkg, site_dir=None): + import pkg_resources + if site_dir is not None: + site_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + working_set = pkg_resources.WorkingSet([site_dir] + sys.path[:]) + for dist in working_set: + root = dist.location + base_name = dist.project_name if dist.project_name else dist.key + name = None + if "top_level.txt" in dist.metadata_listdir(""): + name = next(iter([l.strip() for l in dist.get_metadata_lines("top_level.txt") if l is not None]), None) + if name is None: + name = pkg_resources.safe_name(base_name).replace("-", "_") + if not any(pkg == _ for _ in [base_name, name]): + continue + path_options = [name, "{0}.py".format(name)] + path_options = [os.path.join(root, p) for p in path_options if p is not None] + path = next(iter(p for p in path_options if os.path.exists(p)), None) + if path is not None: + return (dist, path) + return (None, None) + + +def _patch_path(pipenv_site=None): import site pipenv_libdir = os.path.dirname(os.path.abspath(__file__)) pipenv_site_dir = os.path.dirname(pipenv_libdir) - site.addsitedir(pipenv_site_dir) - for _dir in ("vendor", "patched"): + pipenv_dist = None + if pipenv_site is not None: + pipenv_dist, pipenv_path = find_site_path("pipenv", site_dir=pipenv_site) + else: + pipenv_dist, pipenv_path = find_site_path("pipenv", site_dir=pipenv_site_dir) + if pipenv_dist is not None: + pipenv_dist.activate() + else: + site.addsitedir(next(iter( + sitedir for sitedir in (pipenv_site, pipenv_site_dir) + if sitedir is not None + ), None)) + if pipenv_path is not None: + pipenv_libdir = pipenv_path + for _dir in ("vendor", "patched", pipenv_libdir): sys.path.insert(0, os.path.join(pipenv_libdir, _dir)) @@ -24,8 +60,11 @@ def get_parser(): parser.add_argument("--verbose", "-v", action="count", default=False) parser.add_argument("--debug", action="store_true", default=False) parser.add_argument("--system", action="store_true", default=False) + parser.add_argument("--parse-only", action="store_true", default=False) + parser.add_argument("--pipenv-site", metavar="pipenv_site_dir", action="store", + default=os.environ.get("PIPENV_SITE_DIR")) parser.add_argument("--requirements-dir", metavar="requirements_dir", action="store", - default=os.environ.get("PIPENV_REQ_DIR")) + default=os.environ.get("PIPENV_REQ_DIR")) parser.add_argument("packages", nargs="*") return parser @@ -41,23 +80,56 @@ def handle_parsed_args(parsed): logging.getLogger("notpip").setLevel(logging.DEBUG) elif parsed.verbose > 0: logging.getLogger("notpip").setLevel(logging.INFO) + os.environ["PIPENV_VERBOSITY"] = str(parsed.verbose) if "PIPENV_PACKAGES" in os.environ: parsed.packages += os.environ.get("PIPENV_PACKAGES", "").strip().split("\n") return parsed -def _main(pre, clear, verbose, system, requirements_dir, packages): - os.environ["PIP_PYTHON_VERSION"] = ".".join([str(s) for s in sys.version_info[:3]]) - os.environ["PIP_PYTHON_PATH"] = str(sys.executable) +def parse_packages(packages, pre, clear, system, requirements_dir=None): + from pipenv.vendor.requirementslib.models.requirements import Requirement + from pipenv.vendor.vistir.contextmanagers import cd, temp_path + from pipenv.utils import parse_indexes + parsed_packages = [] + os.environ["PIP_NO_BUILD_ISOLATION"] = "1" + os.environ["PIP_NO_USE_PEP517"] = "1" + os.environ["PIP_NO_DEPS"] = "1" + for package in packages: + indexes, trusted_hosts, line = parse_indexes(package) + line = " ".join(line) + pf = dict() + req = Requirement.from_line(line) + if not req.name: + with temp_path(), cd(req.req.setup_info.base_dir): + sys.path.insert(0, req.req.setup_info.base_dir) + req.req._setup_info.get_info() + req.update_name_from_path(req.req.setup_info.base_dir) + print(os.listdir(req.req.setup_info.base_dir)) + try: + name, entry = req.pipfile_entry + except Exception: + continue + else: + if name is not None and entry is not None: + pf[name] = entry + parsed_packages.append(pf) + print("RESULTS:") + if parsed_packages: + print(json.dumps(parsed_packages)) + else: + print(json.dumps([])) + +def resolve_packages(pre, clear, verbose, system, requirements_dir, packages): from pipenv.utils import create_mirror_source, resolve_deps, replace_pypi_sources - pypi_mirror_source = ( create_mirror_source(os.environ["PIPENV_PYPI_MIRROR"]) if "PIPENV_PYPI_MIRROR" in os.environ else None ) - + os.environ["PIP_NO_BUILD_ISOLATION"] = "1" + os.environ["PIP_NO_USE_PEP517"] = "1" + os.environ["PIP_NO_DEPS"] = "1" def resolve(packages, pre, project, sources, clear, system, requirements_dir=None): return resolve_deps( packages, @@ -76,15 +148,8 @@ def _main(pre, clear, verbose, system, requirements_dir, packages): if pypi_mirror_source else project.pipfile_sources ) - results = resolve( - packages, - pre=pre, - project=project, - sources=sources, - clear=clear, - system=system, - requirements_dir=requirements_dir, - ) + results = resolve(packages, pre=pre, project=project, sources=sources, clear=clear, + system=system, requirements_dir=requirements_dir) print("RESULTS:") if results: print(json.dumps(results)) @@ -92,36 +157,46 @@ def _main(pre, clear, verbose, system, requirements_dir, packages): print(json.dumps([])) -def main(): - _patch_path() - import warnings - from pipenv.vendor.vistir.compat import ResourceWarning - warnings.simplefilter("ignore", category=ResourceWarning) - import io - import six - if six.PY3: - import atexit - stdout_wrapper = io.TextIOWrapper(sys.stdout.buffer, encoding='utf8') - atexit.register(stdout_wrapper.close) - stderr_wrapper = io.TextIOWrapper(sys.stderr.buffer, encoding='utf8') - atexit.register(stderr_wrapper.close) - sys.stdout = stdout_wrapper - sys.stderr = stderr_wrapper +def _main(pre, clear, verbose, system, requirements_dir, packages, parse_only=False): + os.environ["PIP_PYTHON_VERSION"] = ".".join([str(s) for s in sys.version_info[:3]]) + os.environ["PIP_PYTHON_PATH"] = str(sys.executable) + if parse_only: + parse_packages( + packages, + pre=pre, + clear=clear, + system=system, + requirements_dir=requirements_dir, + ) else: - from pipenv._compat import force_encoding - force_encoding() - os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = str("1") - os.environ["PYTHONIOENCODING"] = str("utf-8") + resolve_packages(pre, clear, verbose, system, requirements_dir, packages) + + +def main(): parser = get_parser() parsed, remaining = parser.parse_known_args() - # sys.argv = remaining + _patch_path(pipenv_site=parsed.pipenv_site) + import warnings + from pipenv.vendor import colorama + colorama.init() + from pipenv.vendor.vistir.compat import ResourceWarning + from pipenv.vendor.vistir.misc import get_wrapped_stream + warnings.simplefilter("ignore", category=ResourceWarning) + import six + if six.PY3: + stdout = sys.stdout.buffer + stderr = sys.stderr.buffer + else: + stdout = sys.stdout + stderr = sys.stderr + sys.stderr = get_wrapped_stream(stderr) + sys.stdout = get_wrapped_stream(stdout) + os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = str("1") + os.environ["PYTHONIOENCODING"] = str("utf-8") parsed = handle_parsed_args(parsed) _main(parsed.pre, parsed.clear, parsed.verbose, parsed.system, - parsed.requirements_dir, parsed.packages) + parsed.requirements_dir, parsed.packages, parse_only=parsed.parse_only) if __name__ == "__main__": - _patch_path() - from pipenv.vendor import colorama - colorama.init() main() diff --git a/pipenv/utils.py b/pipenv/utils.py index bd3c5f70..a5f9a02b 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -23,8 +23,7 @@ six.add_move(six.MovedAttribute("Sequence", "collections", "collections.abc")) six.add_move(six.MovedAttribute("Set", "collections", "collections.abc")) # noqa from six.moves import Mapping, Sequence, Set from six.moves.urllib.parse import urlparse -from urllib3 import util as urllib3_util -from vistir.compat import ResourceWarning +from vistir.compat import ResourceWarning, lru_cache from vistir.misc import fs_str import crayons @@ -33,10 +32,13 @@ import parse from . import environments from .exceptions import PipenvUsageError from .pep508checker import lookup +from .vendor.urllib3 import util as urllib3_util if environments.MYPY_RUNNING: - from typing import Tuple, Dict, Any, List, Union + from typing import Tuple, Dict, Any, List, Union, Callable, Optional + from .vendor.requirementslib.models.requirements import Requirement, Line + from .project import Project logging.basicConfig(level=logging.ERROR) @@ -306,6 +308,17 @@ def prepare_pip_source_args(sources, pip_args=None): # markers_lookup[req.name] = req.markers.replace('"', "'") # return constraints, skipped +@lru_cache() +def get_pipenv_sitedir(): + # type: () -> Optional[str] + import pkg_resources + site_dir = next( + iter(d for d in pkg_resources.working_set if d.key.lower() == "pipenv"), None + ) + if site_dir is not None: + return site_dir.location + return None + class Resolver(object): def __init__(self, constraints, req_dir, project, sources, clear=False, pre=False): @@ -339,7 +352,9 @@ class Resolver(object): "sources={self.sources})>".format(self=self) ) - def _get_pip_command(self): + @staticmethod + @lru_cache() + def _get_pip_command(): from pip_shims.shims import Command class PipCommand(Command): @@ -356,18 +371,19 @@ class Resolver(object): deps, # type: List[str] index_lookup, # type: Dict[str, str] markers_lookup, # type: Dict[str, str] - project, # type: 'pipenv.project.Project' + project, # type: Project sources # type: Dict[str, str] ): - # type: (...) -> Set() - constraints = set() - skipped = {} + # type: (...) -> Tuple[Set[str], Dict[str, Dict[str, Union[str, bool, List[str]]]]] + constraints = set() # type: Set[str] + skipped = dict() # type: Dict[str, Dict[str, Union[str, bool, List[str]]]] for dep in deps: if not dep: continue - constraint_update, lockfile_update = cls.get_deps_from_line( + req = cls.parse_line( dep, index_lookup=index_lookup, markers_lookup=markers_lookup, sources=sources ) + constraint_update, lockfile_update = cls.get_deps_from_req(req) constraints |= constraint_update skipped.update(lockfile_update) return constraints, skipped @@ -376,7 +392,8 @@ class Resolver(object): def parse_line(cls, line, index_lookup=None, markers_lookup=None, sources=None): from .vendor.requirementslib.models.requirements import Requirement if sources is None: - sources = [] # type: List[Dict[str, Union[str, bool]]] + from .core import project + sources = project.sources # type: List[Dict[str, Union[str, bool]]] url = None indexes, trusted_hosts, remainder = parse_indexes(line) if indexes: @@ -396,37 +413,63 @@ class Resolver(object): return req @classmethod - def get_deps_from_line(cls, line, index_lookup=None, markers_lookup=None, sources=None): - from .vendor.requirementslib.models.utils import _requirement_to_str_lowercase_name - if sources is None: - sources = [] # type: List[Dict[str, Union[str, bool]]] - req = cls.parse_line( - line, index_lookup=index_lookup, markers_lookup=markers_lookup, sources=sources - ) - parsed_line = req.line_instance - constraints = set() - locked_deps = {} - if ((parsed_line.is_direct_url and parsed_line.is_vcs) or - (parsed_line.is_file or parsed_line.is_url and not - (parsed_line.is_vcs and parsed_line.editable)) + def get_deps_from_line(cls, line): + # type: (str) -> Tuple[Set[str], Dict[str, Dict[str, Union[str, bool, List[str]]]]] + return cls.get_deps_from_req(cls.parse_line(line)) + + @classmethod + def get_deps_from_req(cls, req): + # type: (Requirement) -> Tuple[Set[str], Dict[str, Dict[str, Union[str, bool, List[str]]]]] + parsed_line = req.req.parsed_line # type: Line + constraints = set() # type: Set[str] + locked_deps = dict() # type: Dict[str, Dict[str, Union[str, bool, List[str]]]] + setup_info = None # type: Any + if parsed_line.is_file or parsed_line.is_vcs or parsed_line.is_url and not ( + parsed_line.is_wheel ): # for local packages with setup.py files and potential direct url deps: name, entry = req.pipfile_entry - # TODO: This might belong as a conditional include after we do the other logic in the for loop (line 427) - setup_info = parsed_line.setup_info - requirements = [v for v in setup_info.get_info().get("requires", {}).values()] + setup_info = req.req.setup_info + locked_deps[name] = entry + requirements = [v for v in getattr(setup_info, "requires", {}).values()] for r in requirements: if getattr(r, "url", None) and not getattr(r, "editable", False): - new_constraints, new_lock = cls.get_deps_from_line( - _requirement_to_str_lowercase_name(r) - ) - constraints |= new_constraints + line = str(r) + if line == "None": + if not r.url: + continue + line = r.url + new_req = cls.parse_line(line) + new_constraints, new_lock = cls.get_deps_from_req(new_req) locked_deps.update(new_lock) - continue - constraints.add(_requirement_to_str_lowercase_name(r)) - locked_deps[name] = entry + constraints |= new_constraints + else: + line = str(r) + if line is not None: + constraints.add(line) + # ensure the top level entry remains as provided + # note that we shouldn't pin versions for editable vcs deps + if (not req.is_vcs or (req.is_vcs and not req.editable)): + if req.specifiers: + locked_deps[name]["version"] = req.specifiers + elif parsed_line.setup_info and parsed_line.setup_info.version: + locked_deps[name]["version"] = "=={}".format( + parsed_line.setup_info.version + ) + if not req.is_vcs: + locked_deps.update({name: entry}) + else: + # Lock the current requirement if it's a VCS requiement (get the hash) + _, vcs_lockfile = get_vcs_deps(reqs=[req]) + locked_deps[name].update(vcs_lockfile[name]) + if req.editable: + constraints.add(req.constraint_line) + if req.is_file_or_url and req.req.is_local and req.editable and ( + req.req.setup_path is not None and os.path.exists(req.req.setup_path)): + constraints.add(req.constraint_line) else: constraints.add(req.constraint_line) + return constraints, locked_deps return constraints, locked_deps @property @@ -550,6 +593,47 @@ class Resolver(object): self.resolved_tree.update(results) return self.resolved_tree + def collect_hashes(self, ireq): + collected_hashes = [] + if ireq in self.hashes: + collected_hashes += list(self.hashes.get(ireq, [])) + if self._should_include_hash(ireq): + try: + hash_map = self.get_hash(ireq) + collected_hashes += list(hash_map) + except (ValueError, KeyError, IndexError, ConnectionError): + pass + elif any( + "python.org" in source["url"] or "pypi.org" in source["url"] + for source in self.sources + ): + pkg_url = "https://pypi.org/pypi/{0}/json".format(ireq.name) + session = _get_requests_session() + try: + # Grab the hashes from the new warehouse API. + r = session.get(pkg_url, timeout=10) + api_releases = r.json()["releases"] + cleaned_releases = {} + for api_version, api_info in api_releases.items(): + api_version = clean_pkg_version(api_version) + cleaned_releases[api_version] = api_info + version = "" + if ireq.specifier: + spec = next(iter(s for s in list(ireq.specifier._specs)), None) + if spec: + version = spec.version + for release in cleaned_releases[version]: + collected_hashes.append(release["digests"]["sha256"]) + collected_hashes = ["sha256:" + s for s in collected_hashes] + except (ValueError, KeyError, ConnectionError): + if environments.is_verbose(): + click_echo( + "{0}: Error generating hash for {1}".format( + crayons.red("Warning", bold=True), ireq.name + ), err=True + ) + return collected_hashes + @staticmethod def _should_include_hash(ireq): from pipenv.vendor.vistir.compat import Path, to_native_string @@ -632,7 +716,6 @@ def actually_resolve_deps( ): from pipenv.vendor.vistir.path import create_tracked_tempdir from pipenv.vendor.requirementslib.models.requirements import Requirement - from pipenv.vendor import pip_shims if not req_dir: req_dir = create_tracked_tempdir(suffix="-requirements", prefix="pipenv-") @@ -644,27 +727,68 @@ def actually_resolve_deps( ) resolver = Resolver(constraints, req_dir, project, sources, clear=clear, pre=pre) resolved_tree = resolver.resolve() - for k, v in skipped.items(): - url = v.get("file") - req = Requirement.from_pipfile(k, v) - is_url = url and not url.startswith("file:") - path = v.get("path") - if not is_url and not path: - try: - path = pip_shims.shims.url_to_path(url) - except AttributeError: - path = None - if is_url or (path and os.path.exists(path) and not os.path.isdir(path)) or req.is_vcs: - existing = next(iter(req for req in resolved_tree if req.name == k), None) - if existing: - resolved_tree.remove(existing) - resolved_tree.add(req.as_ireq()) hashes = resolver.resolve_hashes() - + reqs = [(Requirement.from_ireq(ireq), ireq) for ireq in resolved_tree] + results = {} + for req, ireq in reqs: + if (req.vcs and req.editable and not req.is_direct_url): + continue + collected_hashes = resolver.collect_hashes(ireq) + if collected_hashes: + req = req.add_hashes(collected_hashes) + elif resolver._should_include_hash(ireq): + existing_hashes = hashes.get(ireq, set()) + discovered_hashes = existing_hashes | resolver.get_hash(ireq) + if discovered_hashes: + req = req.add_hashes(discovered_hashes) + resolver.hashes[ireq] = discovered_hashes + if req.specifiers: + version = str(req.get_version()) + else: + version = None + index = index_lookup.get(req.name) + req.index = index + name, pf_entry = req.pipfile_entry + name = pep423_name(req.name) + entry = {} + if isinstance(pf_entry, six.string_types): + entry["version"] = pf_entry.lstrip("=") + else: + entry.update(pf_entry) + if version is not None: + entry["version"] = version + if req.line_instance.is_direct_url: + entry["file"] = req.req.uri + if collected_hashes: + entry["hashes"] = sorted(set(collected_hashes)) + entry["name"] = name + if index and index != next(iter(project.sources), {}).get("name"): + entry.update({"index": index}) + if markers_lookup.get(req.name): + entry.update({"markers": markers_lookup.get(req.name)}) + entry = translate_markers(entry) + if name in results: + results[name].update(entry) + else: + results[name] = entry + for k in list(skipped.keys()): + req = Requirement.from_pipfile(k, skipped[k]) + ireq = req.as_ireq() + entry = skipped[k].copy() + entry["name"] = pep423_name(k) + if resolver._should_include_hash(ireq): + collected_hashes = resolver.collect_hashes(ireq) + if collected_hashes: + entry["hashes"] = sorted(set(collected_hashes)) + if k in results: + results[k].update(entry) + else: + results[k] = entry + results = list(results.values()) for warning in warning_list: _show_warning(warning.message, warning.category, warning.filename, warning.lineno, warning.line) - return (resolved_tree, hashes, markers_lookup, resolver) + return (results, hashes, markers_lookup, resolver, skipped) @contextlib.contextmanager @@ -721,7 +845,7 @@ def resolve(cmd, sp): return c -def get_locked_dep(dep, pipfile_section, prefer_pipfile=False): +def get_locked_dep(dep, pipfile_section, prefer_pipfile=True): # the prefer pipfile flag is not used yet, but we are introducing # it now for development purposes # TODO: Is this implementation clear? How can it be improved? @@ -754,16 +878,18 @@ def get_locked_dep(dep, pipfile_section, prefer_pipfile=False): def prepare_lockfile(results, pipfile, lockfile): from .vendor.requirementslib.utils import is_vcs for dep in results: + if not dep: + continue # Merge in any relevant information from the pipfile entry, including # markers, normalized names, URL info, etc that we may have dropped during lock - if not is_vcs(dep): - lockfile_entry = get_locked_dep(dep, pipfile) - name = next(iter(k for k in lockfile_entry.keys())) - current_entry = lockfile.get(name) - if current_entry and not is_vcs(current_entry): - lockfile[name].update(lockfile_entry[name]) - else: - lockfile[name] = lockfile_entry[name] + # if not is_vcs(dep): + lockfile_entry = get_locked_dep(dep, pipfile) + name = next(iter(k for k in lockfile_entry.keys())) + current_entry = lockfile.get(name) + if current_entry: + lockfile[name].update(lockfile_entry[name]) + else: + lockfile[name] = lockfile_entry[name] return lockfile @@ -779,6 +905,30 @@ def venv_resolve_deps( pipfile=None, lockfile=None ): + """ + Resolve dependencies for a pipenv project, acts as a portal to the target environment. + + Regardless of whether a virtual environment is present or not, this will spawn + a subproces which is isolated to the target environment and which will perform + dependency resolution. This function reads the output of that call and mutates + the provided lockfile accordingly, returning nothing. + + :param List[:class:`~requirementslib.Requirement`] deps: A list of dependencies to resolve. + :param Callable which: [description] + :param project: The pipenv Project instance to use during resolution + :param Optional[bool] pre: Whether to resolve pre-release candidates, defaults to False + :param Optional[bool] clear: Whether to clear the cache during resolution, defaults to False + :param Optional[bool] allow_global: Whether to use *sys.executable* as the python binary, defaults to False + :param Optional[str] pypi_mirror: A URL to substitute any time *pypi.org* is encountered, defaults to None + :param Optional[bool] dev: Whether to target *dev-packages* or not, defaults to False + :param pipfile: A Pipfile section to operate on, defaults to None + :type pipfile: Optional[Dict[str, Union[str, Dict[str, bool, List[str]]]]] + :param Dict[str, Any] lockfile: A project lockfile to mutate, defaults to None + :raises RuntimeError: Raised on resolution failure + :return: Nothing + :rtype: None + """ + from .vendor.vistir.misc import fs_str from .vendor.vistir.compat import Path, to_native_string, JSONDecodeError from .vendor.vistir.path import create_tracked_tempdir @@ -787,11 +937,14 @@ def venv_resolve_deps( vcs_deps = [] vcs_lockfile = {} + # url_lockfile = {} results = [] pipfile_section = "dev_packages" if dev else "packages" lockfile_section = "develop" if dev else "default" vcs_section = "vcs_{0}".format(pipfile_section) if project.pipfile_exists: + # This is a requirementslib pipfile instance which provides `Requirement` instances + # rather than simply locked dependencies in a lockfile format deps = project._pipfile.dev_requirements if dev else project._pipfile.requirements vcs_deps = [r for r in deps if r.is_vcs] else: @@ -804,23 +957,30 @@ def venv_resolve_deps( if not lockfile: lockfile = project._lockfile req_dir = create_tracked_tempdir(prefix="pipenv", suffix="requirements") - for dep in deps: - if dep.is_file_or_url and not dep.is_vcs: - name, entry = dep.pipfile_entry - lockfile[lockfile_section][name] = entry + constraints = set() + # for dep in deps: + # if dep.is_file_or_url and not dep.is_vcs: + # with temp_environ(): + # os.environ["PIP_NO_USE_PEP_517"] = fs_str("1") + # os.environ["PIPENV_SITE_DIR"] = get_pipenv_sitedir() + # name, entry = dep.pipfile_entry + # constraint_update, lockfile_update = Resolver.get_deps_from_req(dep) + # url_lockfile[name] = entry + # if name in lockfile_update: + # url_lockfile[name].update(lockfile_update[name]) + # lockfile[lockfile_section].update(lockfile_update) + # url_lockfile.update(lockfile_update) + # constraints |= constraint_update if vcs_deps: - with create_spinner(text=fs_str("Pinning VCS Packages...")) as sp: - vcs_reqs, vcs_lockfile = get_vcs_deps( - project, - which=which, - clear=clear, - pre=pre, - allow_global=allow_global, - dev=dev, - ) + with temp_environ(), create_spinner(text=fs_str("Pinning VCS Packages...")) as sp: + os.environ["PIPENV_SITE_DIR"] = get_pipenv_sitedir() + vcs_reqs, vcs_lockfile = get_vcs_deps(project=project, dev=dev) vcs_deps = [req.as_line() for req in vcs_reqs if req.editable] lockfile[lockfile_section].update(vcs_lockfile) - deps = [r.as_line() for r in deps if not r.is_vcs] + # new_constraints = {r.as_line() for r in deps if not (r.is_vcs or (r.is_file_or_url + # and r.line_instance and not r.line_instance.is_wheel))} + # constraints |= new_constraints + constraints = {r.as_line() for r in deps} # if not r.is_vcs} cmd = [ which("python", allow_global=allow_global), Path(resolver.__file__.rstrip("co")).as_posix() @@ -833,50 +993,55 @@ def venv_resolve_deps( cmd.append("--system") with temp_environ(): os.environ = {fs_str(k): fs_str(val) for k, val in os.environ.items()} - os.environ["PIPENV_PACKAGES"] = str("\n".join(deps)) + os.environ["PIPENV_PACKAGES"] = str("\n".join(constraints)) if pypi_mirror: os.environ["PIPENV_PYPI_MIRROR"] = str(pypi_mirror) os.environ["PIPENV_VERBOSITY"] = str(environments.PIPENV_VERBOSITY) os.environ["PIPENV_REQ_DIR"] = fs_str(req_dir) os.environ["PIP_NO_INPUT"] = fs_str("1") + os.environ["PIPENV_SITE_DIR"] = get_pipenv_sitedir() + os.environ["PIP_NO_USE_PEP517"] = fs_str("1") + os.environ["PIP_NO_BUILD_ISOLATION"] = fs_str("1") with create_spinner(text=fs_str("Locking...")) as sp: c = resolve(cmd, sp) - results = c.out - if vcs_deps: - with temp_environ(): - os.environ["PIPENV_PACKAGES"] = str("\n".join(vcs_deps)) - sp.text = to_native_string("Locking VCS Dependencies...") - vcs_c = resolve(cmd, sp) - vcs_results, vcs_err = vcs_c.out, vcs_c.err - else: - vcs_results, vcs_err = "", "" - sp.green.ok(environments.PIPENV_SPINNER_OK_TEXT.format("Success!")) - outputs = [results, vcs_results] + results = c.out.strip() + # if vcs_deps: + # with temp_environ(): + # os.environ["PIPENV_PACKAGES"] = str("\n".join(vcs_deps)) + # sp.text = to_native_string("Locking VCS Dependencies...") + # vcs_c = resolve(cmd, sp) + # vcs_results, vcs_err = vcs_c.out.strip(), vcs_c.err.strip() + # else: + # vcs_results, vcs_err = "", "" + # sp.green.ok(environments.PIPENV_SPINNER_OK_TEXT.format("Success!")) if environments.is_verbose(): - for output in outputs: - click_echo(output.split("RESULTS:")[0], err=True) + click_echo(results.split("RESULTS:")[1], err=True) + # for output in outputs: + # click_echo(output.split("RESULTS:")[0], err=True) try: results = json.loads(results.split("RESULTS:")[1].strip()) - if vcs_results: + # if vcs_results: # For vcs dependencies, treat the initial pass at locking (i.e. checkout) # as the pipfile entry because it gets us an actual ref to use - vcs_results = json.loads(vcs_results.split("RESULTS:")[1].strip()) - vcs_lockfile = prepare_lockfile(vcs_results, vcs_lockfile.copy(), vcs_lockfile) - else: - vcs_results = [] + # vcs_results = json.loads(vcs_results.split("RESULTS:")[1].strip()) + # vcs_lockfile = prepare_lockfile(vcs_results, vcs_lockfile.copy(), vcs_lockfile) + # else: + # vcs_results = [] except (IndexError, JSONDecodeError): - for out, err in [(c.out, c.err), (vcs_results, vcs_err)]: - click_echo(out.strip(), err=True) - click_echo(err.strip(), err=True) + click_echo(c.out.strip(), err=True) + click_echo(c.err.strip(), err=True) + # for out, err in [(c.out, c.err), (vcs_results, vcs_err)]: + # click_echo(out.strip(), err=True) + # click_echo(err.strip(), err=True) raise RuntimeError("There was a problem with locking.") lockfile[lockfile_section] = prepare_lockfile(results, pipfile, lockfile[lockfile_section]) - for k, v in vcs_lockfile.items(): - if k in getattr(project, vcs_section, {}): - if not (isinstance(v, six.string_types) and isinstance(k, Mapping)): - lockfile[lockfile_section][k].update(v) - else: - lockfile[lockfile_section][k] = v + # for k, v in vcs_lockfile.items(): + # if k in getattr(project, vcs_section, {}): + # if not (isinstance(v, six.string_types) and isinstance(k, Mapping)): + # lockfile[lockfile_section][k].update(v) + # else: + # lockfile[lockfile_section][k] = v def resolve_deps( @@ -902,6 +1067,8 @@ def resolve_deps( if not os.environ.get("PIP_SRC"): os.environ["PIP_SRC"] = project.virtualenv_src_location backup_python_path = sys.executable + os.environ["PIP_NO_BUILD_ISOLATION"] = "1" + os.environ["PIP_NO_USE_PEP517"] = "1" results = [] if not deps: return results @@ -912,7 +1079,7 @@ def resolve_deps( req_dir = create_tracked_tempdir(prefix="pipenv-", suffix="-requirements") with HackedPythonVersion(python_version=python, python_path=python_path): try: - resolved_tree, hashes, markers_lookup, resolver = actually_resolve_deps( + resolved_tree, hashes, markers_lookup, resolver, skipped = actually_resolve_deps( deps, index_lookup, markers_lookup, @@ -934,7 +1101,7 @@ def resolve_deps( try: # Attempt to resolve again, with different Python version information, # particularly for particularly particular packages. - resolved_tree, hashes, markers_lookup, resolver = actually_resolve_deps( + resolved_tree, hashes, markers_lookup, resolver, skipped = actually_resolve_deps( deps, index_lookup, markers_lookup, @@ -946,70 +1113,30 @@ def resolve_deps( ) except RuntimeError: sys.exit(1) - for result in resolved_tree: - if not result.editable: - req = Requirement.from_ireq(result) - name = pep423_name(req.name) - name, pf_entry = req.pipfile_entry - if req.specifiers: - version = str(req.get_version()) - else: - version = None - index = index_lookup.get(result.name) - req.index = index - collected_hashes = [] - if result in hashes: - collected_hashes = list(hashes.get(result)) - elif resolver._should_include_hash(result): - try: - hash_map = resolver.get_hash(result) - collected_hashes = list(hash_map) - except (ValueError, KeyError, IndexError, ConnectionError): - pass - elif any( - "python.org" in source["url"] or "pypi.org" in source["url"] - for source in sources - ): - pkg_url = "https://pypi.org/pypi/{0}/json".format(name) - session = _get_requests_session() - try: - # Grab the hashes from the new warehouse API. - r = session.get(pkg_url, timeout=10) - api_releases = r.json()["releases"] - cleaned_releases = {} - for api_version, api_info in api_releases.items(): - api_version = clean_pkg_version(api_version) - cleaned_releases[api_version] = api_info - for release in cleaned_releases[version]: - collected_hashes.append(release["digests"]["sha256"]) - collected_hashes = ["sha256:" + s for s in collected_hashes] - except (ValueError, KeyError, ConnectionError): - if environments.is_verbose(): - click_echo( - "{0}: Error generating hash for {1}".format( - crayons.red("Warning", bold=True), name - ), err=True - ) - req.hashes = sorted(set(collected_hashes)) - entry = {} - if isinstance(pf_entry, six.string_types): - entry["version"] = pf_entry.lstrip("=") - else: - entry.update(pf_entry) - if version is not None: - entry["version"] = version - if req.line_instance.is_direct_url: - entry["file"] = req.req.uri - if collected_hashes: - entry["hashes"] = sorted(set(collected_hashes)) - entry["name"] = name - # if index: - # d.update({"index": index}) - if markers_lookup.get(result.name): - entry.update({"markers": markers_lookup.get(result.name)}) - entry = translate_markers(entry) - results.append(entry) - return results + # for req in resolved_tree: + # if not req.editable: + # result = req.as_ireq() + + # req.hashes = sorted(set(collected_hashes)) + # entry = {} + # if isinstance(pf_entry, six.string_types): + # entry["version"] = pf_entry.lstrip("=") + # else: + # entry.update(pf_entry) + # if version is not None: + # entry["version"] = version + # if req.line_instance.is_direct_url: + # entry["file"] = req.req.uri + # if collected_hashes: + # entry["hashes"] = sorted(set(collected_hashes)) + # entry["name"] = name + # # if index: + # # d.update({"index": index}) + # if markers_lookup.get(result.name): + # entry.update({"markers": markers_lookup.get(result.name)}) + # entry = translate_markers(entry) + # results.append(entry) + return resolved_tree def is_star(val): @@ -1488,37 +1615,94 @@ def safe_expandvars(value): def get_vcs_deps( - project, - which=None, - clear=False, - pre=False, - allow_global=False, + project=None, dev=False, pypi_mirror=None, + packages=None, + reqs=None ): from .vendor.requirementslib.models.requirements import Requirement + from .vendor import attr section = "vcs_dev_packages" if dev else "vcs_packages" - reqs = [] + if reqs is None: + reqs = [] lockfile = {} - try: - packages = getattr(project, section) - except AttributeError: - return [], [] - for pkg_name, pkg_pipfile in packages.items(): - requirement = Requirement.from_pipfile(pkg_name, pkg_pipfile) + if not reqs: + if not project and not packages: + raise ValueError( + "Must supply either a project or a pipfile section to lock vcs dependencies." + ) + if not packages: + try: + packages = getattr(project, section) + except AttributeError: + return [], [] + reqs = [Requirement.from_pipfile(name, entry) for name, entry in packages.items()] + result = [] + updated_reqs = [] + for requirement in reqs: name = requirement.normalized_name commit_hash = None if requirement.is_vcs: try: - with locked_repository(requirement) as repo: + with temp_path(), locked_repository(requirement) as repo: + from pipenv.vendor.requirementslib.models.requirements import Requirement + from distutils.sysconfig import get_python_lib + sys.path = [repo.checkout_directory, "", ".", get_python_lib(plat_specific=0)] commit_hash = repo.get_commit_hash() + name = requirement.normalized_name + version = requirement._specifiers = "=={0}".format(requirement.req.setup_info.version) lockfile[name] = requirement.pipfile_entry[1] lockfile[name]['ref'] = commit_hash - reqs.append(requirement) + # new_req = Requirement.from_line(repo.checkout_directory) + # si = new_req.req.setup_info + # hookcaller = pep517.wrappers.Pep517HookCaller(new_req.req.setup_info.base_dir, new_req.req.setup_info.build_backend) + # deps = hookcaller.get_requires_for_build_wheel() + hookcaller.get_requires_for_build_sdist() + # hookcaller.prepare_metadata_for_build_wheel(new_req.req.setup_info.egg_base) + # r._specifiers = new_req.specifiers + # pkging_req = r.req.req + # pkging_req.specifier = new_req.req.req.specifier + # pkging_req.spec = new_req.req.req.spec + # new_parsed_line = r.req._parsed_line + # new_parsed_line._setup_info = new_req.req.parsed_line.setup_info + # new_parsed_line.specifiers = new_req.req.parsed_line.specifiers + # new_parsed_line._requirement = pkging_req + # new_parsed_line.ireq.req = pkging_req + # req = attr.evolve( + # r.req, + # setup_info=new_req.req.setup_info, + # parsed_line=new_parsed_line + # ) + # r = attr.evolve( + # r, + # req=req, + # line_instance=new_parsed_line + # ) + result.append(requirement) + version = requirement.specifiers + if not version and requirement.specifiers: + version = requirement.specifiers + if version: + lockfile[name]['version'] = version + # new_req = Requirement.from_line(repo.checkout_directory) + # requirement._specifiers = new_req.specifiers + # new_parsed_line = requirement.req._parsed_line + # new_parsed_line._setup_info = new_req.req.parsed_line.setup_info + # new_parsed_line.specifiers = new_req.req.parsed_line.specifiers + # req = attr.evolve( + # requirement.req, + # _setup_info=new_req.req.setup_info, + # _parsed_line=new_parsed_line + # ) + # requirement = attr.evolve( + # requirement, + # req=req, + # line_instance=new_parsed_line + # ) except OSError: continue - return reqs, lockfile + return result, lockfile def translate_markers(pipfile_entry): @@ -1561,23 +1745,38 @@ def translate_markers(pipfile_entry): def clean_resolved_dep(dep, is_top_level=False, pipfile_entry=None): + from .vendor.requirementslib.utils import is_vcs name = pep423_name(dep["name"]) + lockfile = {} # We use this to determine if there are any markers on top level packages # So we can make sure those win out during resolution if the packages reoccur - lockfile = {"version": "=={0}".format(dep["version"])} - for key in ["hashes", "index", "extras"]: + if "version" in dep: + version = "{0}".format(dep["version"]) + if not version.startswith("=="): + version = "=={0}".format(version) + lockfile["version"] = version + if is_vcs(dep): + ref = dep.get("ref", None) + if ref is not None: + lockfile["ref"] = ref + vcs_type = next(iter(k for k in dep.keys() if k in VCS_LIST), None) + if vcs_type: + lockfile[vcs_type] = dep[vcs_type] + if "subdirectory" in dep: + lockfile["subdirectory"] = dep["subdirectory"] + for key in ["hashes", "index", "extras", "editable"]: if key in dep: lockfile[key] = dep[key] # In case we lock a uri or a file when the user supplied a path # remove the uri or file keys from the entry and keep the path - if pipfile_entry and any(k in pipfile_entry for k in ["file", "path"]): - fs_key = next(iter(k for k in ["path", "file"] if k in dep), None) - if fs_key is not None: - lockfile[fs_key] = pipfile_entry[fs_key] - elif any(k in dep for k in ["file", "path"]): - fs_key = next(iter(k for k in ["path", "file"] if k in dep), None) - if fs_key is not None: - lockfile[fs_key] = dep[fs_key] + fs_key = next(iter(k for k in ["path", "file"] if k in dep), None) + pipfile_fs_key = None + if pipfile_entry: + pipfile_fs_key = next(iter(k for k in ["path", "file"] if k in pipfile_entry), None) + if fs_key and pipfile_fs_key and fs_key != pipfile_fs_key: + lockfile[pipfile_fs_key] = pipfile_entry[pipfile_fs_key] + elif fs_key is not None: + lockfile[fs_key] = dep[fs_key] # If a package is **PRESENT** in the pipfile but has no markers, make sure we # **NEVER** include markers in the lockfile diff --git a/pipenv/vendor/pytoml/__init__.py b/pipenv/vendor/pytoml/__init__.py new file mode 100644 index 00000000..8ed060ff --- /dev/null +++ b/pipenv/vendor/pytoml/__init__.py @@ -0,0 +1,4 @@ +from .core import TomlError +from .parser import load, loads +from .test import translate_to_test +from .writer import dump, dumps \ No newline at end of file diff --git a/pipenv/vendor/pytoml/core.py b/pipenv/vendor/pytoml/core.py new file mode 100644 index 00000000..c182734e --- /dev/null +++ b/pipenv/vendor/pytoml/core.py @@ -0,0 +1,13 @@ +class TomlError(RuntimeError): + def __init__(self, message, line, col, filename): + RuntimeError.__init__(self, message, line, col, filename) + self.message = message + self.line = line + self.col = col + self.filename = filename + + def __str__(self): + return '{}({}, {}): {}'.format(self.filename, self.line, self.col, self.message) + + def __repr__(self): + return 'TomlError({!r}, {!r}, {!r}, {!r})'.format(self.message, self.line, self.col, self.filename) diff --git a/pipenv/vendor/pytoml/parser.py b/pipenv/vendor/pytoml/parser.py new file mode 100644 index 00000000..3493aa64 --- /dev/null +++ b/pipenv/vendor/pytoml/parser.py @@ -0,0 +1,341 @@ +import string, re, sys, datetime +from .core import TomlError +from .utils import rfc3339_re, parse_rfc3339_re + +if sys.version_info[0] == 2: + _chr = unichr +else: + _chr = chr + +def load(fin, translate=lambda t, x, v: v, object_pairs_hook=dict): + return loads(fin.read(), translate=translate, object_pairs_hook=object_pairs_hook, filename=getattr(fin, 'name', repr(fin))) + +def loads(s, filename='', translate=lambda t, x, v: v, object_pairs_hook=dict): + if isinstance(s, bytes): + s = s.decode('utf-8') + + s = s.replace('\r\n', '\n') + + root = object_pairs_hook() + tables = object_pairs_hook() + scope = root + + src = _Source(s, filename=filename) + ast = _p_toml(src, object_pairs_hook=object_pairs_hook) + + def error(msg): + raise TomlError(msg, pos[0], pos[1], filename) + + def process_value(v, object_pairs_hook): + kind, text, value, pos = v + if kind == 'str' and value.startswith('\n'): + value = value[1:] + if kind == 'array': + if value and any(k != value[0][0] for k, t, v, p in value[1:]): + error('array-type-mismatch') + value = [process_value(item, object_pairs_hook=object_pairs_hook) for item in value] + elif kind == 'table': + value = object_pairs_hook([(k, process_value(value[k], object_pairs_hook=object_pairs_hook)) for k in value]) + return translate(kind, text, value) + + for kind, value, pos in ast: + if kind == 'kv': + k, v = value + if k in scope: + error('duplicate_keys. Key "{0}" was used more than once.'.format(k)) + scope[k] = process_value(v, object_pairs_hook=object_pairs_hook) + else: + is_table_array = (kind == 'table_array') + cur = tables + for name in value[:-1]: + if isinstance(cur.get(name), list): + d, cur = cur[name][-1] + else: + d, cur = cur.setdefault(name, (None, object_pairs_hook())) + + scope = object_pairs_hook() + name = value[-1] + if name not in cur: + if is_table_array: + cur[name] = [(scope, object_pairs_hook())] + else: + cur[name] = (scope, object_pairs_hook()) + elif isinstance(cur[name], list): + if not is_table_array: + error('table_type_mismatch') + cur[name].append((scope, object_pairs_hook())) + else: + if is_table_array: + error('table_type_mismatch') + old_scope, next_table = cur[name] + if old_scope is not None: + error('duplicate_tables') + cur[name] = (scope, next_table) + + def merge_tables(scope, tables): + if scope is None: + scope = object_pairs_hook() + for k in tables: + if k in scope: + error('key_table_conflict') + v = tables[k] + if isinstance(v, list): + scope[k] = [merge_tables(sc, tbl) for sc, tbl in v] + else: + scope[k] = merge_tables(v[0], v[1]) + return scope + + return merge_tables(root, tables) + +class _Source: + def __init__(self, s, filename=None): + self.s = s + self._pos = (1, 1) + self._last = None + self._filename = filename + self.backtrack_stack = [] + + def last(self): + return self._last + + def pos(self): + return self._pos + + def fail(self): + return self._expect(None) + + def consume_dot(self): + if self.s: + self._last = self.s[0] + self.s = self[1:] + self._advance(self._last) + return self._last + return None + + def expect_dot(self): + return self._expect(self.consume_dot()) + + def consume_eof(self): + if not self.s: + self._last = '' + return True + return False + + def expect_eof(self): + return self._expect(self.consume_eof()) + + def consume(self, s): + if self.s.startswith(s): + self.s = self.s[len(s):] + self._last = s + self._advance(s) + return True + return False + + def expect(self, s): + return self._expect(self.consume(s)) + + def consume_re(self, re): + m = re.match(self.s) + if m: + self.s = self.s[len(m.group(0)):] + self._last = m + self._advance(m.group(0)) + return m + return None + + def expect_re(self, re): + return self._expect(self.consume_re(re)) + + def __enter__(self): + self.backtrack_stack.append((self.s, self._pos)) + + def __exit__(self, type, value, traceback): + if type is None: + self.backtrack_stack.pop() + else: + self.s, self._pos = self.backtrack_stack.pop() + return type == TomlError + + def commit(self): + self.backtrack_stack[-1] = (self.s, self._pos) + + def _expect(self, r): + if not r: + raise TomlError('msg', self._pos[0], self._pos[1], self._filename) + return r + + def _advance(self, s): + suffix_pos = s.rfind('\n') + if suffix_pos == -1: + self._pos = (self._pos[0], self._pos[1] + len(s)) + else: + self._pos = (self._pos[0] + s.count('\n'), len(s) - suffix_pos) + +_ews_re = re.compile(r'(?:[ \t]|#[^\n]*\n|#[^\n]*\Z|\n)*') +def _p_ews(s): + s.expect_re(_ews_re) + +_ws_re = re.compile(r'[ \t]*') +def _p_ws(s): + s.expect_re(_ws_re) + +_escapes = { 'b': '\b', 'n': '\n', 'r': '\r', 't': '\t', '"': '"', + '\\': '\\', 'f': '\f' } + +_basicstr_re = re.compile(r'[^"\\\000-\037]*') +_short_uni_re = re.compile(r'u([0-9a-fA-F]{4})') +_long_uni_re = re.compile(r'U([0-9a-fA-F]{8})') +_escapes_re = re.compile(r'[btnfr\"\\]') +_newline_esc_re = re.compile('\n[ \t\n]*') +def _p_basicstr_content(s, content=_basicstr_re): + res = [] + while True: + res.append(s.expect_re(content).group(0)) + if not s.consume('\\'): + break + if s.consume_re(_newline_esc_re): + pass + elif s.consume_re(_short_uni_re) or s.consume_re(_long_uni_re): + v = int(s.last().group(1), 16) + if 0xd800 <= v < 0xe000: + s.fail() + res.append(_chr(v)) + else: + s.expect_re(_escapes_re) + res.append(_escapes[s.last().group(0)]) + return ''.join(res) + +_key_re = re.compile(r'[0-9a-zA-Z-_]+') +def _p_key(s): + with s: + s.expect('"') + r = _p_basicstr_content(s, _basicstr_re) + s.expect('"') + return r + if s.consume('\''): + if s.consume('\'\''): + r = s.expect_re(_litstr_ml_re).group(0) + s.expect('\'\'\'') + else: + r = s.expect_re(_litstr_re).group(0) + s.expect('\'') + return r + return s.expect_re(_key_re).group(0) + +_float_re = re.compile(r'[+-]?(?:0|[1-9](?:_?\d)*)(?:\.\d(?:_?\d)*)?(?:[eE][+-]?(?:\d(?:_?\d)*))?') + +_basicstr_ml_re = re.compile(r'(?:""?(?!")|[^"\\\000-\011\013-\037])*') +_litstr_re = re.compile(r"[^'\000\010\012-\037]*") +_litstr_ml_re = re.compile(r"(?:(?:|'|'')(?:[^'\000-\010\013-\037]))*") +def _p_value(s, object_pairs_hook): + pos = s.pos() + + if s.consume('true'): + return 'bool', s.last(), True, pos + if s.consume('false'): + return 'bool', s.last(), False, pos + + if s.consume('"'): + if s.consume('""'): + r = _p_basicstr_content(s, _basicstr_ml_re) + s.expect('"""') + else: + r = _p_basicstr_content(s, _basicstr_re) + s.expect('"') + return 'str', r, r, pos + + if s.consume('\''): + if s.consume('\'\''): + r = s.expect_re(_litstr_ml_re).group(0) + s.expect('\'\'\'') + else: + r = s.expect_re(_litstr_re).group(0) + s.expect('\'') + return 'str', r, r, pos + + if s.consume_re(rfc3339_re): + m = s.last() + return 'datetime', m.group(0), parse_rfc3339_re(m), pos + + if s.consume_re(_float_re): + m = s.last().group(0) + r = m.replace('_','') + if '.' in m or 'e' in m or 'E' in m: + return 'float', m, float(r), pos + else: + return 'int', m, int(r, 10), pos + + if s.consume('['): + items = [] + with s: + while True: + _p_ews(s) + items.append(_p_value(s, object_pairs_hook=object_pairs_hook)) + s.commit() + _p_ews(s) + s.expect(',') + s.commit() + _p_ews(s) + s.expect(']') + return 'array', None, items, pos + + if s.consume('{'): + _p_ws(s) + items = object_pairs_hook() + if not s.consume('}'): + k = _p_key(s) + _p_ws(s) + s.expect('=') + _p_ws(s) + items[k] = _p_value(s, object_pairs_hook=object_pairs_hook) + _p_ws(s) + while s.consume(','): + _p_ws(s) + k = _p_key(s) + _p_ws(s) + s.expect('=') + _p_ws(s) + items[k] = _p_value(s, object_pairs_hook=object_pairs_hook) + _p_ws(s) + s.expect('}') + return 'table', None, items, pos + + s.fail() + +def _p_stmt(s, object_pairs_hook): + pos = s.pos() + if s.consume( '['): + is_array = s.consume('[') + _p_ws(s) + keys = [_p_key(s)] + _p_ws(s) + while s.consume('.'): + _p_ws(s) + keys.append(_p_key(s)) + _p_ws(s) + s.expect(']') + if is_array: + s.expect(']') + return 'table_array' if is_array else 'table', keys, pos + + key = _p_key(s) + _p_ws(s) + s.expect('=') + _p_ws(s) + value = _p_value(s, object_pairs_hook=object_pairs_hook) + return 'kv', (key, value), pos + +_stmtsep_re = re.compile(r'(?:[ \t]*(?:#[^\n]*)?\n)+[ \t]*') +def _p_toml(s, object_pairs_hook): + stmts = [] + _p_ews(s) + with s: + stmts.append(_p_stmt(s, object_pairs_hook=object_pairs_hook)) + while True: + s.commit() + s.expect_re(_stmtsep_re) + stmts.append(_p_stmt(s, object_pairs_hook=object_pairs_hook)) + _p_ews(s) + s.expect_eof() + return stmts diff --git a/pipenv/vendor/pytoml/test.py b/pipenv/vendor/pytoml/test.py new file mode 100644 index 00000000..ec8abfc6 --- /dev/null +++ b/pipenv/vendor/pytoml/test.py @@ -0,0 +1,30 @@ +import datetime +from .utils import format_rfc3339 + +try: + _string_types = (str, unicode) + _int_types = (int, long) +except NameError: + _string_types = str + _int_types = int + +def translate_to_test(v): + if isinstance(v, dict): + return { k: translate_to_test(v) for k, v in v.items() } + if isinstance(v, list): + a = [translate_to_test(x) for x in v] + if v and isinstance(v[0], dict): + return a + else: + return {'type': 'array', 'value': a} + if isinstance(v, datetime.datetime): + return {'type': 'datetime', 'value': format_rfc3339(v)} + if isinstance(v, bool): + return {'type': 'bool', 'value': 'true' if v else 'false'} + if isinstance(v, _int_types): + return {'type': 'integer', 'value': str(v)} + if isinstance(v, float): + return {'type': 'float', 'value': '{:.17}'.format(v)} + if isinstance(v, _string_types): + return {'type': 'string', 'value': v} + raise RuntimeError('unexpected value: {!r}'.format(v)) diff --git a/pipenv/vendor/pytoml/utils.py b/pipenv/vendor/pytoml/utils.py new file mode 100644 index 00000000..636a680b --- /dev/null +++ b/pipenv/vendor/pytoml/utils.py @@ -0,0 +1,67 @@ +import datetime +import re + +rfc3339_re = re.compile(r'(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(\.\d+)?(?:Z|([+-]\d{2}):(\d{2}))') + +def parse_rfc3339(v): + m = rfc3339_re.match(v) + if not m or m.group(0) != v: + return None + return parse_rfc3339_re(m) + +def parse_rfc3339_re(m): + r = map(int, m.groups()[:6]) + if m.group(7): + micro = float(m.group(7)) + else: + micro = 0 + + if m.group(8): + g = int(m.group(8), 10) * 60 + int(m.group(9), 10) + tz = _TimeZone(datetime.timedelta(0, g * 60)) + else: + tz = _TimeZone(datetime.timedelta(0, 0)) + + y, m, d, H, M, S = r + return datetime.datetime(y, m, d, H, M, S, int(micro * 1000000), tz) + + +def format_rfc3339(v): + offs = v.utcoffset() + offs = int(offs.total_seconds()) // 60 if offs is not None else 0 + + if offs == 0: + suffix = 'Z' + else: + if offs > 0: + suffix = '+' + else: + suffix = '-' + offs = -offs + suffix = '{0}{1:02}:{2:02}'.format(suffix, offs // 60, offs % 60) + + if v.microsecond: + return v.strftime('%Y-%m-%dT%H:%M:%S.%f') + suffix + else: + return v.strftime('%Y-%m-%dT%H:%M:%S') + suffix + +class _TimeZone(datetime.tzinfo): + def __init__(self, offset): + self._offset = offset + + def utcoffset(self, dt): + return self._offset + + def dst(self, dt): + return None + + def tzname(self, dt): + m = self._offset.total_seconds() // 60 + if m < 0: + res = '-' + m = -m + else: + res = '+' + h = m // 60 + m = m - h * 60 + return '{}{:.02}{:.02}'.format(res, h, m) diff --git a/pipenv/vendor/pytoml/writer.py b/pipenv/vendor/pytoml/writer.py new file mode 100644 index 00000000..73b5089c --- /dev/null +++ b/pipenv/vendor/pytoml/writer.py @@ -0,0 +1,106 @@ +from __future__ import unicode_literals +import io, datetime, math, string, sys + +from .utils import format_rfc3339 + +if sys.version_info[0] == 3: + long = int + unicode = str + + +def dumps(obj, sort_keys=False): + fout = io.StringIO() + dump(obj, fout, sort_keys=sort_keys) + return fout.getvalue() + + +_escapes = {'\n': 'n', '\r': 'r', '\\': '\\', '\t': 't', '\b': 'b', '\f': 'f', '"': '"'} + + +def _escape_string(s): + res = [] + start = 0 + + def flush(): + if start != i: + res.append(s[start:i]) + return i + 1 + + i = 0 + while i < len(s): + c = s[i] + if c in '"\\\n\r\t\b\f': + start = flush() + res.append('\\' + _escapes[c]) + elif ord(c) < 0x20: + start = flush() + res.append('\\u%04x' % ord(c)) + i += 1 + + flush() + return '"' + ''.join(res) + '"' + + +_key_chars = string.digits + string.ascii_letters + '-_' +def _escape_id(s): + if any(c not in _key_chars for c in s): + return _escape_string(s) + return s + + +def _format_value(v): + if isinstance(v, bool): + return 'true' if v else 'false' + if isinstance(v, int) or isinstance(v, long): + return unicode(v) + if isinstance(v, float): + if math.isnan(v) or math.isinf(v): + raise ValueError("{0} is not a valid TOML value".format(v)) + else: + return repr(v) + elif isinstance(v, unicode) or isinstance(v, bytes): + return _escape_string(v) + elif isinstance(v, datetime.datetime): + return format_rfc3339(v) + elif isinstance(v, list): + return '[{0}]'.format(', '.join(_format_value(obj) for obj in v)) + elif isinstance(v, dict): + return '{{{0}}}'.format(', '.join('{} = {}'.format(_escape_id(k), _format_value(obj)) for k, obj in v.items())) + else: + raise RuntimeError(v) + + +def dump(obj, fout, sort_keys=False): + tables = [((), obj, False)] + + while tables: + name, table, is_array = tables.pop() + if name: + section_name = '.'.join(_escape_id(c) for c in name) + if is_array: + fout.write('[[{0}]]\n'.format(section_name)) + else: + fout.write('[{0}]\n'.format(section_name)) + + table_keys = sorted(table.keys()) if sort_keys else table.keys() + new_tables = [] + has_kv = False + for k in table_keys: + v = table[k] + if isinstance(v, dict): + new_tables.append((name + (k,), v, False)) + elif isinstance(v, list) and v and all(isinstance(o, dict) for o in v): + new_tables.extend((name + (k,), d, True) for d in v) + elif v is None: + # based on mojombo's comment: https://github.com/toml-lang/toml/issues/146#issuecomment-25019344 + fout.write( + '#{} = null # To use: uncomment and replace null with value\n'.format(_escape_id(k))) + has_kv = True + else: + fout.write('{0} = {1}\n'.format(_escape_id(k), _format_value(v))) + has_kv = True + + tables.extend(reversed(new_tables)) + + if (name or has_kv) and tables: + fout.write('\n') diff --git a/pipenv/vendor/requirementslib/models/pipfile.py b/pipenv/vendor/requirementslib/models/pipfile.py index 84a4a26d..f04f5ba1 100644 --- a/pipenv/vendor/requirementslib/models/pipfile.py +++ b/pipenv/vendor/requirementslib/models/pipfile.py @@ -18,7 +18,7 @@ from ..exceptions import RequirementError from ..utils import is_editable, is_vcs, merge_items from .project import ProjectFile from .requirements import Requirement -from .utils import optional_instance_of +from .utils import optional_instance_of, get_url_name from ..environment import MYPY_RUNNING if MYPY_RUNNING: @@ -87,9 +87,10 @@ def reorder_source_keys(data): sources = data["source"] # type: sources_type for i, entry in enumerate(sources): table = tomlkit.table() # type: Mapping - table["name"] = entry["name"] - table["url"] = entry["url"] - table["verify_ssl"] = entry["verify_ssl"] + source_entry = PipfileLoader.populate_source(entry.copy()) + table["name"] = source_entry["name"] + table["url"] = source_entry["url"] + table["verify_ssl"] = source_entry["verify_ssl"] data["source"][i] = table return data @@ -106,6 +107,18 @@ class PipfileLoader(plette.pipfiles.Pipfile): except Exception: pass + @classmethod + def populate_source(cls, source): + """Derive missing values of source from the existing fields.""" + # Only URL pararemter is mandatory, let the KeyError be thrown. + if "name" not in source: + source["name"] = get_url_name(source["url"]) + if "verify_ssl" not in source: + source["verify_ssl"] = "https://" in source["url"] + if not isinstance(source["verify_ssl"], bool): + source["verify_ssl"] = source["verify_ssl"].lower() == "true" + return source + @classmethod def load(cls, f, encoding=None): # type: (Any, str) -> PipfileLoader diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index 84d13f97..be126e28 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -6,7 +6,11 @@ import collections import copy import hashlib import os +import re +import string +import sys +from distutils.sysconfig import get_python_lib from contextlib import contextmanager from functools import partial @@ -14,6 +18,7 @@ import attr import pep517 import pep517.wrappers import pip_shims +import six import vistir from first import first @@ -24,6 +29,7 @@ from packaging.utils import canonicalize_name from six.moves.urllib import parse as urllib_parse from six.moves.urllib.parse import unquote from vistir.compat import Path +from vistir.contextmanagers import temp_path from vistir.misc import dedup from vistir.path import ( create_tracked_tempdir, @@ -58,6 +64,7 @@ from .utils import ( parse_extras, specs_to_string, split_markers_from_line, + split_ref_from_uri, split_vcs_method_from_uri, validate_path, validate_specifiers, @@ -65,12 +72,16 @@ from .utils import ( normalize_name, create_link, get_pyproject, + convert_direct_url_to_url, + convert_url_to_direct_url, + URL_RE, + DIRECT_URL_RE ) from ..environment import MYPY_RUNNING if MYPY_RUNNING: - from typing import Optional, TypeVar, List, Dict, Union, Any, Tuple, NoReturn + from typing import Optional, TypeVar, List, Dict, Union, Any, Tuple, Generator, Set from pip_shims.shims import Link, InstallRequirement RequirementType = TypeVar('RequirementType', covariant=True, bound=PackagingRequirement) from six.moves.urllib.parse import SplitResult @@ -84,26 +95,29 @@ run = partial(vistir.misc.run, combine_stderr=False, return_object=True, nospin= class Line(object): - def __init__(self, line): + def __init__(self, line, extras=None): # type: (str) -> None - self.editable = line.startswith("-e ") - if self.editable: + self.editable = False # type: bool + if line.startswith("-e "): line = line[len("-e "):] - self.line = line + self.editable = True + self.extras = () # type: Tuple[str] + if extras is not None: + self.extras = tuple(sorted(set(extras))) + self.line = line # type: str self.hashes = [] # type: List[str] - self.extras = [] # type: List[str] self.markers = None # type: Optional[str] self.vcs = None # type: Optional[str] self.path = None # type: Optional[str] self.relpath = None # type: Optional[str] self.uri = None # type: Optional[str] self._link = None # type: Optional[Link] - self.is_local = False - self.name = None # type: Optional[str] - self.specifier = None # type: Optional[str] + self.is_local = False # type: bool + self._name = None # type: Optional[str] + self._specifier = None # type: Optional[str] self.parsed_marker = None # type: Optional[Marker] self.preferred_scheme = None # type: Optional[str] - self.requirement = None # type: Optional[PackagingRequirement] + self._requirement = None # type: Optional[PackagingRequirement] self.is_direct_url = False # type: bool self._parsed_url = None # type: Optional[urllib_parse.ParseResult] self._setup_cfg = None # type: Optional[str] @@ -121,6 +135,25 @@ class Line(object): super(Line, self).__init__() self.parse() + def __hash__(self): + return hash(( + self.editable, self.line, self.markers, tuple(self.extras), + tuple(self.hashes), self.vcs, self.ireq) + ) + + def __repr__(self): + try: + return ( + "".format( + self=self + )) + except Exception: + return "".format(self.__dict__.values()) + @classmethod def split_hashes(cls, line): # type: (str) -> Tuple[str, List[str]] @@ -142,12 +175,61 @@ class Line(object): def line_with_prefix(self): # type: () -> str line = self.line + extras_str = extras_to_string(self.extras) if self.is_direct_url: line = self.link.url + # if self.link.egg_info and self.extras: + # line = "{0}{1}".format(line, extras_str) + elif extras_str: + if self.is_vcs: + line = self.link.url + if "git+file:/" in line and "git+file:///" not in line: + line = line.replace("git+file:/", "git+file:///") + else: + line = "{0}{1}".format(line, extras_str) if self.editable: return "-e {0}".format(line) return line + @property + def line_for_ireq(self): + # type: () -> str + line = "" + if self.is_file or self.is_url and not self.is_vcs: + scheme = self.preferred_scheme if self.preferred_scheme is not None else "uri" + local_line = next(iter([ + os.path.dirname(os.path.abspath(f)) for f in [ + self.setup_py, self.setup_cfg, self.pyproject_toml + ] if f is not None + ]), None) + if local_line and self.extras: + local_line = "{0}{1}".format(local_line, extras_to_string(self.extras)) + line = local_line if local_line is not None else self.line + if scheme == "path": + if not line and self.base_path is not None: + line = os.path.abspath(self.base_path) + else: + if DIRECT_URL_RE.match(self.line): + self._requirement = init_requirement(self.line) + line = convert_direct_url_to_url(self.line) + + if self.editable: + if not line: + if self.is_path or self.is_file: + if not self.path: + line = pip_shims.shims.url_to_path(self.url) + else: + line = self.path + if self.extras: + line = "{0}{1}".format(line, extras_to_string(self.extras)) + else: + line = self.link.url + elif self.is_vcs and not self.editable: + line = add_ssh_scheme_to_git_uri(self.line) + if not line: + line = self.line + return line + @property def base_path(self): # type: () -> Optional[str] @@ -185,6 +267,84 @@ class Line(object): self.populate_setup_paths() return self._pyproject_toml + @property + def specifier(self): + # type: () -> Optional[str] + options = [self._specifier] + for req in (self.ireq, self.requirement): + if req is not None and getattr(req, "specifier", None): + options.append(req.specifier) + specifier = next(iter(spec for spec in options if spec is not None), None) + if specifier is not None: + specifier = specs_to_string(specifier) + elif specifier is None and not self.is_named and self.setup_info is not None: + if self.setup_info.version: + specifier = "=={0}".format(self.setup_info.version) + if specifier: + self._specifier = specifier + return self._specifier + + @property + def specifiers(self): + # type: () -> Optional[SpecifierSet] + ireq_needs_specifier = False + req_needs_specifier = False + if self.ireq is None or self.ireq.req is None or not self.ireq.req.specifier: + ireq_needs_specifier = True + if self.requirement is None or not self.requirement.specifier: + req_needs_specifier = True + if any([ireq_needs_specifier, req_needs_specifier]): + # TODO: Should we include versions for VCS dependencies? IS there a reason not + # to? For now we are using hashes as the equivalent to pin + # note: we need versions for direct dependencies at the very least + if self.is_file or self.is_url or self.is_path or (self.is_vcs and not self.editable): + if self.specifier is not None: + self.specifiers = self.specifier + if self.ireq is not None and self.ireq.req is not None: + return self.ireq.req.specifier + elif self.requirement is not None: + return self.requirement.specifier + return None + + @specifiers.setter + def specifiers(self, specifiers): + # type: (Union[str, SpecifierSet]) -> None + if type(specifiers) is not SpecifierSet: + if type(specifiers) in six.string_types: + specifiers = SpecifierSet(specifiers) + else: + raise TypeError("Must pass a string or a SpecifierSet") + specs = self.get_requirement_specs(specifiers) + if self.ireq is not None and self.ireq.req is not None: + self._ireq.req.specifier = specifiers + self._ireq.req.specs = specs + if self.requirement is not None: + self.requirement.specifier = specifiers + self.requirement.specs = specs + + @classmethod + def get_requirement_specs(cls, specifierset): + # type: (SpecifierSet) -> List[Tuple[str, str]] + specs = [] + spec = next(iter(specifierset._specs), None) + if spec: + specs.append(spec._spec) + return specs + + @property + def requirement(self): + # type: () -> Optional[PackagingRequirement] + if self._requirement is None: + self.parse_requirement() + if self._requirement is None and self._name is not None: + self._requirement = init_requirement(canonicalize_name(self.name)) + if self.is_file or self.is_url and self._requirement is not None: + self._requirement.url = self.url + if self._requirement and self._requirement.specifier and not self._requirement.specs: + specs = self.get_requirement_specs(self._requirement.specifier) + self._requirement.specs = specs + return self._requirement + def populate_setup_paths(self): # type: () -> None if not self.link and not self.path: @@ -201,6 +361,7 @@ class Line(object): @property def pyproject_requires(self): + # type: () -> Optional[List[str]] if self._pyproject_requires is None and self.pyproject_toml is not None: pyproject_requires, pyproject_backend = get_pyproject(self.path) self._pyproject_requires = pyproject_requires @@ -209,11 +370,12 @@ class Line(object): @property def pyproject_backend(self): + # type: () -> Optional[str] if self._pyproject_requires is None and self.pyproject_toml is not None: pyproject_requires, pyproject_backend = get_pyproject(self.path) if not pyproject_backend and self.setup_cfg is not None: setup_dict = SetupInfo.get_setup_cfg(self.setup_cfg) - pyproject_backend = "setuptools.build_meta" + pyproject_backend = "setuptools.build_meta:__legacy__" pyproject_requires = setup_dict.get("build_requires", ["setuptools", "wheel"]) self._pyproject_requires = pyproject_requires @@ -241,23 +403,83 @@ class Line(object): """ extras = None - if "@" in self.line: - parsed = urllib_parse.urlparse(add_ssh_scheme_to_git_uri(self.line)) - if not parsed.scheme: - name, _, line = self.line.partition("@") - name = name.strip() - line = line.strip() - if is_vcs(line) or is_valid_url(line): - self.is_direct_url = True - name, extras = pip_shims.shims._strip_extras(name) - self.name = name - self.line = line + if "@" in self.line or self.is_vcs or self.is_url: + line = "{0}".format(self.line) + match = DIRECT_URL_RE.match(line) + if match is None: + match = URL_RE.match(line) + else: + self.is_direct_url = True + if match is not None: + match_dict = match.groupdict() + name = match_dict.get("name") + extras = match_dict.get("extras") + scheme = match_dict.get("scheme") + host = match_dict.get("host") + path = match_dict.get("path") + ref = match_dict.get("ref") + subdir = match_dict.get("subdirectory") + pathsep = match_dict.get("pathsep", "/") + url = scheme + if host: + url = "{0}{1}".format(url, host) + if path: + url = "{0}{1}{2}".format(url, pathsep, path) + if self.is_vcs and ref: + url = "{0}@{1}".format(url, ref) + if name: + url = "{0}#egg={1}".format(url, name) + if extras: + url = "{0}{1}".format(url, extras) + elif is_file_url(url) and extras and not name and self.editable: + url = "{0}{1}{2}".format(pathsep, path, extras) + if subdir: + url = "{0}&subdirectory={1}".format(url, subdir) + elif extras and not path: + url = "{0}{1}".format(url, extras) + self.line = add_ssh_scheme_to_git_uri(url) + if name: + self._name = name + # line = add_ssh_scheme_to_git_uri(self.line) + # parsed = urllib_parse.urlparse(line) + # if not parsed.scheme and "@" in line: + # matched = URL_RE.match(line) + # if matched is None: + # matched = NAME_RE.match(line) + # if matched: + # name = matched.groupdict().get("name") + # if name is not None: + # self._name = name + # extras = matched.groupdict().get("extras") + # else: + # name, _, line = self.line.partition("@") + # name = name.strip() + # line = line.strip() + # matched = NAME_RE.match(name) + # match_dict = matched.groupdict() + # name = match_dict.get("name") + # extras = match_dict.get("extras") + # if is_vcs(line) or is_valid_url(line): + # self.is_direct_url = True + # # name, extras = pip_shims.shims._strip_extras(name) + # self._name = name + # self.line = line else: self.line, extras = pip_shims.shims._strip_extras(self.line) else: self.line, extras = pip_shims.shims._strip_extras(self.line) if extras is not None: - self.extras = parse_extras(extras) + extras = set(parse_extras(extras)) + if self._name: + self._name, name_extras = pip_shims.shims._strip_extras(self._name) + if name_extras: + name_extras = set(parse_extras(name_extras)) + if extras: + extras |= name_extras + else: + extras = name_extras + if extras is not None: + self.extras = tuple(sorted(extras)) def get_url(self): # type: () -> str @@ -267,31 +489,61 @@ class Line(object): if self.vcs is not None and self.line.startswith("{0}+".format(self.vcs)): _, _, _parseable = self.line.partition("+") parsed = urllib_parse.urlparse(add_ssh_scheme_to_git_uri(_parseable)) + line, _ = split_ref_from_uri(line) else: parsed = urllib_parse.urlparse(add_ssh_scheme_to_git_uri(line)) if "@" in self.line and parsed.scheme == "": name, _, url = self.line.partition("@") - if self.name is None: - self.name = name + if self._name is None: + url = url.strip() + self._name = name.strip() if is_valid_url(url): self.is_direct_url = True line = url.strip() parsed = urllib_parse.urlparse(line) + url_path = parsed.path + if "@" in url_path: + url_path, _, _ = url_path.rpartition("@") + parsed = parsed._replace(path=url_path) self._parsed_url = parsed return line + @property + def name(self): + # type: () -> Optional[str] + if self._name is None: + self.parse_name() + if self._name is None and not self.is_named and not self.is_wheel: + setup_info = self.setup_info + self._name = setup_info.name + return self._name + + @name.setter + def name(self, name): + # type: (str) -> None + self._name = name + self._setup_info.name = name + if self.requirement: + self._requirement.name = name + if self.ireq and self.ireq.req: + self._ireq.req.name = name + @property def url(self): # type: () -> Optional[str] if self.uri is not None: url = add_ssh_scheme_to_git_uri(self.uri) - url = getattr(self.link, "url_without_fragment", None) + else: + url = getattr(self.link, "url_without_fragment", None) if url is not None: url = add_ssh_scheme_to_git_uri(unquote(url)) if url is not None and self._parsed_url is None: if self.vcs is not None: _, _, _parseable = url.partition("+") self._parsed_url = urllib_parse.urlparse(_parseable) + if self.is_vcs: + # strip the ref from the url + url, _ = split_ref_from_uri(url) return url @property @@ -345,30 +597,45 @@ class Line(object): if self.path and ( self.path.startswith(".") or os.path.isabs(self.path) or os.path.exists(self.path) + ) and is_installable_file(self.path): + return True + elif (os.path.exists(self.line) and is_installable_file(self.line)) or ( + os.path.exists(self.get_url()) and is_installable_file(self.get_url()) ): return True - elif os.path.exists(self.line) or os.path.exists(self.get_url()): + return False + + @property + def is_file_url(self): + # type: () -> bool + url = self.get_url() + parsed_url_scheme = self._parsed_url.scheme if self._parsed_url else "" + if url and is_file_url(self.get_url()) or parsed_url_scheme == "file": return True return False @property def is_file(self): # type: () -> bool - if self.is_path or is_file_url(self.get_url()) or (self._parsed_url and self._parsed_url.scheme == "file"): + if self.is_path or ( + is_file_url(self.get_url()) and is_installable_file(self.get_url()) + ) or ( + self._parsed_url and self._parsed_url.scheme == "file" and + is_installable_file(urllib_parse.urlunparse(self._parsed_url)) + ): return True return False @property def is_named(self): # type: () -> bool - return not (self.is_file or self.is_url or self.is_vcs) + return not (self.is_file_url or self.is_url or self.is_file or self.is_vcs) @property def ref(self): # type: () -> Optional[str] - if self._ref is None: - if self.relpath and "@" in self.relpath: - self._relpath, _, self._ref = self.relpath.rpartition("@") + if self._ref is None and self.relpath is not None: + self.relpath, self._ref = split_ref_from_uri(self.relpath) return self._ref @property @@ -381,17 +648,31 @@ class Line(object): @property def is_installable(self): # type: () -> bool - if is_installable_file(self.line) or is_installable_file(self.get_url()) or is_installable_file(self.path) or is_installable_file(self.base_path): - return True - return False + possible_paths = (self.line, self.get_url(), self.path, self.base_path) + return any(is_installable_file(p) for p in possible_paths if p is not None) + + @property + def wheel_kwargs(self): + if not self._wheel_kwargs: + self._wheel_kwargs = _prepare_wheel_building_kwargs(self.ireq) + return self._wheel_kwargs @property def setup_info(self): # type: () -> Optional[SetupInfo] - if self._setup_info is None: + if self._setup_info is None and not self.is_named and not self.is_wheel: self._setup_info = SetupInfo.from_ireq(self.ireq) + if self._setup_info is not None: + self._setup_info.get_info() return self._setup_info + @setup_info.setter + def setup_info(self, setup_info): + # type: (SetupInfo) -> None + self._setup_info = setup_info + if self._parsed_line: + self._parsed_line._setup_info = setup_info + def _get_vcsrepo(self): # type: () -> Optional[VCSRepository] from .vcs import VCSRepository @@ -406,56 +687,90 @@ class Line(object): vcs_type=self.vcs, subdirectory=self.subdirectory, ) - if not self.link.scheme.startswith("file"): + if not ( + self.link.scheme.startswith("file") and + self.editable + ): vcsrepo.obtain() return vcsrepo @property def vcsrepo(self): # type: () -> Optional[VCSRepository] - if self._vcsrepo is None: + if self._vcsrepo is None and self.is_vcs: self._vcsrepo = self._get_vcsrepo() return self._vcsrepo + @vcsrepo.setter + def vcsrepo(self, repo): + # type (VCSRepository) -> None + self._vcsrepo = repo + ireq = self.ireq + wheel_kwargs = self.wheel_kwargs.copy() + wheel_kwargs["src_dir"] = repo.checkout_directory + ireq.source_dir = wheel_kwargs["src_dir"] + build_dir = ireq.build_location(wheel_kwargs["build_dir"]) + ireq._temp_build_dir.path = wheel_kwargs["build_dir"] + with temp_path(): + sys.path = [repo.checkout_directory, "", ".", get_python_lib(plat_specific=0)] + setupinfo = SetupInfo.create( + repo.checkout_directory, ireq=ireq, subdirectory=self.subdirectory, + kwargs=wheel_kwargs + ) + self._setup_info = setupinfo + self._setup_info.reload() + def get_ireq(self): # type: () -> InstallRequirement + line = self.line_for_ireq + if self.editable: + ireq = pip_shims.shims.install_req_from_editable(line) + else: + ireq = pip_shims.shims.install_req_from_line(line) if self.is_named: ireq = pip_shims.shims.install_req_from_line(self.line) - elif (self.is_file or self.is_url) and not self.is_vcs: - line = self.line - if self.is_direct_url: - line = self.link.url - scheme = self.preferred_scheme if self.preferred_scheme is not None else "uri" - local_line = next(iter([ - os.path.dirname(os.path.abspath(f)) for f in [ - self.setup_py, self.setup_cfg, self.pyproject_toml - ] if f is not None - ]), None) - line = local_line if local_line is not None else self.line - if scheme == "path": - if not line and self.base_path is not None: - line = os.path.abspath(self.base_path) - else: - if self.link is not None: - line = self.link.url_without_fragment - else: - if self.uri is not None: - line = self.uri - else: - line = self.path - if self.editable: - ireq = pip_shims.shims.install_req_from_editable(self.link.url) - else: - ireq = pip_shims.shims.install_req_from_line(line) - else: - if self.editable: - ireq = pip_shims.shims.install_req_from_editable(self.link.url) - else: - ireq = pip_shims.shims.install_req_from_line(self.link.url) + if self.is_file or self.is_url: + ireq.link = self.link + # elif (self.is_file or self.is_url) and not self.is_vcs: + # line = self.line + # if self.is_direct_url: + # line = self.link.url + # scheme = self.preferred_scheme if self.preferred_scheme is not None else "uri" + # local_line = next(iter([ + # os.path.dirname(os.path.abspath(f)) for f in [ + # self.setup_py, self.setup_cfg, self.pyproject_toml + # ] if f is not None + # ]), None) + # line = local_line if local_line is not None else self.line + # if scheme == "path": + # if not line and self.base_path is not None: + # line = os.path.abspath(self.base_path) + # else: + # if self.link is not None: + # line = self.link.url_without_fragment + # else: + # if self.uri is not None: + # line = self.uri + # else: + # line = self.path + # if self.editable: + # ireq = pip_shims.shims.install_req_from_editable(self.link.url) + # ireq.link = self.link + # else: + # ireq = pip_shims.shims.install_req_from_line(line) + # else: + # if self.editable: + # ireq = pip_shims.shims.install_req_from_editable(self.link.url) + # ireq.link = self.link + # else: + # ireq = pip_shims.shims.install_req_from_line(self.link.url) + # ireq.link = self.link if self.extras and not ireq.extras: ireq.extras = set(self.extras) if self.parsed_marker is not None and not ireq.markers: ireq.markers = self.parsed_marker + if not ireq.req and self._requirement is not None: + ireq.req = copy.deepcopy(self._requirement) return ireq def parse_ireq(self): @@ -474,12 +789,14 @@ class Line(object): _wheel = Wheel(self.link.filename) name = _wheel.name version = _wheel.version - self.specifier = "=={0}".format(version) + self._specifier = "=={0}".format(version) return name def _parse_name_from_link(self): # type: () -> Optional[str] + if self.link is None: + return None if getattr(self.link, "egg_fragment", None): return self.link.egg_fragment elif self.is_wheel: @@ -491,18 +808,27 @@ class Line(object): if not self.is_named: pass - name = self.line - specifier_match = next( - iter(spec for spec in SPECIFIERS_BY_LENGTH if spec in self.line), None - ) - if specifier_match is not None: - name, specifier_match, version = name.partition(specifier_match) - self.specifier = "{0}{1}".format(specifier_match, version) + try: + self._requirement = init_requirement(self.line) + except Exception: + raise RequirementError("Failed parsing requirement from {0!r}".format(self.line)) + name = self._requirement.name + self._specifier = specs_to_string(self._requirement.specifier) + if self._requirement.extras and not self.extras: + self.extras = self._requirement.extras + if not name: + name = self.line + specifier_match = next( + iter(spec for spec in SPECIFIERS_BY_LENGTH if spec in self.line), None + ) + if specifier_match is not None: + name, specifier_match, version = name.partition(specifier_match) + self._specifier = "{0}{1}".format(specifier_match, version) return name def parse_name(self): # type: () -> None - if self.name is None: + if self._name is None: name = None if self.link is not None: name = self._parse_name_from_link() @@ -514,97 +840,161 @@ class Line(object): if "&" in name: # subdirectory fragments might also be in here name, _, _ = name.partition("&") - if self.is_named and name is None: + if self.is_named: name = self._parse_name_from_line() if name is not None: name, extras = pip_shims.shims._strip_extras(name) if extras is not None and not self.extras: - self.extras = parse_extras(extras) - self.name = name + self.extras = tuple(sorted(set(parse_extras(extras)))) + self._name = name def _parse_requirement_from_vcs(self): # type: () -> Optional[PackagingRequirement] - name = self.name if self.name else self.link.egg_fragment - url = self.uri if self.uri else unquote(self.link.url) - if self.is_direct_url: - url = self.link.url - if not name: - raise ValueError( - "pipenv requires an #egg fragment for version controlled " - "dependencies. Please install remote dependency " - "in the form {0}#egg=.".format(url) - ) - req = init_requirement(canonicalize_name(name)) # type: PackagingRequirement - req.editable = self.editable - if not getattr(req, "url") and self.link: - req.url = url - req.line = self.link.url + # name = self._name if self._name else self.link.egg_fragment + # url = self.url if self.url else self.uri + # if self.is_direct_url: + # url = self.link.url + # if not name: + # raise ValueError( + # "pipenv requires an #egg fragment for version controlled " + # "dependencies. Please install remote dependency " + # "in the form {0}#egg=.".format(url) + # ) + # req = init_requirement(canonicalize_name(name)) # type: PackagingRequirement + # req.editable = self.editable + # if not getattr(req, "url") and self.link: + # req.url = url + # req.line = self.link.url if ( - self.uri != unquote(self.link.url_without_fragment) - and "git+ssh://" in self.link.url + self.uri != unquote(self.url) + and "git+ssh://" in self.url and (self.uri is not None and "git+git@" in self.uri) ): - req.line = self.uri - req.url = self.uri + self._requirement.line = self.uri + self._requirement.url = self.url + self._requirement.link = create_link(build_vcs_uri( + vcs=self.vcs, + uri=self.url, + ref=self.ref, + subdirectory=self.subdirectory, + extras=self.extras, + name=self.name + )) + # else: + # req.link = self.link if self.ref: if self._vcsrepo is not None: - req.revision = self._vcsrepo.get_commit_hash() + self._requirement.revision = self._vcsrepo.get_commit_hash() else: - req.revision = self.ref - if self.extras: - req.extras = self.extras - req.vcs = self.vcs - req.link = self.link - if self.path and self.link and self.link.scheme.startswith("file"): - req.local_file = True - req.path = self.path - return req + self._requirement.revision = self.ref + # if self.extras: + # req.extras = self.extras + # req.vcs = self.vcs + # if self.path and self.link and self.link.scheme.startswith("file"): + # req.local_file = True + # req.path = self.path + return self._requirement def parse_requirement(self): # type: () -> None - if self.name is None: + if self._name is None: self.parse_name() - if self.is_named: - self.requirement = init_requirement(self.line) - elif self.is_vcs: - self.requirement = self._parse_requirement_from_vcs() - if self.name is None and ( - self.requirement is not None and self.requirement.name is not None - ): - self.name = self.requirement.name - if self.name is not None and self.requirement is None: - self.requirement = init_requirement(self.name) - if self.requirement: + if not self._name and not self.is_vcs and not self.is_named: + setup_info = self.setup_info + self._name = setup_info.name + name, extras, url = self.requirement_info + if name: + self._requirement = init_requirement(name) # type: PackagingRequirement + if extras: + self._requirement.extras = set(extras) + if url: + self._requirement.url = url + if self.is_direct_url: + url = self.link.url + if self.link: + self._requirement.link = self.link + self._requirement.editable = self.editable + if self.path and self.link and self.link.scheme.startswith("file"): + self._requirement.local_file = True + self._requirement.path = self.path + if self.is_vcs: + self._requirement.vcs = self.vcs + self._requirement.line = self.link.url + self._parse_requirement_from_vcs() + else: + self._requirement.line = self.line if self.parsed_marker is not None: - self.requirement.marker = self.parsed_marker - if self.is_url or self.is_file and (self.link or self.url) and not self.is_vcs: - if self.uri: - self.requirement.url = self.uri - elif self.link: - self.requirement.url = unquote(self.link.url_without_fragment) - else: - self.requirement.url = self.url - if self.extras and not self.requirement.extras: - self.requirement.extras = set(self.extras) + self._requirement.marker = self.parsed_marker + if self.specifiers: + self._requirement.specifier = self.specifiers + specs = [] + spec = next(iter(s for s in self.specifiers._specs), None) + if spec: + specs.append(spec._spec) + self._requirement.spec = spec + else: + if self.is_vcs: + raise ValueError( + "pipenv requires an #egg fragment for version controlled " + "dependencies. Please install remote dependency " + "in the form {0}#egg=.".format(url) + ) + # if self.is_named: + # self._requirement = init_requirement(self.line) + # elif self.is_vcs: + # self._requirement = self._parse_requirement_from_vcs() + # if self._name is None and ( + # self._requirement is not None and self._requirement.name is not None + # ): + # self.name = self._requirement.name + # if self._name is not None and self._requirement is None: + # self._requirement = init_requirement(self._name) + # if self._requirement: + # if self.parsed_marker is not None: + # self._requirement.marker = self.parsed_marker + # if self.is_url or self.is_file and (self.link or self.url) and not self.is_vcs: + # if self.uri: + # self._requirement.url = self.url + # elif self.link: + # self._requirement.url = unquote(self.link.url_without_fragment) + # else: + # self._requirement.url = self.uri + # if self.extras and not self._requirement.extras: + # self._requirement.extras = set(self.extras) def parse_link(self): # type: () -> None if self.is_file or self.is_url or self.is_vcs: vcs, prefer, relpath, path, uri, link = FileRequirement.get_link_from_line(self.line) ref = None - if link is not None and "@" in link.path and uri is not None: - uri, _, ref = uri.rpartition("@") + if link is not None and "@" in unquote(link.path) and uri is not None: + uri, _, ref = unquote(uri).rpartition("@") if relpath is not None and "@" in relpath: relpath, _, ref = relpath.rpartition("@") + if path is not None and "@" in path: + path, _ = split_ref_from_uri(path) + link_url = link.url_without_fragment + if "@" in link_url: + link_url, _ = split_ref_from_uri(link_url) self._ref = ref self.vcs = vcs self.preferred_scheme = prefer self.relpath = relpath self.path = path self.uri = uri - if self.is_direct_url and self.name is not None: + if link.egg_fragment: + name, extras = pip_shims.shims._strip_extras(link.egg_fragment) + self.extras = tuple(sorted(set(parse_extras(extras)))) + self._name = name + else: + # set this so we can call `self.name` without a recursion error + self._link = link + if (self.is_direct_url or vcs) and self.name is not None and vcs is not None: self._link = create_link( - build_vcs_uri(vcs=vcs, uri=uri, ref=ref, extras=self.extras, name=self.name) + build_vcs_uri(vcs=vcs, uri=link_url, ref=ref, + extras=self.extras, name=self.name, + subdirectory=link.subdirectory_fragment + ) ) else: self._link = link @@ -615,6 +1005,72 @@ class Line(object): markers = PackagingRequirement("fakepkg; {0}".format(self.markers)).marker self.parsed_marker = markers + @property + def requirement_info(self): + # type: () -> Tuple(Optional[str], Tuple[Optional[str]], Optional[str]) + """ + Generates a 3-tuple of the requisite *name*, *extras* and *url* to generate a + :class:`~packaging.requirements.Requirement` out of. + + :return: A Tuple containing an optional name, a Tuple of extras names, and an optional URL. + :rtype: Tuple[Optional[str], Tuple[Optional[str]], Optional[str]] + """ + + # Direct URLs can be converted to packaging requirements directly, but + # only if they are `file://` (with only two slashes) + name = None + extras = () + url = None + # if self.is_direct_url: + if self._name: + name = canonicalize_name(self._name) + if self.is_file or self.is_url or self.is_path or self.is_file_url or self.is_vcs: + url = "" + if self.is_vcs: + url = self.url if self.url else self.uri + if self.is_direct_url: + url = self.link.url_without_fragment + else: + if self.link: + url = self.link.url_without_fragment + elif self.url: + url = self.url + if self.ref: + url = "{0}@{1}".format(url, self.ref) + else: + url = self.uri + if self.link and name is None: + self._name = self.link.egg_fragment + if self._name: + name = canonicalize_name(self._name) + # return "{0}{1}@ {2}".format( + # normalize_name(self.name), extras_to_string(self.extras), url + # ) + return (name, extras, url) + + @property + def line_is_installable(self): + # type: () -> bool + """ + This is a safeguard against decoy requirements when a user installs a package + whose name coincides with the name of a folder in the cwd, e.g. install *alembic* + when there is a folder called *alembic* in the working directory. + + In this case we first need to check that the given requirement is a valid + URL, VCS requirement, or installable filesystem path before deciding to treat it as + a file requirement over a named requirement. + """ + line = self.line + if is_file_url(line): + link = create_link(line) + line = link.url_without_fragment + line, _ = split_ref_from_uri(line) + if (is_vcs(line) or (is_valid_url(line) and ( + not is_file_url(line) or is_installable_file(line))) + or is_installable_file(line)): + return True + return False + def parse(self): # type: () -> None self.parse_hashes() @@ -624,20 +1080,26 @@ class Line(object): if self.line.startswith("git+file:/") and not self.line.startswith("git+file:///"): self.line = self.line.replace("git+file:/", "git+file:///") self.parse_markers() - if self.is_file: - self.populate_setup_paths() + if self.is_file_url: + if self.line_is_installable: + self.populate_setup_paths() + else: + raise RequirementError( + "Supplied requirement is not installable: {0!r}".format(self.line) + ) self.parse_link() self.parse_requirement() self.parse_ireq() -@attr.s(slots=True) +@attr.s(slots=True, hash=True) class NamedRequirement(object): name = attr.ib() # type: str - version = attr.ib(validator=attr.validators.optional(validate_specifiers)) # type: Optional[str] + version = attr.ib() # type: Optional[str] req = attr.ib() # type: PackagingRequirement - extras = attr.ib(default=attr.Factory(list)) # type: List[str] + extras = attr.ib(default=attr.Factory(list)) # type: Tuple[str] editable = attr.ib(default=False) # type: bool + _parsed_line = attr.ib(default=None) # type: Optional[Line] @req.default def get_requirement(self): @@ -647,9 +1109,16 @@ class NamedRequirement(object): ) return req + @property + def parsed_line(self): + # type: () -> Optional[Line] + if self._parsed_line is None: + self._parsed_line = Line(self.line_part) + return self._parsed_line + @classmethod - def from_line(cls, line): - # type: (str) -> NamedRequirement + def from_line(cls, line, parsed_line=None): + # type: (str, Optional[Line]) -> NamedRequirement req = init_requirement(line) specifiers = None # type: Optional[str] if req.specifier: @@ -662,11 +1131,18 @@ class NamedRequirement(object): if not name: name = getattr(req, "key", line) req.name = name - extras = None # type: Optional[List[str]] + creation_kwargs = { + "name": name, + "version": specifiers, + "req": req, + "parsed_line": parsed_line, + "extras": None + } + extras = None # type: Optional[Tuple[str]] if req.extras: extras = list(req.extras) - return cls(name=name, version=specifiers, req=req, extras=extras) - return cls(name=name, version=specifiers, req=req) + creation_kwargs["extras"] = extras + return cls(**creation_kwargs) @classmethod def from_pipfile(cls, name, pipfile): @@ -691,7 +1167,7 @@ class NamedRequirement(object): # FIXME: This should actually be canonicalized but for now we have to # simply lowercase it and replace underscores, since full canonicalization # also replaces dots and that doesn't actually work when querying the index - return "{0}".format(normalize_name(self.name)) + return normalize_name(self.name) @property def pipfile_part(self): @@ -699,6 +1175,8 @@ class NamedRequirement(object): pipfile_dict = attr.asdict(self, filter=filter_none).copy() # type: ignore if "version" not in pipfile_dict: pipfile_dict["version"] = "*" + if "_parsed_line" in pipfile_dict: + pipfile_dict.pop("_parsed_line") name = pipfile_dict.pop("name") return {name: pipfile_dict} @@ -708,37 +1186,38 @@ LinkInfo = collections.namedtuple( ) -@attr.s(slots=True) +@attr.s(slots=True, cmp=True, hash=True) class FileRequirement(object): """File requirements for tar.gz installable files or wheels or setup.py containing directories.""" #: Path to the relevant `setup.py` location - setup_path = attr.ib(default=None) # type: Optional[str] + setup_path = attr.ib(default=None, cmp=True) # type: Optional[str] #: path to hit - without any of the VCS prefixes (like git+ / http+ / etc) - path = attr.ib(default=None) # type: Optional[str] + path = attr.ib(default=None, cmp=True) # type: Optional[str] #: Whether the package is editable - editable = attr.ib(default=False) # type: bool + editable = attr.ib(default=False, cmp=True) # type: bool #: Extras if applicable - extras = attr.ib(default=attr.Factory(list)) # type: List[str] - _uri_scheme = attr.ib(default=None) # type: Optional[str] + extras = attr.ib(default=attr.Factory(tuple), cmp=True) # type: Tuple[str] + _uri_scheme = attr.ib(default=None, cmp=True) # type: Optional[str] #: URI of the package - uri = attr.ib() # type: Optional[str] + uri = attr.ib(cmp=True) # type: Optional[str] #: Link object representing the package to clone - link = attr.ib() # type: Optional[Link] + link = attr.ib(cmp=True) # type: Optional[Link] #: PyProject Requirements - pyproject_requires = attr.ib(default=attr.Factory(list)) # type: List + pyproject_requires = attr.ib(default=attr.Factory(tuple), cmp=True) # type: Tuple #: PyProject Build System - pyproject_backend = attr.ib(default=None) # type: Optional[str] + pyproject_backend = attr.ib(default=None, cmp=True) # type: Optional[str] #: PyProject Path - pyproject_path = attr.ib(default=None) # type: Optional[str] + pyproject_path = attr.ib(default=None, cmp=True) # type: Optional[str] #: Setup metadata e.g. dependencies - setup_info = attr.ib(default=None) # type: SetupInfo - _has_hashed_name = attr.ib(default=False) # type: bool + _setup_info = attr.ib(default=None, cmp=True) # type: Optional[SetupInfo] + _has_hashed_name = attr.ib(default=False, cmp=True) # type: bool + _parsed_line = attr.ib(default=None, cmp=False, hash=True) # type: Optional[Line] #: Package name - name = attr.ib() # type: Optional[str] + name = attr.ib(cmp=True) # type: Optional[str] #: A :class:`~pkg_resources.Requirement` isntance - req = attr.ib() # type: Optional[PackagingRequirement] + req = attr.ib(cmp=True) # type: Optional[PackagingRequirement] @classmethod def get_link_from_line(cls, line): @@ -861,11 +1340,42 @@ class FileRequirement(object): setup_deps.extend(setup_info.get("setup_requires", [])) build_deps.extend(setup_info.get("build_requires", [])) if self.pyproject_requires: - build_deps.extend(self.pyproject_requires) + build_deps.extend(list(self.pyproject_requires)) setup_deps = list(set(setup_deps)) build_deps = list(set(build_deps)) return deps, setup_deps, build_deps + def __attrs_post_init__(self): + if self.name is None and self.parsed_line: + if self.parsed_line.setup_info: + self._setup_info = self.parsed_line.setup_info + if self.parsed_line.setup_info.name: + self.name = self.parsed_line.setup_info.name + if self.req is None and self._parsed_line.requirement is not None: + self.req = self._parsed_line.requirement + if self._parsed_line and self._parsed_line.ireq and not self._parsed_line.ireq.req: + if self.req is not None: + self._parsed_line._ireq.req = self.req + + @property + def setup_info(self): + from .setup_info import SetupInfo + if self._setup_info is None and self.parsed_line: + if self.parsed_line.setup_info: + self._setup_info = self.parsed_line.setup_info + elif self.parsed_line.ireq and not self.parsed_line.is_wheel: + self._setup_info = SetupInfo.from_ireq(self.parsed_line.ireq) + else: + if self.link and not self.link.is_wheel: + self._setup_info = Line(self.line_part).setup_info + return self._setup_info + + @setup_info.setter + def setup_info(self, setup_info): + self._setup_info = setup_info + if self._parsed_line: + self._parsed_line._setup_info = setup_info + @uri.default def get_uri(self): # type: () -> str @@ -901,12 +1411,18 @@ class FileRequirement(object): )): _ireq = None if self.editable: - line = pip_shims.shims.path_to_url(self.setup_py_dir) + if self.setup_path: + line = pip_shims.shims.path_to_url(self.setup_py_dir) + else: + line = pip_shims.shims.path_to_url(os.path.abspath(self.path)) if self.extras: line = "{0}[{1}]".format(line, ",".join(self.extras)) _ireq = pip_shims.shims.install_req_from_editable(line) else: - line = Path(self.setup_py_dir).as_posix() + if self.setup_path: + line = Path(self.setup_py_dir).as_posix() + else: + line = Path(os.path.abspath(self.path)).as_posix() if self.extras: line = "{0}[{1}]".format(line, ",".join(self.extras)) _ireq = pip_shims.shims.install_req_from_line(line) @@ -916,13 +1432,13 @@ class FileRequirement(object): _ireq.extras = set(self.extras) from .setup_info import SetupInfo subdir = getattr(self, "subdirectory", None) - setupinfo = None if self.setup_info is not None: setupinfo = self.setup_info else: setupinfo = SetupInfo.from_ireq(_ireq, subdir=subdir) if setupinfo: - self.setup_info = setupinfo + self._setup_info = setupinfo + self.setup_info.get_info() setupinfo_dict = setupinfo.as_dict() setup_name = setupinfo_dict.get("name", None) if setup_name: @@ -931,7 +1447,7 @@ class FileRequirement(object): build_requires = setupinfo_dict.get("build_requires") build_backend = setupinfo_dict.get("build_backend") if build_requires and not self.pyproject_requires: - self.pyproject_requires = build_requires + self.pyproject_requires = tuple(build_requires) if build_backend and not self.pyproject_backend: self.pyproject_backend = build_backend if not name or name.lower() == "unknown": @@ -957,7 +1473,21 @@ class FileRequirement(object): def get_requirement(self): # type: () -> PackagingRequirement if self.name is None: - raise ValueError("Failed to generate a requirement: missing name for {0!r}".format(self)) + if self._parsed_line is not None and self._parsed_line.name is not None: + self.name = self._parsed_line.name + else: + raise ValueError( + "Failed to generate a requirement: missing name for {0!r}".format(self) + ) + if self._parsed_line: + try: + # initialize specifiers to make sure we capture them + self._parsed_line.specifiers + except Exception: + pass + req = copy.deepcopy(self._parsed_line.requirement) + return req + req = init_requirement(normalize_name(self.name)) req.editable = False if self.link is not None: @@ -982,6 +1512,13 @@ class FileRequirement(object): req.link = self.link return req + @property + def parsed_line(self): + # type: () -> Optional[Line] + if self._parsed_line is None: + self._parsed_line = Line(self.line_part) + return self._parsed_line + @property def is_local(self): # type: () -> bool @@ -1014,6 +1551,8 @@ class FileRequirement(object): @property def is_direct_url(self): # type: () -> bool + if self._parsed_line is not None and self._parsed_line.is_direct_url: + return True return self.is_remote_artifact @property @@ -1032,7 +1571,7 @@ class FileRequirement(object): path=None, # type: Optional[str] uri=None, # type: str editable=False, # type: bool - extras=None, # type: Optional[List[str]] + extras=None, # type: Optional[Tuple[str]] link=None, # type: Link vcs_type=None, # type: Optional[Any] name=None, # type: Optional[str] @@ -1041,10 +1580,10 @@ class FileRequirement(object): uri_scheme=None, # type: str setup_path=None, # type: Optional[Any] relpath=None, # type: Optional[Any] + parsed_line=None, # type: Optional[Line] ): # type: (...) -> FileRequirement - parsed_line = None - if line: + if parsed_line is None and line is not None: parsed_line = Line(line) if relpath and not path: path = relpath @@ -1065,14 +1604,15 @@ class FileRequirement(object): if not uri: uri = unquote(link.url_without_fragment) if not extras: - extras = [] + extras = () pyproject_path = None + pyproject_requires = None + pyproject_backend = None if path is not None: pyproject_requires = get_pyproject(path) - pyproject_backend = None - pyproject_requires = None if pyproject_requires is not None: pyproject_requires, pyproject_backend = pyproject_requires + pyproject_requires = tuple(pyproject_requires) if path: setup_paths = get_setup_paths(path) if setup_paths["pyproject_toml"] is not None: @@ -1092,6 +1632,7 @@ class FileRequirement(object): "pyproject_requires": pyproject_requires, "pyproject_backend": pyproject_backend, "path": path or relpath, + "parsed_line": parsed_line } if vcs_type: creation_kwargs["vcs"] = vcs_type @@ -1099,19 +1640,25 @@ class FileRequirement(object): creation_kwargs["name"] = name _line = None ireq = None + setup_info = None + if parsed_line: + if parsed_line.name: + name = parsed_line.name + if parsed_line.setup_info: + name = parsed_line.setup_info.as_dict().get("name", name) if not name or not parsed_line: if link is not None and link.url is not None: _line = unquote(link.url_without_fragment) if name: _line = "{0}#egg={1}".format(_line, name) - # if extras: - # _line = "{0}[{1}]".format(_line, ",".join(sorted(set(extras)))) + if extras and extras_to_string(extras) not in _line: + _line = "{0}[{1}]".format(_line, ",".join(sorted(set(extras)))) elif uri is not None: - _line = uri + _line = unquote(uri) else: - _line = line + _line = unquote(line) if editable: - if extras and ( + if extras and extras_to_string(extras) not in _line and ( (link and link.scheme == "file") or (uri and uri.startswith("file")) or (not uri and not link) ): @@ -1120,31 +1667,32 @@ class FileRequirement(object): ireq = pip_shims.shims.install_req_from_editable(_line) else: _line = path if (uri_scheme and uri_scheme == "path") else _line - if extras: + if extras and extras_to_string(extras) not in _line: _line = "{0}[{1}]".format(_line, ",".join(sorted(set(extras)))) if ireq is None: ireq = pip_shims.shims.install_req_from_line(_line) - if parsed_line is None: if editable: _line = "-e {0}".format(editable) - parsed_line = Line(_line) + parsed_line = Line(_line) if ireq is None: ireq = parsed_line.ireq if extras and not ireq.extras: ireq.extras = set(extras) - if not ireq.is_wheel: + if setup_info is None: setup_info = SetupInfo.from_ireq(ireq) - setupinfo_dict = setup_info.as_dict() - setup_name = setupinfo_dict.get("name", None) - if setup_name: - name = setup_name - build_requires = setupinfo_dict.get("build_requires", []) - build_backend = setupinfo_dict.get("build_backend", []) - if not creation_kwargs.get("pyproject_requires") and build_requires: - creation_kwargs["pyproject_requires"] = build_requires - if not creation_kwargs.get("pyproject_backend") and build_backend: - creation_kwargs["pyproject_backend"] = build_backend - creation_kwargs["setup_info"] = setup_info + setupinfo_dict = setup_info.as_dict() + setup_name = setupinfo_dict.get("name", None) + if setup_name: + name = setup_name + build_requires = setupinfo_dict.get("build_requires", ()) + build_backend = setupinfo_dict.get("build_backend", ()) + if not creation_kwargs.get("pyproject_requires") and build_requires: + creation_kwargs["pyproject_requires"] = tuple(build_requires) + if not creation_kwargs.get("pyproject_backend") and build_backend: + creation_kwargs["pyproject_backend"] = build_backend + if setup_info is None and parsed_line and parsed_line.setup_info: + setup_info = parsed_line.setup_info + creation_kwargs["setup_info"] = setup_info if path or relpath: creation_kwargs["path"] = relpath if relpath else path if req is not None: @@ -1154,17 +1702,24 @@ class FileRequirement(object): creation_req_line = getattr(creation_req, "line", None) if creation_req_line is None and line is not None: creation_kwargs["req"].line = line # type: ignore - if parsed_line.name: + if parsed_line and parsed_line.name: if name and len(parsed_line.name) != 7 and len(name) == 7: name = parsed_line.name if name: creation_kwargs["name"] = name cls_inst = cls(**creation_kwargs) # type: ignore + if parsed_line and not cls_inst._parsed_line: + cls_inst._parsed_line = parsed_line + if not cls_inst._parsed_line: + cls_inst._parsed_line = Line(cls_inst.line_part) + if cls_inst._parsed_line and cls_inst.parsed_line.ireq and not cls_inst.parsed_line.ireq.req: + if cls_inst.req: + cls_inst._parsed_line._ireq.req = cls_inst.req return cls_inst @classmethod - def from_line(cls, line, extras=None): - # type: (str, Optional[List[str]]) -> FileRequirement + def from_line(cls, line, extras=None, parsed_line=None): + # type: (str, Optional[Tuple[str]], Optional[Line]) -> FileRequirement line = line.strip('"').strip("'") link = None path = None @@ -1174,7 +1729,9 @@ class FileRequirement(object): name = None req = None if not extras: - extras = [] + extras = () + else: + extras = tuple(extras) if not any([is_installable_file(line), is_valid_url(line), is_file_url(line)]): try: req = init_requirement(line) @@ -1199,6 +1756,8 @@ class FileRequirement(object): } if req is not None: arg_dict["req"] = req + if parsed_line is not None: + arg_dict["parsed_line"] = parsed_line if link and link.is_wheel: from pip_shims import Wheel @@ -1251,21 +1810,34 @@ class FileRequirement(object): "editable": pipfile.get("editable", False), "link": link, "uri_scheme": uri_scheme, - "extras": pipfile.get("extras", None) + "extras": pipfile.get("extras", None), } - extras = pipfile.get("extras", []) + extras = pipfile.get("extras", ()) + if extras: + extras = tuple(extras) line = "" - if name: + if pipfile.get("editable", False) and uri_scheme == "path": + line = "{0}".format(path) if extras: - line_name = "{0}[{1}]".format(name, ",".join(sorted(set(extras)))) - else: - line_name = "{0}".format(name) - line = "{0}@ {1}".format(line_name, link.url_without_fragment) + line = "{0}{1}".format(line, extras_to_string(extras)) else: - line = link.url + if name: + if extras: + line_name = "{0}{1}".format(name, extras_to_string(extras)) + else: + line_name = "{0}".format(name) + line = "{0}#egg={1}".format(unquote(link.url_without_fragment), line_name) + else: + line = unquote(link.url) + if extras: + line = "{0}{1}".format(line, extras_to_string(extras)) + if "subdirectory" in pipfile: + arg_dict["subdirectory"] = pipfile["subdirectory"] + line = "{0}&subdirectory={1}".format(pipfile["subdirectory"]) if pipfile.get("editable", False): line = "-e {0}".format(line) + arg_dict["line"] = line return cls.create(**arg_dict) @property @@ -1295,7 +1867,7 @@ class FileRequirement(object): # type: () -> Dict[str, Dict[str, Any]] excludes = [ "_base_line", "_has_hashed_name", "setup_path", "pyproject_path", "_uri_scheme", - "pyproject_requires", "pyproject_backend", "setup_info", "_parsed_line" + "pyproject_requires", "pyproject_backend", "_setup_info", "_parsed_line" ] filter_func = lambda k, v: bool(v) is True and k.name not in excludes # noqa pipfile_dict = attr.asdict(self, filter=filter_func).copy() @@ -1341,7 +1913,7 @@ class FileRequirement(object): return {name: pipfile_dict} -@attr.s(slots=True) +@attr.s(slots=True, hash=True) class VCSRequirement(FileRequirement): #: Whether the repository is editable editable = attr.ib(default=None) # type: Optional[bool] @@ -1357,12 +1929,12 @@ class VCSRequirement(FileRequirement): subdirectory = attr.ib(default=None) # type: Optional[str] _repo = attr.ib(default=None) # type: Optional['VCSRepository'] _base_line = attr.ib(default=None) # type: Optional[str] - _parsed_line = attr.ib(default=None) # type: Optional[Line] name = attr.ib() link = attr.ib() req = attr.ib() def __attrs_post_init__(self): + # type: () -> None if not self.uri: if self.path: self.uri = pip_shims.shims.path_to_url(self.path) @@ -1375,9 +1947,14 @@ class VCSRequirement(FileRequirement): new_uri = urllib_parse.urlunsplit((scheme,) + rest[:-1] + ("",)) new_uri = "{0}{1}".format(vcs_type, new_uri) self.uri = new_uri + if self.req and ( + self.parsed_line.ireq and not self.parsed_line.ireq.req + ): + self.parsed_line._ireq.req = self.req @link.default def get_link(self): + # type: () -> pip_shims.shims.Link uri = self.uri if self.uri else pip_shims.shims.path_to_url(self.path) vcs_uri = build_vcs_uri( self.vcs, @@ -1391,6 +1968,7 @@ class VCSRequirement(FileRequirement): @name.default def get_name(self): + # type: () -> Optional[str] return ( self.link.egg_fragment or self.req.name if getattr(self, "req", None) @@ -1399,13 +1977,34 @@ class VCSRequirement(FileRequirement): @property def vcs_uri(self): + # type: () -> Optional[str] uri = self.uri if not any(uri.startswith("{0}+".format(vcs)) for vcs in VCS_LIST): uri = "{0}+{1}".format(self.vcs, uri) return uri + @property + def setup_info(self): + if self._repo: + from .setup_info import SetupInfo + self._setup_info = SetupInfo.from_ireq(Line(self._repo.checkout_directory).ireq) + return self._setup_info + if self._parsed_line and self._parsed_line.setup_info: + return self._parsed_line.setup_info + ireq = self.parsed_line.ireq + from .setup_info import SetupInfo + self._setup_info = SetupInfo.from_ireq(ireq) + return self._setup_info + + @setup_info.setter + def setup_info(self, setup_info): + self._setup_info = setup_info + if self._parsed_line: + self._parsed_line.setup_info = setup_info + @req.default def get_requirement(self): + # type: () -> PackagingRequirement name = self.name or self.link.egg_fragment url = None if self.uri: @@ -1459,6 +2058,8 @@ class VCSRequirement(FileRequirement): # type: () -> VCSRepository if self._repo is None: self._repo = self.get_vcs_repo() + if self._parsed_line: + self._parsed_line.vcsrepo = self._repo return self._repo def get_checkout_dir(self, src_dir=None): @@ -1504,16 +2105,18 @@ class VCSRequirement(FileRequirement): pyproject_info = get_pyproject(checkout_dir) if pyproject_info is not None: pyproject_requires, pyproject_backend = pyproject_info - self.pyproject_requires = pyproject_requires + self.pyproject_requires = tuple(pyproject_requires) self.pyproject_backend = pyproject_backend return vcsrepo def get_commit_hash(self): + # type: () -> str hash_ = None hash_ = self.repo.get_commit_hash() return hash_ def update_repo(self, src_dir=None, ref=None): + # type: (Optional[str], Optional[str]) -> str if ref: self.ref = ref else: @@ -1528,23 +2131,48 @@ class VCSRequirement(FileRequirement): @contextmanager def locked_vcs_repo(self, src_dir=None): + # type: (Optional[str]) -> Generator[VCSRepository, None, None] if not src_dir: src_dir = create_tracked_tempdir(prefix="requirementslib-", suffix="-src") vcsrepo = self.get_vcs_repo(src_dir=src_dir) - self.req.revision = vcsrepo.get_commit_hash() + if not self.req: + if self.parsed_line is not None: + self.req = self.parsed_line.requirement + else: + self.req = self.get_requirement() + revision = self.req.revision = vcsrepo.get_commit_hash() # Remove potential ref in the end of uri after ref is parsed if "@" in self.link.show_url and "@" in self.uri: - uri, ref = self.uri.rsplit("@", 1) - checkout = self.req.revision - if checkout and ref in checkout: + uri, ref = split_ref_from_uri(self.uri) + checkout = revision + if checkout and ref and ref in checkout: self.uri = uri - - yield vcsrepo + orig_repo = self._repo self._repo = vcsrepo + if self._parsed_line: + self._parsed_line.vcsrepo = vcsrepo + if self._setup_info: + self._setup_info._requirements = () + self._setup_info._extras_requirements = () + self._setup_info.build_requires = () + self._setup_info.setup_requires = () + self._setup_info.version = None + self._setup_info.metadata = None + if self.parsed_line: + self._parsed_line.vcsrepo = vcsrepo + # self._parsed_line._specifier = "=={0}".format(self.setup_info.version) + # self._parsed_line.specifiers = self._parsed_line._specifier + if self.req: + self.req.specifier = SpecifierSet("=={0}".format(self.setup_info.version)) + try: + yield vcsrepo + finally: + self._repo = orig_repo @classmethod def from_pipfile(cls, name, pipfile): + # type: (str, Dict[str, Union[List[str], str, bool]]) -> VCSRequirement creation_args = {} pipfile_keys = [ k @@ -1587,12 +2215,30 @@ class VCSRequirement(FileRequirement): else: creation_args[key] = pipfile.get(key) creation_args["name"] = name - return cls(**creation_args) + cls_inst = cls(**creation_args) + if cls_inst._parsed_line is None: + vcs_uri = build_vcs_uri( + vcs=cls_inst.vcs, uri=add_ssh_scheme_to_git_uri(cls_inst.uri), + name=cls_inst.name, ref=cls_inst.ref, subdirectory=cls_inst.subdirectory, + extras=cls_inst.extras + ) + if cls_inst.editable: + vcs_uri = "-e {0}".format(vcs_uri) + cls_inst._parsed_line = Line(vcs_uri) + if not cls_inst.name and cls_inst._parsed_line.name: + cls_inst.name = cls_inst._parsed_line.name + if cls_inst.req and ( + cls_inst._parsed_line.ireq and not cls_inst.parsed_line.ireq.req + ): + cls_inst._parsed_line.ireq.req = cls_inst.req + return cls_inst @classmethod - def from_line(cls, line, editable=None, extras=None): + def from_line(cls, line, editable=None, extras=None, parsed_line=None): + # type: (str, Optional[bool], Optional[Tuple[str]], Optional[Line]) -> VCSRequirement relpath = None - parsed_line = Line(line) + if parsed_line is None: + parsed_line = Line(line) if editable: parsed_line.editable = editable if extras: @@ -1618,16 +2264,18 @@ class VCSRequirement(FileRequirement): extras = parse_extras(extras) else: line, extras = pip_shims.shims._strip_extras(line) + if extras: + extras = tuple(extras) subdirectory = link.subdirectory_fragment ref = None - if "@" in link.path and "@" in uri: - uri, _, ref = uri.rpartition("@") + if uri: + uri, ref = split_ref_from_uri(uri) if path is not None and "@" in path: - path, _ref = path.rsplit("@", 1) + path, _ref = split_ref_from_uri(path) if ref is None: ref = _ref if relpath and "@" in relpath: - relpath, ref = relpath.rsplit("@", 1) + relpath, ref = split_ref_from_uri(relpath) creation_args = { "name": name if name else parsed_line.name, @@ -1644,7 +2292,7 @@ class VCSRequirement(FileRequirement): if relpath: creation_args["relpath"] = relpath # return cls.create(**creation_args) - return cls( + cls_inst = cls( name=name, ref=ref, vcs=vcs_type, @@ -1655,10 +2303,17 @@ class VCSRequirement(FileRequirement): uri=uri, extras=extras, base_line=line, + parsed_line=parsed_line ) + if cls_inst.req and ( + cls_inst._parsed_line.ireq and not cls_inst.parsed_line.ireq.req + ): + cls_inst._parsed_line._ireq.req = cls_inst.req + return cls_inst @property def line_part(self): + # type: () -> str """requirements.txt compatible line part sans-extras""" if self.is_local: base_link = self.link @@ -1683,12 +2338,13 @@ class VCSRequirement(FileRequirement): base = "{0}{1}".format(base, extras_to_string(sorted(self.extras))) if "git+file:/" in base and "git+file:///" not in base: base = base.replace("git+file:/", "git+file:///") - if self.editable: + if self.editable and not base.startswith("-e "): base = "-e {0}".format(base) return base @staticmethod def _choose_vcs_source(pipfile): + # type: (Dict[str, Union[List[str], str, bool]]) -> Dict[str, Union[List[str], str, bool]] src_keys = [k for k in pipfile.keys() if k in ["path", "uri", "file"]] if src_keys: chosen_key = first(src_keys) @@ -1701,9 +2357,11 @@ class VCSRequirement(FileRequirement): @property def pipfile_part(self): + # type: () -> Dict[str, Dict[str, Union[List[str], str, bool]]] excludes = [ "_repo", "_base_line", "setup_path", "_has_hashed_name", "pyproject_path", - "pyproject_requires", "pyproject_backend", "_setup_info", "_parsed_line" + "pyproject_requires", "pyproject_backend", "_setup_info", "_parsed_line", + "_uri_scheme" ] filter_func = lambda k, v: bool(v) is True and k.name not in excludes # noqa pipfile_dict = attr.asdict(self, filter=filter_func).copy() @@ -1713,30 +2371,44 @@ class VCSRequirement(FileRequirement): return {name: pipfile_dict} -@attr.s +@attr.s(cmp=True, hash=True) class Requirement(object): - name = attr.ib() # type: str - vcs = attr.ib(default=None, validator=attr.validators.optional(validate_vcs)) # type: Optional[str] - req = attr.ib(default=None) - markers = attr.ib(default=None) - specifiers = attr.ib(validator=attr.validators.optional(validate_specifiers)) - index = attr.ib(default=None) - editable = attr.ib(default=None) - hashes = attr.ib(default=attr.Factory(list), converter=list) - extras = attr.ib(default=attr.Factory(list)) - abstract_dep = attr.ib(default=None) - line_instance = attr.ib(default=None) # type: Optional[Line] - _ireq = None + name = attr.ib(cmp=True) # type: str + vcs = attr.ib(default=None, validator=attr.validators.optional(validate_vcs), cmp=True) # type: Optional[str] + req = attr.ib(default=None, cmp=True) # type: Optional[Union[VCSRequirement, FileRequirement, NamedRequirement]] + markers = attr.ib(default=None, cmp=True) # type: Optional[str] + _specifiers = attr.ib(validator=attr.validators.optional(validate_specifiers), cmp=True) # type: Optional[str] + index = attr.ib(default=None, cmp=True) # type: Optional[str] + editable = attr.ib(default=None, cmp=True) # type: Optional[bool] + hashes = attr.ib(factory=frozenset, converter=frozenset, cmp=True) # type: Optional[Tuple[str]] + extras = attr.ib(default=attr.Factory(tuple), cmp=True) # type: Optional[Tuple[str]] + abstract_dep = attr.ib(default=None, cmp=False) # type: Optional[AbstractDependency] + _line_instance = attr.ib(default=None, cmp=False) # type: Optional[Line] + _ireq = attr.ib(default=None, cmp=False) # type: Optional[pip_shims.InstallRequirement] + + def __hash__(self): + return hash(self.as_line()) @name.default def get_name(self): + # type: () -> Optional[str] return self.req.name @property def requirement(self): + # type: () -> Optional[PackagingRequirement] return self.req.req + def add_hashes(self, hashes): + # type: (Union[List, Set, Tuple]) -> Requirement + if isinstance(hashes, six.string_types): + new_hashes = set(self.hashes).add(hashes) + else: + new_hashes = set(self.hashes) | set(hashes) + return attr.evolve(self, hashes=frozenset(new_hashes)) + def get_hashes_as_pip(self, as_list=False): + # type: () -> Union[str, List[str]] if self.hashes: if as_list: return [HASH_STRING.format(h) for h in self.hashes] @@ -1745,10 +2417,12 @@ class Requirement(object): @property def hashes_as_pip(self): + # type: () -> Union[str, List[str]] self.get_hashes_as_pip() @property def markers_as_pip(self): + # type: () -> str if self.markers: return " ; {0}".format(self.markers).replace('"', "'") @@ -1756,6 +2430,7 @@ class Requirement(object): @property def extras_as_pip(self): + # type: () -> str if self.extras: return "[{0}]".format( ",".join(sorted([extra.lower() for extra in self.extras])) @@ -1773,13 +2448,89 @@ class Requirement(object): commit_hash = repo.get_commit_hash() return commit_hash - @specifiers.default + @_specifiers.default def get_specifiers(self): # type: () -> Optional[str] if self.req and self.req.req and self.req.req.specifier: return specs_to_string(self.req.req.specifier) return "" + def update_name_from_path(self, path): + from .setup_info import get_metadata + metadata = get_metadata(path) + name = self.name + if metadata is not None: + name = metadata.get("name") + if name is not None: + if self.req.name is None: + self.req.name = name + if self.req.req and self.req.req.name is None: + self.req.req.name = name + if self._line_instance._name is None: + self._line_instance.name = name + if self.req._parsed_line._name is None: + self.req._parsed_line.name = name + if self.req._setup_info and self.req._setup_info.name is None: + self.req._setup_info.name = name + + @property + def line_instance(self): + # type: () -> Optional[Line] + if self._line_instance is None: + if self.req.parsed_line is not None: + self._line_instance = self.req.parsed_line + else: + include_extras = True + include_specifiers = True + if self.is_vcs: + include_extras = False + if self.is_file_or_url or self.is_vcs or not self._specifiers: + include_specifiers = False + + parts = [ + self.req.line_part, + self.extras_as_pip if include_extras else "", + self._specifiers if include_specifiers else "", + self.markers_as_pip, + ] + self._line_instance = Line("".join(parts)) + return self._line_instance + + @property + def specifiers(self): + # type: () -> Optional[str] + if self._specifiers: + return self._specifiers + else: + specs = self.get_specifiers() + if specs: + self._specifiers = specs + return specs + if not self._specifiers and self.req and self.req.req and self.req.req.specifier: + self._specifiers = specs_to_string(self.req.req.specifier) + elif self.is_named and not self._specifiers: + self._specifiers = self.req.version + elif self.req.parsed_line.specifiers and not self._specifiers: + self._specifiers = specs_to_string(self.req.parsed_line.specifiers) + elif self.line_instance.specifiers and not self._specifiers: + self._specifiers = specs_to_string(self.line_instance.specifiers) + elif not self._specifiers and (self.is_file_or_url or self.is_vcs): + try: + setupinfo_dict = self.run_requires() + except Exception: + setupinfo_dict = None + if setupinfo_dict is not None: + self._specifiers = "=={0}".format(setupinfo_dict.get("version")) + if self._specifiers: + specset = SpecifierSet(self._specifiers) + if self.line_instance and not self.line_instance.specifiers: + self.line_instance.specifiers = specset + if self.req and self.req.parsed_line and not self.req.parsed_line.specifiers: + self.req._parsed_line.specifiers = specset + if self.req and self.req.req and not self.req.req.specifier: + self.req.req.specifier = specset + return self._specifiers + @property def is_vcs(self): # type: () -> bool @@ -1811,6 +2562,13 @@ class Requirement(object): # type: () -> bool return isinstance(self.req, NamedRequirement) + @property + def is_wheel(self): + # type: () -> bool + if not self.is_named and self.req.link is not None and self.req.link.is_wheel: + return True + return False + @property def normalized_name(self): return canonicalize_name(self.name) @@ -1823,98 +2581,105 @@ class Requirement(object): # type: (str) -> Requirement if isinstance(line, pip_shims.shims.InstallRequirement): line = format_requirement(line) - hashes = None - if "--hash=" in line: - hashes = line.split(" --hash=") - line, hashes = hashes[0], hashes[1:] - line_instance = Line(line) - editable = line.startswith("-e ") - line = line.split(" ", 1)[1] if editable else line - line, markers = split_markers_from_line(line) - line, extras = pip_shims.shims._strip_extras(line) - if extras: - extras = parse_extras(extras) - line = line.strip('"').strip("'").strip() - line_with_prefix = "-e {0}".format(line) if editable else line - vcs = None - # Installable local files and installable non-vcs urls are handled - # as files, generally speaking - line_is_vcs = is_vcs(line) - is_direct_url = False - # check for pep-508 compatible requirements - name, _, possible_url = line.partition("@") - name = name.strip() - if possible_url is not None: - possible_url = possible_url.strip() - is_direct_url = is_valid_url(possible_url) - if not line_is_vcs: - line_is_vcs = is_vcs(possible_url) + parsed_line = Line(line) r = None # type: Optional[Union[VCSRequirement, FileRequirement, NamedRequirement]] - if is_installable_file(line) or ( - (is_valid_url(possible_url) or is_file_url(line) or is_valid_url(line)) and - not (line_is_vcs or is_vcs(possible_url)) - ): - r = FileRequirement.from_line(line_with_prefix, extras=extras) - elif line_is_vcs: - r = VCSRequirement.from_line(line_with_prefix, extras=extras) - if isinstance(r, VCSRequirement): - vcs = r.vcs + if ((parsed_line.is_file and parsed_line.is_installable) or parsed_line.is_url) and not parsed_line.is_vcs: + r = file_req_from_parsed_line(parsed_line) + elif parsed_line.is_vcs: + r = vcs_req_from_parsed_line(parsed_line) elif line == "." and not is_installable_file(line): raise RequirementError( "Error parsing requirement %s -- are you sure it is installable?" % line ) else: - specs = "!=<>~" - spec_matches = set(specs) & set(line) - version = None - name = "{0}".format(line) - if spec_matches: - spec_idx = min((line.index(match) for match in spec_matches)) - name = line[:spec_idx] - version = line[spec_idx:] - if not extras: - name, extras = pip_shims.shims._strip_extras(name) - if extras: - extras = parse_extras(extras) - if version: - name = "{0}{1}".format(name, version) - r = NamedRequirement.from_line(line) + r = named_req_from_parsed_line(parsed_line) + # hashes = None + # if "--hash=" in line: + # hashes = line.split(" --hash=") + # line, hashes = hashes[0], hashes[1:] + # editable = line.startswith("-e ") + # line = line.split(" ", 1)[1] if editable else line + # line, markers = split_markers_from_line(line) + # line, extras = pip_shims.shims._strip_extras(line) + # if extras: + # extras = tuple(parse_extras(extras)) + # line = line.strip('"').strip("'").strip() + # line_with_prefix = "-e {0}".format(line) if editable else line + # vcs = None + # # Installable local files and installable non-vcs urls are handled + # # as files, generally speaking + # line_is_vcs = is_vcs(line) + # is_direct_url = False + # # check for pep-508 compatible requirements + # name, _, possible_url = line.partition("@") + # name = name.strip() + # if possible_url is not None: + # possible_url = possible_url.strip() + # is_direct_url = is_valid_url(possible_url) + # if not line_is_vcs: + # line_is_vcs = is_vcs(possible_url) + # if is_installable_file(line) or ( + # (is_valid_url(possible_url) or is_file_url(line) or is_valid_url(line)) and + # not (line_is_vcs or is_vcs(possible_url)) + # ): + # r = FileRequirement.from_line(line_with_prefix, extras=extras, parsed_line=parsed_line) + # elif line_is_vcs: + # r = VCSRequirement.from_line(line_with_prefix, extras=extras, parsed_line=parsed_line) + # if isinstance(r, VCSRequirement): + # vcs = r.vcs + # elif line == "." and not is_installable_file(line): + # raise RequirementError( + # "Error parsing requirement %s -- are you sure it is installable?" % line + # ) + # else: + # specs = "!=<>~" + # spec_matches = set(specs) & set(line) + # version = None + # name = "{0}".format(line) + # if spec_matches: + # spec_idx = min((line.index(match) for match in spec_matches)) + # name = line[:spec_idx] + # version = line[spec_idx:] + # if not extras: + # name, extras = pip_shims.shims._strip_extras(name) + # if extras: + # extras = tuple(parse_extras(extras)) + # if version: + # name = "{0}{1}".format(name, version) + # r = NamedRequirement.from_line(line, parsed_line=parsed_line) req_markers = None - if markers: - req_markers = PackagingRequirement("fakepkg; {0}".format(markers)) + if parsed_line.markers: + req_markers = PackagingRequirement("fakepkg; {0}".format(parsed_line.markers)) if r is not None and r.req is not None: r.req.marker = getattr(req_markers, "marker", None) if req_markers else None - r.req.local_file = getattr(r.req, "local_file", False) - name = getattr(r, "name", None) - if name is None and getattr(r.req, "name", None) is not None: - name = r.req.name - elif name is None and getattr(r.req, "key", None) is not None: - name = r.req.key - if name is not None and getattr(r.req, "name", None) is None: - r.req.name = name + # r.req.local_file = getattr(r.req, "local_file", False) + # name = getattr(r, "name", None) + # if name is None and getattr(r.req, "name", None) is not None: + # name = r.req.name + # elif name is None and getattr(r.req, "key", None) is not None: + # name = r.req.key + # if name is not None and getattr(r.req, "name", None) is None: + # r.req.name = name args = { - "name": name, - "vcs": vcs, + "name": r.name, + "vcs": parsed_line.vcs, "req": r, - "markers": markers, - "editable": editable, - "line_instance": line_instance + "markers": parsed_line.markers, + "editable": parsed_line.editable, + "line_instance": parsed_line } - if extras: - extras = sorted(dedup([extra.lower() for extra in extras])) + if parsed_line.extras: + extras = tuple(sorted(dedup([extra.lower() for extra in parsed_line.extras]))) args["extras"] = extras if r is not None: r.extras = extras elif r is not None and r.extras is not None: - args["extras"] = sorted(dedup([extra.lower() for extra in r.extras])) # type: ignore + args["extras"] = tuple(sorted(dedup([extra.lower() for extra in r.extras]))) # type: ignore if r.req is not None: r.req.extras = args["extras"] - if hashes: - args["hashes"] = hashes # type: ignore + if parsed_line.hashes: + args["hashes"] = tuple(parsed_line.hashes) # type: ignore cls_inst = cls(**args) - if is_direct_url: - setup_info = cls_inst.run_requires() - cls_inst.specifiers = "=={0}".format(setup_info.get("version")) return cls_inst @classmethod @@ -1953,21 +2718,31 @@ class Requirement(object): extras = _pipfile.get("extras") r.req.specifier = SpecifierSet(_pipfile["version"]) r.req.extras = ( - sorted(dedup([extra.lower() for extra in extras])) if extras else [] + tuple(sorted(dedup([extra.lower() for extra in extras]))) if extras else () ) args = { "name": r.name, "vcs": vcs, "req": r, "markers": markers, - "extras": _pipfile.get("extras"), + "extras": tuple(_pipfile.get("extras", ())), "editable": _pipfile.get("editable", False), "index": _pipfile.get("index"), } if any(key in _pipfile for key in ["hash", "hashes"]): args["hashes"] = _pipfile.get("hashes", [pipfile.get("hash")]) cls_inst = cls(**args) - cls_inst.line_instance = Line(cls_inst.as_line()) + if not cls_inst.req._parsed_line: + parsed_line = Line(cls_inst.as_line()) + cls_inst.req._parsed_line = parsed_line + if not cls_inst.line_instance: + cls_inst.line_instance = parsed_line + if not cls_inst.is_named and not cls_inst.req._setup_info and parsed_line.setup_info: + cls_inst.req._setup_info = parsed_line.setup_info + if not cls_inst.req.name and parsed_line.setup_info.name: + cls_inst.name = cls_inst.req.name = parsed_line.setup_info.name + if not cls_inst.req.name and parsed_line.name: + cls_inst.name = cls_inst.req.name = parsed_line.name return cls_inst def as_line( @@ -2024,6 +2799,7 @@ class Requirement(object): return line def get_markers(self): + # type: () -> Marker markers = self.markers if markers: fake_pkg = PackagingRequirement("fakepkg; {0}".format(markers)) @@ -2031,6 +2807,7 @@ class Requirement(object): return markers def get_specifier(self): + # type: () -> Union[SpecifierSet, LegacySpecifier] try: return Specifier(self.specifiers) except InvalidSpecifier: @@ -2058,7 +2835,9 @@ class Requirement(object): @property def is_direct_url(self): - return self.is_file_or_url and self.req.is_direct_url + return self.is_file_or_url and self.req.is_direct_url or ( + self.line_instance.is_direct_url or self.req.parsed_line.is_direct_url + ) def as_pipfile(self): good_keys = ( @@ -2080,7 +2859,7 @@ class Requirement(object): base_dict = { k: v for k, v in self.req.pipfile_part[name].items() - if k not in ["req", "link", "setup_info"] + if k not in ["req", "link", "_setup_info"] } base_dict.update(req_dict) conflicting_keys = ("file", "path", "uri") @@ -2097,11 +2876,17 @@ class Requirement(object): except AttributeError: hashes.append(_hash) base_dict["hashes"] = sorted(hashes) + if "extras" in base_dict: + base_dict["extras"] = list(base_dict["extras"]) if len(base_dict.keys()) == 1 and "version" in base_dict: base_dict = base_dict.get("version") return {name: base_dict} def as_ireq(self): + if self.line_instance and self.line_instance.ireq: + return self.line_instance.ireq + elif getattr(self.req, "_parsed_line", None) and self.req._parsed_line.ireq: + return self.req._parsed_line.ireq kwargs = { "include_hashes": False, } @@ -2201,6 +2986,8 @@ class Requirement(object): def run_requires(self, sources=None, finder=None): if self.req and self.req.setup_info is not None: info_dict = self.req.setup_info.as_dict() + elif self.line_instance and self.line_instance.setup_info is not None: + info_dict = self.line_instance.setup_info.as_dict() else: from .setup_info import SetupInfo if not finder: @@ -2211,7 +2998,7 @@ class Requirement(object): return {} info_dict = info.get_info() if self.req and not self.req.setup_info: - self.req.setup_info = info + self.req._setup_info = info if self.req._has_hashed_name and info_dict.get("name"): self.req.name = self.name = info_dict["name"] if self.req.req.name != info_dict["name"]: @@ -2226,3 +3013,69 @@ class Requirement(object): new_markers = Marker(" or ".join([str(m) for m in sorted(_markers)])) self.markers = str(new_markers) self.req.req.marker = new_markers + + +def file_req_from_parsed_line(parsed_line): + # type: (Line) -> FileRequirement + path = parsed_line.relpath if parsed_line.relpath else parsed_line.path + return FileRequirement( + setup_path=parsed_line.setup_py, + path=path, + editable=parsed_line.editable, + extras=parsed_line.extras, + uri_scheme=parsed_line.preferred_scheme, + link=parsed_line.link, + uri=parsed_line.uri, + pyproject_requires=tuple(parsed_line.pyproject_requires) if parsed_line.pyproject_requires else None, + pyproject_backend=parsed_line.pyproject_backend, + pyproject_path=Path(parsed_line.pyproject_toml) if parsed_line.pyproject_toml else None, + parsed_line=parsed_line, + name=parsed_line.name, + req=parsed_line.requirement + ) + + +def vcs_req_from_parsed_line(parsed_line): + # type: (Line) -> VCSRequirement + line = "{0}".format(parsed_line.line) + if parsed_line.editable: + line = "-e {0}".format(line) + link = create_link(build_vcs_uri( + vcs=parsed_line.vcs, + uri=parsed_line.url, + name=parsed_line.name, + ref=parsed_line.ref, + subdirectory=parsed_line.subdirectory, + extras=parsed_line.extras + )) + return VCSRequirement( + setup_path=parsed_line.setup_py, + path=parsed_line.path, + editable=parsed_line.editable, + vcs=parsed_line.vcs, + ref=parsed_line.ref, + subdirectory=parsed_line.subdirectory, + extras=parsed_line.extras, + uri_scheme=parsed_line.preferred_scheme, + link=link, + uri=parsed_line.uri, + pyproject_requires=tuple(parsed_line.pyproject_requires) if parsed_line.pyproject_requires else None, + pyproject_backend=parsed_line.pyproject_backend, + pyproject_path=Path(parsed_line.pyproject_toml) if parsed_line.pyproject_toml else None, + parsed_line=parsed_line, + name=parsed_line.name, + req=parsed_line.requirement, + base_line=line, + ) + + +def named_req_from_parsed_line(parsed_line): + # type: (Line) -> NamedRequirement + return NamedRequirement( + name=parsed_line.name, + version=parsed_line.specifier, # type: Optional[str] + req=parsed_line.requirement, + extras=parsed_line.extras, + editable=parsed_line.editable, + parsed_line=parsed_line + ) diff --git a/pipenv/vendor/requirementslib/models/setup_info.py b/pipenv/vendor/requirementslib/models/setup_info.py index 7c03a7e9..31134b03 100644 --- a/pipenv/vendor/requirementslib/models/setup_info.py +++ b/pipenv/vendor/requirementslib/models/setup_info.py @@ -1,32 +1,43 @@ # -*- coding=utf-8 -*- from __future__ import absolute_import, print_function +import atexit import contextlib import os +import shutil import sys import attr -import packaging.version import packaging.specifiers import packaging.utils +import packaging.version +import pep517.envbuild +import pep517.wrappers import six +from appdirs import user_cache_dir +from six.moves import configparser +from six.moves.urllib.parse import unquote, urlparse, urlunparse + +from vistir.compat import Iterable, Path +from vistir.contextmanagers import cd, temp_path, replaced_streams +from vistir.misc import run +from vistir.path import create_tracked_tempdir, ensure_mkdir_p, mkdir_p, rmtree + +from ..environment import MYPY_RUNNING +from ..exceptions import RequirementError +from .utils import ( + get_name_variants, + get_pyproject, + init_requirement, + split_vcs_method_from_uri, + strip_extras_markers_from_requirement +) try: from setuptools.dist import distutils except ImportError: import distutils -from appdirs import user_cache_dir -from six.moves import configparser -from six.moves.urllib.parse import unquote -from vistir.compat import Path, Iterable -from vistir.contextmanagers import cd -from vistir.misc import run -from vistir.path import create_tracked_tempdir, ensure_mkdir_p, mkdir_p - -from .utils import init_requirement, get_pyproject, get_name_variants -from ..environment import MYPY_RUNNING -from ..exceptions import RequirementError try: from os import scandir @@ -35,7 +46,7 @@ except ImportError: if MYPY_RUNNING: - from typing import Any, Dict, List, Generator, Optional, Union + from typing import Any, Dict, List, Generator, Optional, Union, Tuple from pip_shims.shims import InstallRequirement from pkg_resources import Requirement as PkgResourcesRequirement @@ -77,7 +88,7 @@ def _get_src_dir(root): virtual_env = os.environ.get("VIRTUAL_ENV") if virtual_env is not None: return os.path.join(virtual_env, "src") - if not root: + if root is not None: # Intentionally don't match pip's behavior here -- this is a temporary copy src_dir = create_tracked_tempdir(prefix="requirementslib-", suffix="-src") else: @@ -96,34 +107,41 @@ def ensure_reqs(reqs): continue if isinstance(req, six.string_types): req = pkg_resources.Requirement.parse("{0}".format(str(req))) + req = strip_extras_markers_from_requirement(req) new_reqs.append(req) return new_reqs -def _prepare_wheel_building_kwargs(ireq=None, src_root=None, editable=False): - # type: (Optional[InstallRequirement], Optional[str], bool) -> Dict[str, str] +def _prepare_wheel_building_kwargs(ireq=None, src_root=None, src_dir=None, editable=False): + # type: (Optional[InstallRequirement], Optional[str], Optional[str], bool) -> Dict[str, str] download_dir = os.path.join(CACHE_DIR, "pkgs") # type: str mkdir_p(download_dir) wheel_download_dir = os.path.join(CACHE_DIR, "wheels") # type: str mkdir_p(wheel_download_dir) - if ireq is None: - src_dir = _get_src_dir(root=src_root) # type: str - elif ireq is not None and ireq.source_dir is not None: - src_dir = ireq.source_dir - elif ireq is not None and ireq.editable: - src_dir = _get_src_dir(root=src_root) - else: - src_dir = create_tracked_tempdir(prefix="reqlib-src") + if src_dir is None: + if editable and src_root is not None: + src_dir = src_root + elif ireq is None and src_root is not None: + src_dir = _get_src_dir(root=src_root) # type: str + # # elif ireq is not None and ireq.editable is not None and ireq.source_dir is not None: + # # src_dir = ireq.source_dir + elif ireq is not None and ireq.editable and src_root is not None: + src_dir = _get_src_dir(root=src_root) + else: + src_dir = create_tracked_tempdir(prefix="reqlib-src") # This logic matches pip's behavior, although I don't fully understand the # intention. I guess the idea is to build editables in-place, otherwise out # of the source tree? - if ireq is None and editable or (ireq is not None and ireq.editable): - build_dir = src_dir - else: - build_dir = create_tracked_tempdir(prefix="reqlib-build") + # if (ireq is not None and ireq.editable) or editable: + # build_dir = src_dir + # else: + + # Let's always resolve in isolation + # src_dir = create_tracked_tempdir(prefix="reqlib-src") + build_dir = create_tracked_tempdir(prefix="reqlib-build") return { "build_dir": build_dir, @@ -133,46 +151,72 @@ def _prepare_wheel_building_kwargs(ireq=None, src_root=None, editable=False): } -def iter_egginfos(path, pkg_name=None): - # type: (str, Optional[str]) -> Generator +def iter_metadata(path, pkg_name=None, metadata_type="egg-info"): + # type: (str, Optional[str], str) -> Generator if pkg_name is not None: pkg_variants = get_name_variants(pkg_name) non_matching_dirs = [] for entry in scandir(path): if entry.is_dir(): entry_name, ext = os.path.splitext(entry.name) - if ext.endswith("egg-info"): - if pkg_name is None or entry_name in pkg_variants: + if ext.endswith(metadata_type): + if pkg_name is None or entry_name.lower() in pkg_variants: yield entry - elif not entry.name.endswith("egg-info"): + elif not entry.name.endswith(metadata_type): non_matching_dirs.append(entry) for entry in non_matching_dirs: - for dir_entry in iter_egginfos(entry.path, pkg_name=pkg_name): + for dir_entry in iter_metadata(entry.path, pkg_name=pkg_name, metadata_type=metadata_type): yield dir_entry def find_egginfo(target, pkg_name=None): # type: (str, Optional[str]) -> Generator - egg_dirs = (egg_dir for egg_dir in iter_egginfos(target, pkg_name=pkg_name)) + egg_dirs = ( + egg_dir for egg_dir in iter_metadata(target, pkg_name=pkg_name) + if egg_dir is not None + ) if pkg_name: - yield next(iter(egg_dirs), None) + yield next(iter(eggdir for eggdir in egg_dirs if eggdir is not None), None) else: for egg_dir in egg_dirs: yield egg_dir +def find_distinfo(target, pkg_name=None): + # type: (str, Optional[str]) -> Generator + dist_dirs = ( + dist_dir for dist_dir in iter_metadata(target, pkg_name=pkg_name, metadata_type="dist-info") + if dist_dir is not None + ) + if pkg_name: + yield next(iter(dist for dist in dist_dirs if dist is not None), None) + else: + for dist_dir in dist_dirs: + yield dist_dir + + def get_metadata(path, pkg_name=None): egg_dir = next(iter(find_egginfo(path, pkg_name=pkg_name)), None) - if egg_dir is not None: + dist_dir = next(iter(find_distinfo(path, pkg_name=pkg_name)), None) + matched_dir = next(iter(d for d in (dist_dir, egg_dir) if d is not None), None) + metadata_dir = None + base_dir = None + if matched_dir is not None: import pkg_resources - - egg_dir = os.path.abspath(egg_dir.path) - base_dir = os.path.dirname(egg_dir) - path_metadata = pkg_resources.PathMetadata(base_dir, egg_dir) - dist = next( - iter(pkg_resources.distributions_from_metadata(path_metadata.egg_info)), - None, - ) + metadata_dir = os.path.abspath(matched_dir.path) + base_dir = os.path.dirname(metadata_dir) + dist = None + distinfo_dist = None + egg_dist = None + if dist_dir is not None: + distinfo_dist = next(iter(pkg_resources.find_distributions(base_dir)), None) + if egg_dir is not None: + path_metadata = pkg_resources.PathMetadata(base_dir, metadata_dir) + egg_dist = next( + iter(pkg_resources.distributions_from_metadata(path_metadata.egg_info)), + None, + ) + dist = next(iter(d for d in (distinfo_dist, egg_dist) if d is not None), None) if dist: try: requires = dist.requires() @@ -210,22 +254,96 @@ def get_metadata(path, pkg_name=None): } -@attr.s(slots=True) +@attr.s(slots=True, frozen=True) +class BaseRequirement(object): + name = attr.ib(type=str, default="", cmp=True) + requirement = attr.ib(default=None, cmp=True) # type: Optional[PkgResourcesRequirement] + + def __str__(self): + # type: () -> str + return "{0}".format(str(self.requirement)) + + def as_dict(self): + # type: () -> Dict[str, Optional[PkgResourcesRequirement]] + return {self.name: self.requirement} + + def as_tuple(self): + # type: () -> Tuple[str, Optional[PkgResourcesRequirement]] + return (self.name, self.requirement) + + @classmethod + def from_string(cls, line): + # type: (str) -> BaseRequirement + line = line.strip() + req = init_requirement(line) + return cls.from_req(req) + + @classmethod + def from_req(cls, req): + # type: (PkgResourcesRequirement) -> BaseRequirement + name = None + key = getattr(req, "key", None) + name = getattr(req, "name", None) + project_name = getattr(req, "project_name", None) + if key is not None: + name = key + if name is None: + name = project_name + return cls(name=name, requirement=req) + + +@attr.s(slots=True, frozen=True) +class Extra(object): + name = attr.ib(type=str, default=None, cmp=True) + requirements = attr.ib(factory=frozenset, cmp=True, type=frozenset) + + def __str__(self): + # type: () -> str + return "{0}: {{{1}}}".format(self.section, ", ".join([r.name for r in self.requirements])) + + def add(self, req): + # type: (BaseRequirement) -> None + if req not in self.requirements: + return attr.evolve(self, requirements=frozenset(set(self.requirements).add(req))) + return self + + def as_dict(self): + # type: () -> Dict[str, Tuple[PkgResourcesRequirement]] + return {self.name: tuple([r.requirement for r in self.requirements])} + + +@attr.s(slots=True, cmp=True, hash=True) class SetupInfo(object): - name = attr.ib(type=str, default=None) - base_dir = attr.ib(type=Path, default=None) - version = attr.ib(type=packaging.version.Version, default=None) - requires = attr.ib(type=dict, default=attr.Factory(dict)) - build_requires = attr.ib(type=list, default=attr.Factory(list)) - build_backend = attr.ib(type=list, default=attr.Factory(list)) - setup_requires = attr.ib(type=dict, default=attr.Factory(list)) - python_requires = attr.ib(type=packaging.specifiers.SpecifierSet, default=None) - extras = attr.ib(type=dict, default=attr.Factory(dict)) - setup_cfg = attr.ib(type=Path, default=None) - setup_py = attr.ib(type=Path, default=None) - pyproject = attr.ib(type=Path, default=None) - ireq = attr.ib(default=None) - extra_kwargs = attr.ib(default=attr.Factory(dict), type=dict) + name = attr.ib(type=str, default=None, cmp=True) + base_dir = attr.ib(type=str, default=None, cmp=True, hash=False) + version = attr.ib(type=str, default=None, cmp=True) + _requirements = attr.ib(type=frozenset, factory=frozenset, cmp=True, hash=True) + build_requires = attr.ib(type=tuple, default=attr.Factory(tuple), cmp=True) + build_backend = attr.ib(type=str, default="setuptools.build_meta:__legacy__", cmp=True) + setup_requires = attr.ib(type=tuple, default=attr.Factory(tuple), cmp=True) + python_requires = attr.ib(type=packaging.specifiers.SpecifierSet, default=None, cmp=True) + _extras_requirements = attr.ib(type=tuple, default=attr.Factory(tuple), cmp=True) + setup_cfg = attr.ib(type=Path, default=None, cmp=True, hash=False) + setup_py = attr.ib(type=Path, default=None, cmp=True, hash=False) + pyproject = attr.ib(type=Path, default=None, cmp=True, hash=False) + ireq = attr.ib(default=None, cmp=True, hash=False) + extra_kwargs = attr.ib(default=attr.Factory(dict), type=dict, cmp=False, hash=False) + metadata = attr.ib(default=None, type=tuple) + + @property + def requires(self): + return {req.name: req.requirement for req in self._requirements} + + @property + def extras(self): + extras_dict = {} + extras = set(self._extras_requirements) + for section, deps in extras: + if isinstance(deps, BaseRequirement): + extras_dict[section] = deps.requirement + elif isinstance(deps, (list, tuple)): + extras_dict[section] = [d.requirement for d in deps] + return extras_dict @classmethod def get_setup_cfg(cls, setup_cfg_path): @@ -247,32 +365,53 @@ class SetupInfo(object): results["name"] = parser.get("metadata", "name") if parser.has_option("metadata", "version"): results["version"] = parser.get("metadata", "version") - install_requires = {} + install_requires = set() if parser.has_option("options", "install_requires"): - install_requires = { - dep.strip(): init_requirement(dep.strip()) + install_requires = set([ + BaseRequirement.from_string(dep) for dep in parser.get("options", "install_requires").split("\n") if dep - } + ]) results["install_requires"] = install_requires if parser.has_option("options", "python_requires"): results["python_requires"] = parser.get("options", "python_requires") - extras_require = {} + if parser.has_option("options", "build_requires"): + results["build_requires"] = parser.get("options", "build_requires") + extras_require = () if "options.extras_require" in parser.sections(): - extras_require = { - section: [ - init_requirement(dep.strip()) + extras_require = tuple([ + (section, tuple([ + BaseRequirement.from_string(dep) for dep in parser.get( "options.extras_require", section ).split("\n") if dep - ] + ])) for section in parser.options("options.extras_require") if section not in ["options", "metadata"] - } + ]) results["extras_require"] = extras_require return results + @property + def egg_base(self): + base = None # type: Optional[Path] + if self.setup_py.exists(): + base = self.setup_py.parent + elif self.pyproject.exists(): + base = self.pyproject.parent + elif self.setup_cfg.exists(): + base = self.setup_cfg.parent + if base is None: + base = Path(self.base_dir) + if base is None: + base = Path(self.extra_kwargs["build_dir"]) + egg_base = base.joinpath("reqlib-metadata") + if not egg_base.exists(): + atexit.register(rmtree, egg_base.as_posix()) + egg_base.mkdir(parents=True, exist_ok=True) + return egg_base.as_posix() + def parse_setup_cfg(self): if self.setup_cfg is not None and self.setup_cfg.exists(): parsed = self.get_setup_cfg(self.setup_cfg.as_posix()) @@ -280,25 +419,34 @@ class SetupInfo(object): self.name = parsed.get("name") if self.version is None: self.version = parsed.get("version") - self.requires.update(parsed["install_requires"]) + build_requires = parsed.get("build_requires", []) + if self.build_requires: + self.build_requires = tuple(set(self.build_requires) | set(build_requires)) + self._requirements = frozenset( + set(self._requirements) | parsed["install_requires"] + ) if self.python_requires is None: self.python_requires = parsed.get("python_requires") - self.extras.update(parsed["extras_require"]) + if not self._extras_requirements: + self._extras_requirements = (parsed["extras_require"]) + else: + self._extras_requirements = self._extras_requirements + parsed["extras_require"] if self.ireq is not None and self.ireq.extras: - self.requires.update({ - extra: self.extras[extra] - for extra in self.ireq.extras if extra in self.extras - }) + for extra in self.ireq.extras: + if extra in self.extras: + extras_tuple = tuple([BaseRequirement.from_req(req) for req in self.extras[extra]]) + self._extras_requirements += ((extra, extras_tuple),) def run_setup(self): if self.setup_py is not None and self.setup_py.exists(): target_cwd = self.setup_py.parent.as_posix() - with cd(target_cwd), _suppress_distutils_logs(): + with temp_path(), cd(target_cwd), _suppress_distutils_logs(): # This is for you, Hynek # see https://github.com/hynek/environ_config/blob/69b1c8a/setup.py script_name = self.setup_py.as_posix() - args = ["egg_info"] + args = ["egg_info", "--egg-base", self.egg_base] g = {"__file__": script_name, "__name__": "__main__"} + sys.path.insert(0, os.path.dirname(os.path.abspath(script_name))) local_dict = {} if sys.version_info < (3, 5): save_argv = sys.argv @@ -334,41 +482,101 @@ class SetupInfo(object): self.python_requires = packaging.specifiers.SpecifierSet( dist.python_requires ) + if not self._extras_requirements: + self._extras_requirements = () if dist.extras_require and not self.extras: - self.extras = dist.extras_require + for extra, extra_requires in dist.extras_require: + extras_tuple = tuple( + BaseRequirement.from_req(req) for req in extra_requires + ) + self._extras_requirements += ((extra, extras_tuple),) install_requires = dist.get_requires() if not install_requires: install_requires = dist.install_requires if install_requires and not self.requires: - requirements = [init_requirement(req) for req in install_requires] + requirements = set([ + BaseRequirement.from_req(req) for req in install_requires + ]) if getattr(self.ireq, "extras", None): for extra in self.ireq.extras: - requirements.extend(list(self.extras.get(extra, []))) - self.requires.update({req.key: req for req in requirements}) + requirements |= set(list(self.extras.get(extra, []))) + self._requirements = frozenset( + set(self._requirements) | requirements + ) if dist.setup_requires and not self.setup_requires: - self.setup_requires = dist.setup_requires + self.setup_requires = tuple(dist.setup_requires) if not self.version: self.version = dist.get_version() + def run_pep517(self, build=False): + # type: () -> str + with pep517.envbuild.BuildEnvironment(): + hookcaller = pep517.wrappers.Pep517HookCaller( + self.base_dir, self.build_backend + ) + build_deps = hookcaller.get_requires_for_build_wheel() + if self.ireq.editable: + build_deps += hookcaller.get_requires_for_build_sdist() + metadata_dirname = hookcaller.prepare_metadata_for_build_wheel(self.egg_base) + metadata_dir = os.path.join(self.egg_base, metadata_dirname) + + if build: + return self.build_pep517() + return metadata_dir + + def build_pep517(self, hookcaller): + # type: (pep517.wrappers.Pep517HookCaller) -> Optional[str] + dist_path = None + try: + dist_path = hookcaller.build_wheel( + self.extra_kwargs["build_dir"], + metadata_directory=self.egg_base + ) + except Exception: + dist_path = hookcaller.build_sdist(self.extra_kwargs["build_dir"]) + return dist_path + + def reload(self): + # type: () -> None + """ + Wipe existing distribution info metadata for rebuilding. + """ + for metadata_dir in os.listdir(self.egg_base): + shutil.rmtree(metadata_dir, ignore_errors=True) + self.metadata = None + self._requirements = frozenset() + self._extras_requirements = () + self.get_info() + def get_egg_metadata(self): - if self.setup_py is not None and self.setup_py.exists(): - metadata = get_metadata(self.setup_py.parent.as_posix(), pkg_name=self.name) + package_indicators = [self.pyproject, self.setup_py, self.setup_cfg] + # if self.setup_py is not None and self.setup_py.exists(): + if any([fn is not None and fn.exists() for fn in package_indicators]): + metadata = get_metadata(self.egg_base, pkg_name=self.name) if metadata: + self.metadata = tuple([(k, v) for k, v in metadata.items()]) if self.name is None: self.name = metadata.get("name", self.name) if not self.version: self.version = metadata.get("version", self.version) - self.requires.update( - {req.key: req for req in metadata.get("requires", {})} + self._requirements = frozenset( + set(self._requirements) | set([ + BaseRequirement.from_req(req) + for req in metadata.get("requires", []) + ]) ) if getattr(self.ireq, "extras", None): for extra in self.ireq.extras: extras = metadata.get("extras", {}).get(extra, []) if extras: - extras = ensure_reqs(extras) - self.extras[extra] = set(extras) - self.requires.update( - {req.key: req for req in extras if req is not None} + extras_tuple = tuple([ + BaseRequirement.from_req(req) + for req in ensure_reqs(extras) + if req is not None + ]) + self._extras_requirements += ((extra, extras_tuple),) + self._requirements = frozenset( + set(self._requirements) | set(extras_tuple) ) def run_pyproject(self): @@ -378,35 +586,37 @@ class SetupInfo(object): requires, backend = result if backend: self.build_backend = backend + else: + self.build_backend = "setuptools.build_meta:__legacy__" + self.build_requires = ("setuptools", "wheel") if requires and not self.build_requires: - self.build_requires = requires + self.build_requires = tuple(requires) def get_info(self): - initial_path = os.path.abspath(os.getcwd()) if self.setup_cfg and self.setup_cfg.exists(): - try: + with cd(self.base_dir): self.parse_setup_cfg() - finally: - os.chdir(initial_path) - if self.setup_py and self.setup_py.exists(): - if not self.requires or not self.name: - try: - self.run_setup() - except Exception: - self.get_egg_metadata() - finally: - os.chdir(initial_path) - if not self.requires or not self.name: - try: - self.get_egg_metadata() - finally: - os.chdir(initial_path) if self.pyproject and self.pyproject.exists(): - try: + with cd(self.base_dir), replaced_streams(): self.run_pyproject() - finally: - os.chdir(initial_path) + self.run_pep517() + self.get_egg_metadata() + + if self.setup_py and self.setup_py.exists() and self.metadata is None: + if not self.requires or not self.name: + try: + with cd(self.base_dir): + for metadata_dir in os.listdir(self.egg_base): + shutil.rmtree(metadata_dir, ignore_errors=True) + self.run_setup() + except Exception: + with cd(self.base_dir): + self.get_egg_metadata() + if self.metadata is None or not self.name: + with cd(self.base_dir): + self.get_egg_metadata() + return self.as_dict() def as_dict(self): @@ -438,15 +648,31 @@ class SetupInfo(object): @classmethod def from_ireq(cls, ireq, subdir=None, finder=None): import pip_shims.shims - + if not ireq.link: + return if ireq.link.is_wheel: return if not finder: from .dependencies import get_finder finder = get_finder() + vcs_method, uri = split_vcs_method_from_uri(unquote(ireq.link.url_without_fragment)) + parsed = urlparse(uri) + url_path = parsed.path + if "@" in url_path: + url_path, _, _ = url_path.rpartition("@") + parsed = parsed._replace(path=url_path) + uri = urlunparse(parsed) + path = None + if ireq.link.scheme == "file" or uri.startswith("file://"): + if "file:/" in uri and "file:///" not in uri: + uri = uri.replace("file:/", "file:///") + path = pip_shims.shims.url_to_path(uri) + # if pip_shims.shims.is_installable_dir(path) and ireq.editable: + # ireq.source_dir = path kwargs = _prepare_wheel_building_kwargs(ireq) - ireq.populate_link(finder, False, False) + ireq.source_dir = kwargs["src_dir"] + # os.environ["PIP_BUILD_DIR"] = kwargs["build_dir"] ireq.ensure_has_source_dir(kwargs["build_dir"]) if not ( ireq.editable @@ -459,37 +685,31 @@ class SetupInfo(object): else: only_download = False download_dir = kwargs["download_dir"] - ireq_src_dir = None - if ireq.link.scheme == "file": - path = pip_shims.shims.url_to_path(unquote(ireq.link.url_without_fragment)) - if pip_shims.shims.is_installable_dir(path): - ireq_src_dir = path - elif os.path.isdir(path): - raise RequirementError( - "The file URL points to a directory not installable: {}" - .format(ireq.link) - ) - if not ireq.editable or not ireq.link.scheme == "file": - pip_shims.shims.unpack_url( - ireq.link, - ireq.source_dir, - download_dir, - only_download=only_download, - session=finder.session, - hashes=ireq.hashes(False), - progress_bar="off", + elif path is not None and os.path.isdir(path): + raise RequirementError( + "The file URL points to a directory not installable: {}" + .format(ireq.link) ) - if ireq.editable: - created = cls.create( - ireq.source_dir, subdirectory=subdir, ireq=ireq, kwargs=kwargs - ) - else: + if not ireq.editable: build_dir = ireq.build_location(kwargs["build_dir"]) ireq._temp_build_dir.path = kwargs["build_dir"] - created = cls.create( - build_dir, subdirectory=subdir, ireq=ireq, kwargs=kwargs - ) - created.get_info() + else: + build_dir = ireq.build_location(kwargs["src_dir"]) + ireq._temp_build_dir.path = kwargs["build_dir"] + + ireq.populate_link(finder, False, False) + pip_shims.shims.unpack_url( + ireq.link, + build_dir, + download_dir, + only_download=only_download, + session=finder.session, + hashes=ireq.hashes(False), + progress_bar="off", + ) + created = cls.create( + build_dir, subdirectory=subdir, ireq=ireq, kwargs=kwargs + ) return created @classmethod @@ -498,6 +718,7 @@ class SetupInfo(object): return creation_kwargs = {"extra_kwargs": kwargs} + if not isinstance(base_dir, Path): base_dir = Path(base_dir) creation_kwargs["base_dir"] = base_dir.as_posix() diff --git a/pipenv/vendor/requirementslib/models/utils.py b/pipenv/vendor/requirementslib/models/utils.py index 90bca388..a5e781f4 100644 --- a/pipenv/vendor/requirementslib/models/utils.py +++ b/pipenv/vendor/requirementslib/models/utils.py @@ -3,6 +3,8 @@ from __future__ import absolute_import, print_function import io import os +import re +import string import sys from collections import defaultdict @@ -16,7 +18,10 @@ from attr import validators from first import first from packaging.markers import InvalidMarker, Marker, Op, Value, Variable from packaging.specifiers import InvalidSpecifier, Specifier, SpecifierSet +from six.moves.urllib import parse as urllib_parse +from urllib3 import util as urllib3_util from vistir.misc import dedup +from vistir.path import is_valid_url from ..utils import SCHEME_LIST, VCS_LIST, is_star, add_ssh_scheme_to_git_uri @@ -24,15 +29,34 @@ from ..utils import SCHEME_LIST, VCS_LIST, is_star, add_ssh_scheme_to_git_uri from ..environment import MYPY_RUNNING if MYPY_RUNNING: - from typing import Union, Optional, List, Set, Any, TypeVar + from typing import Union, Optional, List, Set, Any, TypeVar, Tuple, Sequence, Dict from attr import _ValidatorType + from packaging.requirements import Requirement as PackagingRequirement from pkg_resources import Requirement as PkgResourcesRequirement - from pip_shims import Link + from pkg_resources.extern.packaging.markers import Marker as PkgResourcesMarker + from pip_shims.shims import Link _T = TypeVar("_T") + TMarker = Union[Marker, PkgResourcesMarker] + TRequirement = Union[PackagingRequirement, PkgResourcesRequirement] HASH_STRING = " --hash={0}" +ALPHA_NUMERIC = r"[{0}{1}]".format(string.ascii_letters, string.digits) +PUNCTUATION = r"[\-_\.]" +ALPHANUM_PUNCTUATION = r"[{0}{1}\-_\.]".format(string.ascii_letters, string.digits) +NAME = r"{0}+{1}*{2}".format(ALPHANUM_PUNCTUATION, PUNCTUATION, ALPHA_NUMERIC) +REF = r"[{0}{1}\-\_\./]".format(string.ascii_letters, string.digits) +EXTRAS = r"(?P\[{0}(?:,{0})*\])".format(NAME) +NAME_WITH_EXTRAS = r"(?P{0}){1}?".format(NAME, EXTRAS) +NAME_RE = re.compile(NAME_WITH_EXTRAS) +SUBDIR_RE = r"(?:[&#]subdirectory=(?P.*))" +URL_NAME = r"(?:#egg={0})".format(NAME_WITH_EXTRAS) +REF_RE = r"(?:@(?P{0}+)?)".format(REF) +URL = r"(?P[^ ]+://)(?:(?P[^ ]+?\.?{0}+(?P:\d+)?))?(?P[:/])(?P[^ @]+){1}?".format(ALPHA_NUMERIC, REF_RE) +URL_RE = re.compile(r"{0}(?:{1}?{2}?)?".format(URL, URL_NAME, SUBDIR_RE)) +DIRECT_URL_RE = re.compile(r"{0}\s?@\s?{1}".format(NAME_WITH_EXTRAS, URL)) + def filter_none(k, v): # type: (str, Any) -> bool @@ -51,12 +75,26 @@ def create_link(link): if not isinstance(link, six.string_types): raise TypeError("must provide a string to instantiate a new link") - from pip_shims import Link + from pip_shims.shims import Link return Link(link) +def get_url_name(url): + # type: (str) -> str + """ + Given a url, derive an appropriate name to use in a pipfile. + + :param str url: A url to derive a string from + :returns: The name of the corresponding pipfile entry + :rtype: str + """ + if not isinstance(url, six.string_types): + raise TypeError("Expected a string, got {0!r}".format(url)) + return urllib3_util.parse_url(url).host + + def init_requirement(name): - # type: (str) -> PkgResourcesRequirement + # type: (str) -> TRequirement if not isinstance(name, six.string_types): raise TypeError("must supply a name to generate a requirement") @@ -70,6 +108,7 @@ def init_requirement(name): def extras_to_string(extras): + # type: (Sequence) -> str """Turn a list of extras into a string""" if isinstance(extras, six.string_types): if extras.startswith("["): @@ -77,7 +116,9 @@ def extras_to_string(extras): else: extras = [extras] - return "[{0}]".format(",".join(sorted(extras))) + if not extras: + return "" + return "[{0}]".format(",".join(sorted(set(extras)))) def parse_extras(extras_str): @@ -109,7 +150,7 @@ def specs_to_string(specs): def build_vcs_uri( - vcs, # type: str + vcs, # type: Optional[str] uri, # type: str name=None, # type: Optional[str] ref=None, # type: Optional[str] @@ -119,9 +160,11 @@ def build_vcs_uri( # type: (...) -> str if extras is None: extras = [] - vcs_start = "{0}+".format(vcs) - if not uri.startswith(vcs_start): - uri = "{0}{1}".format(vcs_start, uri) + vcs_start = "" + if vcs is not None: + vcs_start = "{0}+".format(vcs) + if not uri.startswith(vcs_start): + uri = "{0}{1}".format(vcs_start, uri) if ref: uri = "{0}@{1}".format(uri, ref) if name: @@ -134,21 +177,140 @@ def build_vcs_uri( return uri +def convert_direct_url_to_url(direct_url): + # type: (str) -> str + """ + Given a direct url as defined by *PEP 508*, convert to a :class:`~pip_shims.shims.Link` + compatible URL by moving the name and extras into an **egg_fragment**. + + :param str direct_url: A pep-508 compliant direct url. + :return: A reformatted URL for use with Link objects and :class:`~pip_shims.shims.InstallRequirement` objects. + :rtype: str + """ + direct_match = DIRECT_URL_RE.match(direct_url) + if direct_match is None: + url_match = URL_RE.match(direct_url) + if url_match or is_valid_url(direct_url): + return url_match + match_dict = direct_match.groupdict() + url = [match_dict.get(segment) for segment in ("scheme", "host", "path", "pathsep")] + url = "".join([s for s in url if s is not None]) + new_url = build_vcs_uri( + None, + url, + ref=match_dict.get("ref"), + name=match_dict.get("name"), + extras=match_dict.get("extras"), + subdir=match_dict.get("subdirectory") + ) + return new_url + + +def convert_url_to_direct_url(url, name=None): + # type: (str, Optional[str]) -> str + """ + Given a :class:`~pip_shims.shims.Link` compatible URL, convert to a direct url as + defined by *PEP 508* by extracting the name and extras from the **egg_fragment**. + + :param str url: A :class:`~pip_shims.shims.InstallRequirement` compliant URL. + :param Optiona[str] name: A name to use in case the supplied URL doesn't provide one. + :return: A pep-508 compliant direct url. + :rtype: str + + :raises ValueError: Raised when the URL can't be parsed or a name can't be found. + :raises TypeError: When a non-string input is provided. + """ + if not isinstance(url, six.string_types): + raise TypeError( + "Expected a string to convert to a direct url, got {0!r}".format(url) + ) + direct_match = DIRECT_URL_RE.match(url) + if direct_match: + return url + url_match = URL_RE.match(url) + if url_match is None: + raise ValueError("Failed parse a valid URL from {0!r}".format(url)) + match_dict = url_match.groupdict() + url = [match_dict.get(segment) for segment in ("scheme", "host", "path", "pathsep")] + name = match_dict.get("name", name) + extras = match_dict.get("extras") + new_url = "" + if extras and not name: + url.append(extras) + elif extras and name: + new_url = "{0}{1}@ ".format(name, extras) + else: + if name is not None: + new_url = "{0}@ ".format(name) + else: + raise ValueError( + "Failed to construct direct url: " + "No name could be parsed from {0!r}".format(url) + ) + if match_dict.get("ref"): + url.append("@{0}".format(match_dict.get("ref"))) + url = "".join([s for s in url if s is not None]) + new_url = "{0}{1}".format(new_url, url) + return new_url + + def get_version(pipfile_entry): + # type: (Union[str, Dict[str, bool, List[str]]]) -> str if str(pipfile_entry) == "{}" or is_star(pipfile_entry): return "" elif hasattr(pipfile_entry, "keys") and "version" in pipfile_entry: if is_star(pipfile_entry.get("version")): return "" - return pipfile_entry.get("version", "") + return pipfile_entry.get("version", "").strip().lstrip("(").rstrip(")") if isinstance(pipfile_entry, six.string_types): - return pipfile_entry + return pipfile_entry.strip().lstrip("(").rstrip(")") return "" +def strip_extras_markers_from_requirement(req): + # type: (TRequirement) -> TRequirement + """ + Given a :class:`~packaging.requirements.Requirement` instance with markers defining + *extra == 'name'*, strip out the extras from the markers and return the cleaned + requirement + + :param PackagingRequirement req: A pacakaging requirement to clean + :return: A cleaned requirement + :rtype: PackagingRequirement + """ + if req is None: + raise TypeError("Must pass in a valid requirement, received {0!r}".format(req)) + if req.marker is not None: + req.marker._markers = _strip_extras_markers(req.marker._markers) + return req + + +def _strip_extras_markers(marker): + # type: (TMarker) -> TMarker + if marker is None or not isinstance(marker, (list, tuple)): + raise TypeError("Expecting a marker type, received {0!r}".format(marker)) + markers_to_remove = [] + # iterate forwards and generate a list of indexes to remove first, then reverse the + # list so we can remove the text that normally occurs after (but we will already + # be past it in the loop) + for i, marker_list in enumerate(marker): + if isinstance(marker_list, list): + cleaned = _strip_extras_markers(marker_list) + if not cleaned: + markers_to_remove.append(i) + elif isinstance(marker_list, tuple) and marker_list[0].value == "extra": + markers_to_remove.append(i) + for i in reversed(markers_to_remove): + del marker[i] + if i > 0 and marker[i - 1] == "and": + del marker[i - 1] + return marker + + def get_pyproject(path): + # type: (Union[str, Path]) -> Tuple[List[str], str] """ Given a base path, look for the corresponding ``pyproject.toml`` file and return its build_requires and build_backend. @@ -194,6 +356,7 @@ def get_pyproject(path): def split_markers_from_line(line): + # type: (str) -> Tuple[str, Optional[str]] """Split markers from a dependency""" if not any(line.startswith(uri_prefix) for uri_prefix in SCHEME_LIST): marker_sep = ";" @@ -207,6 +370,7 @@ def split_markers_from_line(line): def split_vcs_method_from_uri(uri): + # type: (str) -> Tuple[Optional[str], str] """Split a vcs+uri formatted uri into (vcs, uri)""" vcs_start = "{0}+" vcs = first([vcs for vcs in VCS_LIST if uri.startswith(vcs_start.format(vcs))]) @@ -215,6 +379,27 @@ def split_vcs_method_from_uri(uri): return vcs, uri +def split_ref_from_uri(uri): + # type: (str) -> Tuple[str, Optional[str]] + """ + Given a path or URI, check for a ref and split it from the path if it is present, + returning a tuple of the original input and the ref or None. + + :param str uri: The path or URI to split + :returns: A 2-tuple of the path or URI and the ref + :rtype: Tuple[str, Optional[str]] + """ + if not isinstance(uri, six.string_types): + raise TypeError("Expected a string, received {0!r}".format(uri)) + parsed = urllib_parse.urlparse(uri) + path = parsed.path + ref = None + if "@" in path: + path, _, ref = path.rpartition("@") + parsed = parsed._replace(path=path) + return (urllib_parse.urlunparse(parsed), ref) + + def validate_vcs(instance, attr_, value): if value not in VCS_LIST: raise ValueError("Invalid vcs {0!r}".format(value)) @@ -622,7 +807,8 @@ def get_name_variants(pkg): raise TypeError("must provide a string to derive package names") from pkg_resources import safe_name from packaging.utils import canonicalize_name - names = {safe_name(pkg), canonicalize_name(pkg)} + pkg = pkg.lower() + names = {safe_name(pkg), canonicalize_name(pkg), pkg.replace("-", "_")} return names diff --git a/pipenv/vendor/requirementslib/models/vcs.py b/pipenv/vendor/requirementslib/models/vcs.py index aa9e051e..9296f605 100644 --- a/pipenv/vendor/requirementslib/models/vcs.py +++ b/pipenv/vendor/requirementslib/models/vcs.py @@ -9,7 +9,7 @@ import six import sys -@attr.s +@attr.s(hash=True) class VCSRepository(object): DEFAULT_RUN_ARGS = None diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index b8b03eb0..f96e6ec6 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -233,9 +233,6 @@ class _PipenvInstance(object): def __enter__(self): if self.chdir: os.chdir(self.path) - os.environ['PIPENV_PIPFILE'] = fs_str(self.pipfile_path) - c = delegator.run("pipenv run pip install /home/hawk/git/pip") - assert c.return_code == 0 return self def __exit__(self, *args): @@ -300,7 +297,6 @@ def PipenvInstance(): os.environ["PIPENV_NOSPIN"] = fs_str("1") os.environ["CI"] = fs_str("1") os.environ['PIPENV_DONT_USE_PYENV'] = fs_str('1') - os.environ['PIPENV_NOSPIN'] = fs_str('1') warnings.simplefilter("ignore", category=ResourceWarning) warnings.filterwarnings("ignore", category=ResourceWarning, message="unclosed.*") try: @@ -308,6 +304,7 @@ def PipenvInstance(): finally: os.umask(original_umask) + @pytest.fixture(autouse=True) def pip_src_dir(request, pathlib_tmpdir): old_src_dir = os.environ.get('PIP_SRC', '') diff --git a/tests/integration/test_install_basic.py b/tests/integration/test_install_basic.py index f858f8ac..be64289c 100644 --- a/tests/integration/test_install_basic.py +++ b/tests/integration/test_install_basic.py @@ -454,17 +454,18 @@ def test_rewrite_outline_table(PipenvInstance, pypi): with open(p.pipfile_path, 'w') as f: contents = """ [packages] -six = {version = "*", editable = true} +six = {version = "*"} [packages.requests] version = "*" +extras = ["socks"] """.strip() f.write(contents) - c = p.pipenv("install -e click") + c = p.pipenv("install plette[validation]") assert c.return_code == 0 with open(p.pipfile_path) as f: contents = f.read() assert "[packages.requests]" not in contents - assert 'six = {version = "*", editable = true}' in contents - assert 'requests = {version = "*"}' in contents - assert 'click = {' in contents + assert 'six = {version = "*"}' in contents + assert 'requests = {version = "*", extras = ["socks"]}' in contents + assert 'plette = {' in contents diff --git a/tests/integration/test_install_twists.py b/tests/integration/test_install_twists.py index bbe626cb..c4ee3fef 100644 --- a/tests/integration/test_install_twists.py +++ b/tests/integration/test_install_twists.py @@ -23,22 +23,21 @@ def test_local_extras_install(PipenvInstance, pypi): contents = """ from setuptools import setup, find_packages setup( -name='testpipenv', -version='0.1', -description='Pipenv Test Package', -author='Pipenv Test', -author_email='test@pipenv.package', -license='MIT', -packages=find_packages(), -install_requires=[], -extras_require={'dev': ['six']}, -zip_safe=False + name='testpipenv', + version='0.1', + description='Pipenv Test Package', + author='Pipenv Test', + author_email='test@pipenv.package', + license='MIT', + packages=find_packages(), + install_requires=[], + extras_require={'dev': ['six']}, + zip_safe=False ) """.strip() fh.write(contents) line = "-e .[dev]" - # pipfile = {"testpipenv": {"path": ".", "editable": True, "extras": ["dev"]}} - project = Project() + pipfile = {"testpipenv": {"path": ".", "editable": True, "extras": ["dev"]}} with open(os.path.join(p.path, 'Pipfile'), 'w') as fh: fh.write(""" [packages] @@ -54,10 +53,11 @@ testpipenv = {path = ".", editable = true, extras = ["dev"]} assert "six" in p.lockfile["default"] c = p.pipenv("--rm") assert c.return_code == 0 + project = Project() project.write_toml({"packages": {}, "dev-packages": {}}) c = p.pipenv("install {0}".format(line)) assert c.return_code == 0 - assert "testpipenv" in p.pipfile["packages"] + assert "testpipenv" in p.pipfile["packages"], "{0}\n{1}\n\n{2}\n\n{3}".format(p.pipfile, Path(p.pipfile_path).read_text(), Path(os.getcwd()).joinpath("setup.py").read_text(), Path(os.path.join(os.getcwd(), "testpipenv.egg-info/PKG-INFO")).read_text()) assert p.pipfile["packages"]["testpipenv"]["path"] == "." assert p.pipfile["packages"]["testpipenv"]["extras"] == ["dev"] assert "six" in p.lockfile["default"] @@ -104,11 +104,10 @@ setup( """Ensure dependency_links are parsed and installed (needed for private repo dependencies). """ with temp_environ(), PipenvInstance(pypi=pypi, chdir=True) as p: - os.environ['PIP_PROCESS_DEPENDENCY_LINKS'] = '1' os.environ["PIP_NO_BUILD_ISOLATION"] = '1' TestDirectDependencies.helper_dependency_links_install_test( p, - 'test-private-dependency-v0.1@ git+https://github.com/atzannes/test-private-dependency@v0.1' + 'test-private-dependency@ git+https://github.com/atzannes/test-private-dependency@v0.1' ) @pytest.mark.needs_github_ssh @@ -118,7 +117,7 @@ setup( os.environ["PIP_NO_BUILD_ISOLATION"] = '1' TestDirectDependencies.helper_dependency_links_install_test( p, - 'test-private-dependency-v0.1@ git+ssh://git@github.com/atzannes/test-private-dependency@v0.1' + 'test-private-dependency@ git+ssh://git@github.com/atzannes/test-private-dependency@v0.1' ) diff --git a/tests/integration/test_install_uri.py b/tests/integration/test_install_uri.py index 4dd35882..eb4a5af1 100644 --- a/tests/integration/test_install_uri.py +++ b/tests/integration/test_install_uri.py @@ -66,8 +66,9 @@ def test_ssh_vcs_install(PipenvInstance, pip_src_dir, pypi): @pytest.mark.urls @pytest.mark.needs_internet @flaky -def test_urls_work(PipenvInstance, pypi, pip_src_dir): +def test_urls_work(PipenvInstance, pypi): with PipenvInstance(pypi=pypi, chdir=True) as p: + # the library this installs is "django-cms" path = p._pipfile.get_url("django", "3.4.x.zip") c = p.pipenv( "install {0}".format(path) @@ -77,7 +78,8 @@ def test_urls_work(PipenvInstance, pypi, pip_src_dir): dep = list(p.pipfile["packages"].values())[0] assert "file" in dep, p.pipfile - dep = list(p.lockfile["default"].values())[0] + # now that we handle resolution with requirementslib, this will resolve to a name + dep = p.lockfile["default"]["django-cms"] assert "file" in dep, p.lockfile diff --git a/tests/integration/test_lock.py b/tests/integration/test_lock.py index d993b1a2..f2630fc1 100644 --- a/tests/integration/test_lock.py +++ b/tests/integration/test_lock.py @@ -582,7 +582,7 @@ six = "*" f.write(contents) c = p.pipenv("lock") assert c.return_code == 0 - assert p.lockfile["default"]["six"]["index"] == "test" + # assert p.lockfile["default"]["six"]["index"] == "test" with open(p.pipfile_path, 'w') as f: f.write(contents.replace('name = "test"', 'name = "custom"')) c = p.pipenv("lock") diff --git a/tests/pypi/enum34/api.json b/tests/pypi/enum34/api.json new file mode 100644 index 00000000..64a47647 --- /dev/null +++ b/tests/pypi/enum34/api.json @@ -0,0 +1,190 @@ +{ + "info": { + "author": "Ethan Furman", + "author_email": "ethan@stoneleaf.us", + "bugtrack_url": null, + "classifiers": [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "License :: OSI Approved :: BSD License", + "Programming Language :: Python", + "Programming Language :: Python :: 2.4", + "Programming Language :: Python :: 2.5", + "Programming Language :: Python :: 2.6", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3.3", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", + "Topic :: Software Development" + ], + "description": "enum --- support for enumerations\n========================================\n\nAn enumeration is a set of symbolic names (members) bound to unique, constant\nvalues. Within an enumeration, the members can be compared by identity, and\nthe enumeration itself can be iterated over.\n\n from enum import Enum\n\n class Fruit(Enum):\n apple = 1\n banana = 2\n orange = 3\n\n list(Fruit)\n # [, , ]\n\n len(Fruit)\n # 3\n\n Fruit.banana\n # \n\n Fruit['banana']\n # \n\n Fruit(2)\n # \n\n Fruit.banana is Fruit['banana'] is Fruit(2)\n # True\n\n Fruit.banana.name\n # 'banana'\n\n Fruit.banana.value\n # 2\n\nRepository and Issue Tracker at https://bitbucket.org/stoneleaf/enum34.", + "description_content_type": null, + "docs_url": null, + "download_url": "", + "downloads": { + "last_day": -1, + "last_month": -1, + "last_week": -1 + }, + "home_page": "https://bitbucket.org/stoneleaf/enum34", + "keywords": "", + "license": "BSD License", + "maintainer": "", + "maintainer_email": "", + "name": "enum34", + "package_url": "https://pypi.org/project/enum34/", + "platform": "UNKNOWN", + "project_url": "https://pypi.org/project/enum34/", + "project_urls": { + "Homepage": "https://bitbucket.org/stoneleaf/enum34" + }, + "release_url": "https://pypi.org/project/enum34/1.1.6/", + "requires_dist": null, + "requires_python": "", + "summary": "Python 3.4 Enum backported to 3.3, 3.2, 3.1, 2.7, 2.6, 2.5, and 2.4", + "version": "1.1.6" + }, + "last_serial": 2117417, + "releases": { + "1.1.6": [ + { + "comment_text": "", + "digests": { + "md5": "68f6982cc07dde78f4b500db829860bd", + "sha256": "6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79" + }, + "downloads": -1, + "filename": "enum34-1.1.6-py2-none-any.whl", + "has_sig": false, + "md5_digest": "68f6982cc07dde78f4b500db829860bd", + "packagetype": "bdist_wheel", + "python_version": "py2", + "requires_python": null, + "size": 12427, + "upload_time": "2016-05-16T03:31:13", + "url": "https://files.pythonhosted.org/packages/c5/db/e56e6b4bbac7c4a06de1c50de6fe1ef3810018ae11732a50f15f62c7d050/enum34-1.1.6-py2-none-any.whl" + }, + { + "comment_text": "", + "digests": { + "md5": "a63ecb4f0b1b85fb69be64bdea999b43", + "sha256": "644837f692e5f550741432dd3f223bbb9852018674981b1664e5dc339387588a" + }, + "downloads": -1, + "filename": "enum34-1.1.6-py3-none-any.whl", + "has_sig": false, + "md5_digest": "a63ecb4f0b1b85fb69be64bdea999b43", + "packagetype": "bdist_wheel", + "python_version": "py3", + "requires_python": null, + "size": 12428, + "upload_time": "2016-05-16T03:31:19", + "url": "https://files.pythonhosted.org/packages/af/42/cb9355df32c69b553e72a2e28daee25d1611d2c0d9c272aa1d34204205b2/enum34-1.1.6-py3-none-any.whl" + }, + { + "comment_text": "", + "digests": { + "md5": "5f13a0841a61f7fc295c514490d120d0", + "sha256": "8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1" + }, + "downloads": -1, + "filename": "enum34-1.1.6.tar.gz", + "has_sig": false, + "md5_digest": "5f13a0841a61f7fc295c514490d120d0", + "packagetype": "sdist", + "python_version": "source", + "requires_python": null, + "size": 40048, + "upload_time": "2016-05-16T03:31:30", + "url": "https://files.pythonhosted.org/packages/bf/3e/31d502c25302814a7c2f1d3959d2a3b3f78e509002ba91aea64993936876/enum34-1.1.6.tar.gz" + }, + { + "comment_text": "", + "digests": { + "md5": "61ad7871532d4ce2d77fac2579237a9e", + "sha256": "2d81cbbe0e73112bdfe6ef8576f2238f2ba27dd0d55752a776c41d38b7da2850" + }, + "downloads": -1, + "filename": "enum34-1.1.6.zip", + "has_sig": false, + "md5_digest": "61ad7871532d4ce2d77fac2579237a9e", + "packagetype": "sdist", + "python_version": "source", + "requires_python": null, + "size": 44773, + "upload_time": "2016-05-16T03:31:48", + "url": "https://files.pythonhosted.org/packages/e8/26/a6101edcf724453845c850281b96b89a10dac6bd98edebc82634fccce6a5/enum34-1.1.6.zip" + } + ] + }, + "urls": [ + { + "comment_text": "", + "digests": { + "md5": "68f6982cc07dde78f4b500db829860bd", + "sha256": "6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79" + }, + "downloads": -1, + "filename": "enum34-1.1.6-py2-none-any.whl", + "has_sig": false, + "md5_digest": "68f6982cc07dde78f4b500db829860bd", + "packagetype": "bdist_wheel", + "python_version": "py2", + "requires_python": null, + "size": 12427, + "upload_time": "2016-05-16T03:31:13", + "url": "https://files.pythonhosted.org/packages/c5/db/e56e6b4bbac7c4a06de1c50de6fe1ef3810018ae11732a50f15f62c7d050/enum34-1.1.6-py2-none-any.whl" + }, + { + "comment_text": "", + "digests": { + "md5": "a63ecb4f0b1b85fb69be64bdea999b43", + "sha256": "644837f692e5f550741432dd3f223bbb9852018674981b1664e5dc339387588a" + }, + "downloads": -1, + "filename": "enum34-1.1.6-py3-none-any.whl", + "has_sig": false, + "md5_digest": "a63ecb4f0b1b85fb69be64bdea999b43", + "packagetype": "bdist_wheel", + "python_version": "py3", + "requires_python": null, + "size": 12428, + "upload_time": "2016-05-16T03:31:19", + "url": "https://files.pythonhosted.org/packages/af/42/cb9355df32c69b553e72a2e28daee25d1611d2c0d9c272aa1d34204205b2/enum34-1.1.6-py3-none-any.whl" + }, + { + "comment_text": "", + "digests": { + "md5": "5f13a0841a61f7fc295c514490d120d0", + "sha256": "8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1" + }, + "downloads": -1, + "filename": "enum34-1.1.6.tar.gz", + "has_sig": false, + "md5_digest": "5f13a0841a61f7fc295c514490d120d0", + "packagetype": "sdist", + "python_version": "source", + "requires_python": null, + "size": 40048, + "upload_time": "2016-05-16T03:31:30", + "url": "https://files.pythonhosted.org/packages/bf/3e/31d502c25302814a7c2f1d3959d2a3b3f78e509002ba91aea64993936876/enum34-1.1.6.tar.gz" + }, + { + "comment_text": "", + "digests": { + "md5": "61ad7871532d4ce2d77fac2579237a9e", + "sha256": "2d81cbbe0e73112bdfe6ef8576f2238f2ba27dd0d55752a776c41d38b7da2850" + }, + "downloads": -1, + "filename": "enum34-1.1.6.zip", + "has_sig": false, + "md5_digest": "61ad7871532d4ce2d77fac2579237a9e", + "packagetype": "sdist", + "python_version": "source", + "requires_python": null, + "size": 44773, + "upload_time": "2016-05-16T03:31:48", + "url": "https://files.pythonhosted.org/packages/e8/26/a6101edcf724453845c850281b96b89a10dac6bd98edebc82634fccce6a5/enum34-1.1.6.zip" + } + ] +} \ No newline at end of file diff --git a/tests/pypi/enum34/enum34-1.1.6-py2-none-any.whl b/tests/pypi/enum34/enum34-1.1.6-py2-none-any.whl new file mode 100644 index 00000000..12be7c7e Binary files /dev/null and b/tests/pypi/enum34/enum34-1.1.6-py2-none-any.whl differ diff --git a/tests/pypi/enum34/enum34-1.1.6-py3-none-any.whl b/tests/pypi/enum34/enum34-1.1.6-py3-none-any.whl new file mode 100644 index 00000000..53c1cb04 Binary files /dev/null and b/tests/pypi/enum34/enum34-1.1.6-py3-none-any.whl differ diff --git a/tests/pypi/plette/plette-0.2.2-py2.py3-none-any.whl b/tests/pypi/plette/plette-0.2.2-py2.py3-none-any.whl new file mode 100644 index 00000000..1ca43416 Binary files /dev/null and b/tests/pypi/plette/plette-0.2.2-py2.py3-none-any.whl differ diff --git a/tests/pypi/tomlkit/api.json b/tests/pypi/tomlkit/api.json new file mode 100644 index 00000000..c19763af --- /dev/null +++ b/tests/pypi/tomlkit/api.json @@ -0,0 +1,123 @@ +{ + "info": { + "author": "S\u00e9bastien Eustace", + "author_email": "sebastien@eustace.io", + "bugtrack_url": null, + "classifiers": [ + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7" + ], + "description": "[github_release]: https://img.shields.io/github/release/sdispater/tomlkit.svg?logo=github&logoColor=white\n[pypi_version]: https://img.shields.io/pypi/v/tomlkit.svg?logo=python&logoColor=white\n[python_versions]: https://img.shields.io/pypi/pyversions/tomlkit.svg?logo=python&logoColor=white\n[github_license]: https://img.shields.io/github/license/sdispater/tomlkit.svg?logo=github&logoColor=white\n[travisci]: https://img.shields.io/travis/com/sdispater/tomlkit/master.svg?logo=travis&logoColor=white&label=Travis%20CI\n[appveyor]: https://img.shields.io/appveyor/ci/sdispater/tomlkit/master.svg?logo=appveyor&logoColor=white&label=AppVeyor\n\n[codecov]: https://img.shields.io/codecov/c/github/sdispater/tomlkit/master.svg?logo=data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iNTAiIGhlaWdodD0iNDgiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+CgogPGc+CiAgPHRpdGxlPmJhY2tncm91bmQ8L3RpdGxlPgogIDxyZWN0IGZpbGw9Im5vbmUiIGlkPSJjYW52YXNfYmFja2dyb3VuZCIgaGVpZ2h0PSI0MDIiIHdpZHRoPSI1ODIiIHk9Ii0xIiB4PSItMSIvPgogPC9nPgogPGc+CiAgPHRpdGxlPkxheWVyIDE8L3RpdGxlPgogIDxwYXRoIGlkPSJzdmdfMSIgZmlsbC1ydWxlPSJldmVub2RkIiBmaWxsPSIjZmZmZmZmIiBkPSJtMjUuMDE0LDBjLTEzLjc4NCwwLjAxIC0yNS4wMDQsMTEuMTQ5IC0yNS4wMTQsMjQuODMybDAsMC4wNjJsNC4yNTQsMi40ODJsMC4wNTgsLTAuMDM5YTEyLjIzOCwxMi4yMzggMCAwIDEgOS4wNzgsLTEuOTI4YTExLjg0NCwxMS44NDQgMCAwIDEgNS45OCwyLjk3NWwwLjczLDAuNjhsMC40MTMsLTAuOTA0YzAuNCwtMC44NzQgMC44NjIsLTEuNjk2IDEuMzc0LC0yLjQ0M2MwLjIwNiwtMC4zIDAuNDMzLC0wLjYwNCAwLjY5MiwtMC45MjlsMC40MjcsLTAuNTM1bC0wLjUyNiwtMC40NGExNy40NSwxNy40NSAwIDAgMCAtOC4xLC0zLjc4MWExNy44NTMsMTcuODUzIDAgMCAwIC04LjM3NSwwLjQ5YzIuMDIzLC04Ljg2OCA5LjgyLC0xNS4wNSAxOS4wMjcsLTE1LjA1N2M1LjE5NSwwIDEwLjA3OCwyLjAwNyAxMy43NTIsNS42NTJjMi42MTksMi41OTggNC40MjIsNS44MzUgNS4yMjQsOS4zNzJhMTcuOTA4LDE3LjkwOCAwIDAgMCAtNS4yMDgsLTAuNzlsLTAuMzE4LC0wLjAwMWExOC4wOTYsMTguMDk2IDAgMCAwIC0yLjA2NywwLjE1M2wtMC4wODcsMC4wMTJjLTAuMzAzLDAuMDQgLTAuNTcsMC4wODEgLTAuODEzLDAuMTI2Yy0wLjExOSwwLjAyIC0wLjIzNywwLjA0NSAtMC4zNTUsMC4wNjhjLTAuMjgsMC4wNTcgLTAuNTU0LDAuMTE5IC0wLjgxNiwwLjE4NWwtMC4yODgsMC4wNzNjLTAuMzM2LDAuMDkgLTAuNjc1LDAuMTkxIC0xLjAwNiwwLjNsLTAuMDYxLDAuMDJjLTAuNzQsMC4yNTEgLTEuNDc4LDAuNTU4IC0yLjE5LDAuOTE0bC0wLjA1NywwLjAyOWMtMC4zMTYsMC4xNTggLTAuNjM2LDAuMzMzIC0wLjk3OCwwLjUzNGwtMC4wNzUsMC4wNDVhMTYuOTcsMTYuOTcgMCAwIDAgLTQuNDE0LDMuNzhsLTAuMTU3LDAuMTkxYy0wLjMxNywwLjM5NCAtMC41NjcsMC43MjcgLTAuNzg3LDEuMDQ4Yy0wLjE4NCwwLjI3IC0wLjM2OSwwLjU2IC0wLjYsMC45NDJsLTAuMTI2LDAuMjE3Yy0wLjE4NCwwLjMxOCAtMC4zNDgsMC42MjIgLTAuNDg3LDAuOWwtMC4wMzMsMC4wNjFjLTAuMzU0LDAuNzExIC0wLjY2MSwxLjQ1NSAtMC45MTcsMi4yMTRsLTAuMDM2LDAuMTExYTE3LjEzLDE3LjEzIDAgMCAwIC0wLjg1NSw1LjY0NGwwLjAwMywwLjIzNGEyMy41NjUsMjMuNTY1IDAgMCAwIDAuMDQzLDAuODIyYzAuMDEsMC4xMyAwLjAyMywwLjI1OSAwLjAzNiwwLjM4OGMwLjAxNSwwLjE1OCAwLjAzNCwwLjMxNiAwLjA1MywwLjQ3MWwwLjAxMSwwLjA4OGwwLjAyOCwwLjIxNGMwLjAzNywwLjI2NCAwLjA4LDAuNTI1IDAuMTMsMC43ODdjMC41MDMsMi42MzcgMS43Niw1LjI3NCAzLjYzNSw3LjYyNWwwLjA4NSwwLjEwNmwwLjA4NywtMC4xMDRjMC43NDgsLTAuODg0IDIuNjAzLC0zLjY4NyAyLjc2LC01LjM2OWwwLjAwMywtMC4wMzFsLTAuMDE1LC0wLjAyOGExMS43MzYsMTEuNzM2IDAgMCAxIC0xLjMzMywtNS40MDdjMCwtNi4yODQgNC45NCwtMTEuNTAyIDExLjI0MywtMTEuODhsMC40MTQsLTAuMDE1YzIuNTYxLC0wLjA1OCA1LjA2NCwwLjY3MyA3LjIzLDIuMTM2bDAuMDU4LDAuMDM5bDQuMTk3LC0yLjQ0bDAuMDU1LC0wLjAzM2wwLC0wLjA2MmMwLjAwNiwtNi42MzIgLTIuNTkyLC0xMi44NjUgLTcuMzE0LC0xNy41NTFjLTQuNzE2LC00LjY3OSAtMTAuOTkxLC03LjI1NSAtMTcuNjcyLC03LjI1NSIvPgogPC9nPgo8L3N2Zz4=&label=Codecov\n\n[![GitHub Release][github_release]](https://github.com/sdispater/tomlkit/releases/)\n[![PyPI Version][pypi_version]](https://pypi.python.org/pypi/tomlkit/)\n[![Python Versions][python_versions]](https://pypi.python.org/pypi/tomlkit/)\n[![License][github_license]](https://github.com/sdispater/tomlkit/blob/master/LICENSE)\n
\n[![Travis CI][travisci]](https://travis-ci.com/sdispater/tomlkit)\n[![AppVeyor][appveyor]](https://ci.appveyor.com/project/sdispater/tomlkit)\n[![Codecov][codecov]](https://codecov.io/gh/sdispater/tomlkit)\n\n# TOML Kit - Style-preserving TOML library for Python\n\nTOML Kit is a **0.5.0-compliant** [TOML](https://github.com/toml-lang/toml) library.\n\nIt includes a parser that preserves all comments, indentations, whitespace and internal element ordering,\nand makes them accessible and editable via an intuitive API.\n\nYou can also create new TOML documents from scratch using the provided helpers.\n\nPart of the implementation as been adapted, improved and fixed from [Molten](https://github.com/LeopoldArkham/Molten).\n\n## Usage\n\n### Parsing\n\nTOML Kit comes with a fast and style-preserving parser to help you access\nthe content of TOML files and strings.\n\n```python\n>>> from tomlkit import dumps\n>>> from tomlkit import parse # you can also use loads\n\n>>> content = \"\"\"[table]\n... foo = \"bar\" # String\n... \"\"\"\n>>> doc = parse(content)\n\n# doc is a TOMLDocument instance that holds all the information\n# about the TOML string.\n# It behaves like a standard dictionary.\n\n>>> assert doc[\"table\"][\"foo\"] == \"bar\"\n\n# The string generated from the document is exactly the same\n# as the original string\n>>> assert dumps(doc) == content\n```\n\n### Modifying\n\nTOML Kit provides an intuitive API to modify TOML documents.\n\n```python\n>>> from tomlkit import dumps\n>>> from tomlkit import parse\n>>> from tomlkit import table\n\n>>> doc = parse(\"\"\"[table]\n... foo = \"bar\" # String\n... \"\"\")\n\n>>> doc[\"table\"][\"baz\"] = 13\n\n>>> dumps(doc)\n\"\"\"[table]\nfoo = \"bar\" # String\nbaz = 13\n\"\"\"\n\n# Add a new table\n>>> tab = table()\n>>> tab.add(\"array\", [1, 2, 3])\n\n>>> doc[\"table2\"] = tab\n\n>>> dumps(doc)\n\"\"\"[table]\nfoo = \"bar\" # String\nbaz = 13\n\n[table2]\narray = [1, 2, 3]\n\"\"\"\n\n# Remove the newly added table\n>>> doc.remove(\"table2\")\n# del doc[\"table2] is also possible\n```\n\n### Writing\n\nYou can also write a new TOML document from scratch.\n\nLet's say we want to create this following document:\n\n```toml\n# This is a TOML document.\n\ntitle = \"TOML Example\"\n\n[owner]\nname = \"Tom Preston-Werner\"\norganization = \"GitHub\"\nbio = \"GitHub Cofounder & CEO\\nLikes tater tots and beer.\"\ndob = 1979-05-27T07:32:00Z # First class dates? Why not?\n\n[database]\nserver = \"192.168.1.1\"\nports = [ 8001, 8001, 8002 ]\nconnection_max = 5000\nenabled = true\n```\n\nIt can be created with the following code:\n\n```python\n>>> from tomlkit import comment\n>>> from tomlkit import document\n>>> from tomlkit import nl\n>>> from tomlkit import table\n\n>>> doc = document()\n>>> doc.add(comment(\"This is a TOML document.\"))\n>>> doc.add(nl())\n>>> doc.add(\"title\", \"TOML Example\")\n# Using doc[\"title\"] = \"TOML Example\" is also possible\n\n>>> owner = table()\n>>> owner.add(\"name\", \"Tom Preston-Werner\")\n>>> owner.add(\"organization\", \"GitHub\")\n>>> owner.add(\"bio\", \"GitHub Cofounder & CEO\\nLikes tater tots and beer.\")\n>>> owner.add(\"dob\", datetime(1979, 5, 27, 7, 32, tzinfo=utc))\n>>> owner[\"dob\"].comment(\"First class dates? Why not?\")\n\n# Adding the table to the document\n>>> doc.add(\"owner\", owner)\n\n>>> database = table()\n>>> database[\"server\"] = \"192.168.1.1\"\n>>> database[\"ports\"] = [8001, 8001, 8002]\n>>> database[\"connection_max\"] = 5000\n>>> database[\"enabled\"] = True\n\n>>> doc[\"database\"] = database\n```\n\n\n## Installation\n\nIf you are using [Poetry](https://poetry.eustace.io),\nadd `tomlkit` to your `pyproject.toml` file by using:\n\n```bash\npoetry add tomlkit\n```\n\nIf not, you can use `pip`:\n\n```bash\npip install tomlkit\n```\n", + "description_content_type": "text/markdown", + "docs_url": null, + "download_url": "", + "downloads": { + "last_day": -1, + "last_month": -1, + "last_week": -1 + }, + "home_page": "https://github.com/sdispater/tomlkit", + "keywords": "", + "license": "MIT", + "maintainer": "S\u00e9bastien Eustace", + "maintainer_email": "sebastien@eustace.io", + "name": "tomlkit", + "package_url": "https://pypi.org/project/tomlkit/", + "platform": "", + "project_url": "https://pypi.org/project/tomlkit/", + "project_urls": { + "Homepage": "https://github.com/sdispater/tomlkit", + "Repository": "https://github.com/sdispater/tomlkit" + }, + "release_url": "https://pypi.org/project/tomlkit/0.5.3/", + "requires_dist": [ + "enum34 (>=1.1,<2.0); python_version >= \"2.7\" and python_version < \"2.8\"", + "functools32 (>=3.2.3,<4.0.0); python_version >= \"2.7\" and python_version < \"2.8\"", + "typing (>=3.6,<4.0); python_version >= \"2.7\" and python_version < \"2.8\" or python_version >= \"3.4\" and python_version < \"3.5\"" + ], + "requires_python": ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*", + "summary": "Style preserving TOML library", + "version": "0.5.3" + }, + "last_serial": 4504211, + "releases": { + "0.5.3": [ + { + "comment_text": "", + "digests": { + "md5": "0a6cf417df5d0fc911f89447c9a662a9", + "sha256": "f077456d35303e7908cc233b340f71e0bec96f63429997f38ca9272b7d64029e" + }, + "downloads": -1, + "filename": "tomlkit-0.5.3-py2.py3-none-any.whl", + "has_sig": false, + "md5_digest": "0a6cf417df5d0fc911f89447c9a662a9", + "packagetype": "bdist_wheel", + "python_version": "py2.py3", + "requires_python": ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*", + "size": 116796, + "upload_time": "2018-11-19T20:05:37", + "url": "https://files.pythonhosted.org/packages/71/c6/06c014b92cc48270765d6a9418d82239b158d8a9b69e031b0e2c6598740b/tomlkit-0.5.3-py2.py3-none-any.whl" + }, + { + "comment_text": "", + "digests": { + "md5": "a708470b53d689013f2fc9f0a7902adf", + "sha256": "d6506342615d051bc961f70bfcfa3d29b6616cc08a3ddfd4bc24196f16fd4ec2" + }, + "downloads": -1, + "filename": "tomlkit-0.5.3.tar.gz", + "has_sig": false, + "md5_digest": "a708470b53d689013f2fc9f0a7902adf", + "packagetype": "sdist", + "python_version": "source", + "requires_python": ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*", + "size": 29864, + "upload_time": "2018-11-19T20:05:39", + "url": "https://files.pythonhosted.org/packages/f7/f7/bbd9213bfe76cb7821c897f9ed74877fd74993b4ca2fe9513eb5a31030f9/tomlkit-0.5.3.tar.gz" + } + ] + }, + "urls": [ + { + "comment_text": "", + "digests": { + "md5": "0a6cf417df5d0fc911f89447c9a662a9", + "sha256": "f077456d35303e7908cc233b340f71e0bec96f63429997f38ca9272b7d64029e" + }, + "downloads": -1, + "filename": "tomlkit-0.5.3-py2.py3-none-any.whl", + "has_sig": false, + "md5_digest": "0a6cf417df5d0fc911f89447c9a662a9", + "packagetype": "bdist_wheel", + "python_version": "py2.py3", + "requires_python": ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*", + "size": 116796, + "upload_time": "2018-11-19T20:05:37", + "url": "https://files.pythonhosted.org/packages/71/c6/06c014b92cc48270765d6a9418d82239b158d8a9b69e031b0e2c6598740b/tomlkit-0.5.3-py2.py3-none-any.whl" + }, + { + "comment_text": "", + "digests": { + "md5": "a708470b53d689013f2fc9f0a7902adf", + "sha256": "d6506342615d051bc961f70bfcfa3d29b6616cc08a3ddfd4bc24196f16fd4ec2" + }, + "downloads": -1, + "filename": "tomlkit-0.5.3.tar.gz", + "has_sig": false, + "md5_digest": "a708470b53d689013f2fc9f0a7902adf", + "packagetype": "sdist", + "python_version": "source", + "requires_python": ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*", + "size": 29864, + "upload_time": "2018-11-19T20:05:39", + "url": "https://files.pythonhosted.org/packages/f7/f7/bbd9213bfe76cb7821c897f9ed74877fd74993b4ca2fe9513eb5a31030f9/tomlkit-0.5.3.tar.gz" + } + ] +} \ No newline at end of file diff --git a/tests/pypi/tomlkit/tomlkit-0.5.3-py2.py3-none-any.whl b/tests/pypi/tomlkit/tomlkit-0.5.3-py2.py3-none-any.whl new file mode 100644 index 00000000..7375595e Binary files /dev/null and b/tests/pypi/tomlkit/tomlkit-0.5.3-py2.py3-none-any.whl differ diff --git a/tests/pypi/typing/api.json b/tests/pypi/typing/api.json new file mode 100644 index 00000000..df641aa8 --- /dev/null +++ b/tests/pypi/typing/api.json @@ -0,0 +1,153 @@ +{ + "info": { + "author": "Guido van Rossum, Jukka Lehtosalo, \u0141ukasz Langa, Ivan Levkivskyi", + "author_email": "jukka.lehtosalo@iki.fi", + "bugtrack_url": null, + "classifiers": [ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Intended Audience :: Developers", + "License :: OSI Approved :: Python Software Foundation License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3.3", + "Programming Language :: Python :: 3.4", + "Topic :: Software Development" + ], + "description": "Typing -- Type Hints for Python\n\nThis is a backport of the standard library typing module to Python\nversions older than 3.5. (See note below for newer versions.)\n\nTyping defines a standard notation for Python function and variable\ntype annotations. The notation can be used for documenting code in a\nconcise, standard format, and it has been designed to also be used by\nstatic and runtime type checkers, static analyzers, IDEs and other\ntools.\n\nNOTE: in Python 3.5 and later, the typing module lives in the stdlib,\nand installing this package has NO EFFECT. To get a newer version of\nthe typing module in Python 3.5 or later, you have to upgrade to a\nnewer Python (bugfix) version. For example, typing in Python 3.6.0 is\nmissing the definition of 'Type' -- upgrading to 3.6.2 will fix this.\n\nAlso note that most improvements to the typing module in Python 3.7\nwill not be included in this package, since Python 3.7 has some\nbuilt-in support that is not present in older versions (See PEP 560.)\n\n\n", + "description_content_type": "", + "docs_url": null, + "download_url": "", + "downloads": { + "last_day": -1, + "last_month": -1, + "last_week": -1 + }, + "home_page": "https://docs.python.org/3/library/typing.html", + "keywords": "typing function annotations type hints hinting checking checker typehints typehinting typechecking backport", + "license": "PSF", + "maintainer": "", + "maintainer_email": "", + "name": "typing", + "package_url": "https://pypi.org/project/typing/", + "platform": "", + "project_url": "https://pypi.org/project/typing/", + "project_urls": { + "Homepage": "https://docs.python.org/3/library/typing.html" + }, + "release_url": "https://pypi.org/project/typing/3.6.6/", + "requires_dist": null, + "requires_python": "", + "summary": "Type Hints for Python", + "version": "3.6.6" + }, + "last_serial": 4208967, + "releases": { + "3.6.6": [ + { + "comment_text": "", + "digests": { + "md5": "7e50dcc98a528f47c8793c980128467c", + "sha256": "a4c8473ce11a65999c8f59cb093e70686b6c84c98df58c1dae9b3b196089858a" + }, + "downloads": -1, + "filename": "typing-3.6.6-py2-none-any.whl", + "has_sig": false, + "md5_digest": "7e50dcc98a528f47c8793c980128467c", + "packagetype": "bdist_wheel", + "python_version": "py2", + "requires_python": null, + "size": 23560, + "upload_time": "2018-08-26T18:46:05", + "url": "https://files.pythonhosted.org/packages/cc/3e/29f92b7aeda5b078c86d14f550bf85cff809042e3429ace7af6193c3bc9f/typing-3.6.6-py2-none-any.whl" + }, + { + "comment_text": "", + "digests": { + "md5": "0c84fda6fd4303fa6aee13a36ea62897", + "sha256": "57dcf675a99b74d64dacf6fba08fb17cf7e3d5fdff53d4a30ea2a5e7e52543d4" + }, + "downloads": -1, + "filename": "typing-3.6.6-py3-none-any.whl", + "has_sig": false, + "md5_digest": "0c84fda6fd4303fa6aee13a36ea62897", + "packagetype": "bdist_wheel", + "python_version": "py3", + "requires_python": null, + "size": 25727, + "upload_time": "2018-08-26T18:46:06", + "url": "https://files.pythonhosted.org/packages/4a/bd/eee1157fc2d8514970b345d69cb9975dcd1e42cd7e61146ed841f6e68309/typing-3.6.6-py3-none-any.whl" + }, + { + "comment_text": "", + "digests": { + "md5": "64614206b4bdc0864fc0e0bccd69efc9", + "sha256": "4027c5f6127a6267a435201981ba156de91ad0d1d98e9ddc2aa173453453492d" + }, + "downloads": -1, + "filename": "typing-3.6.6.tar.gz", + "has_sig": false, + "md5_digest": "64614206b4bdc0864fc0e0bccd69efc9", + "packagetype": "sdist", + "python_version": "source", + "requires_python": null, + "size": 71799, + "upload_time": "2018-08-26T18:46:08", + "url": "https://files.pythonhosted.org/packages/bf/9b/2bf84e841575b633d8d91ad923e198a415e3901f228715524689495b4317/typing-3.6.6.tar.gz" + } + ] + }, + "urls": [ + { + "comment_text": "", + "digests": { + "md5": "7e50dcc98a528f47c8793c980128467c", + "sha256": "a4c8473ce11a65999c8f59cb093e70686b6c84c98df58c1dae9b3b196089858a" + }, + "downloads": -1, + "filename": "typing-3.6.6-py2-none-any.whl", + "has_sig": false, + "md5_digest": "7e50dcc98a528f47c8793c980128467c", + "packagetype": "bdist_wheel", + "python_version": "py2", + "requires_python": null, + "size": 23560, + "upload_time": "2018-08-26T18:46:05", + "url": "https://files.pythonhosted.org/packages/cc/3e/29f92b7aeda5b078c86d14f550bf85cff809042e3429ace7af6193c3bc9f/typing-3.6.6-py2-none-any.whl" + }, + { + "comment_text": "", + "digests": { + "md5": "0c84fda6fd4303fa6aee13a36ea62897", + "sha256": "57dcf675a99b74d64dacf6fba08fb17cf7e3d5fdff53d4a30ea2a5e7e52543d4" + }, + "downloads": -1, + "filename": "typing-3.6.6-py3-none-any.whl", + "has_sig": false, + "md5_digest": "0c84fda6fd4303fa6aee13a36ea62897", + "packagetype": "bdist_wheel", + "python_version": "py3", + "requires_python": null, + "size": 25727, + "upload_time": "2018-08-26T18:46:06", + "url": "https://files.pythonhosted.org/packages/4a/bd/eee1157fc2d8514970b345d69cb9975dcd1e42cd7e61146ed841f6e68309/typing-3.6.6-py3-none-any.whl" + }, + { + "comment_text": "", + "digests": { + "md5": "64614206b4bdc0864fc0e0bccd69efc9", + "sha256": "4027c5f6127a6267a435201981ba156de91ad0d1d98e9ddc2aa173453453492d" + }, + "downloads": -1, + "filename": "typing-3.6.6.tar.gz", + "has_sig": false, + "md5_digest": "64614206b4bdc0864fc0e0bccd69efc9", + "packagetype": "sdist", + "python_version": "source", + "requires_python": null, + "size": 71799, + "upload_time": "2018-08-26T18:46:08", + "url": "https://files.pythonhosted.org/packages/bf/9b/2bf84e841575b633d8d91ad923e198a415e3901f228715524689495b4317/typing-3.6.6.tar.gz" + } + ] +} \ No newline at end of file diff --git a/tests/pypi/typing/typing-3.6.6-py2-none-any.whl b/tests/pypi/typing/typing-3.6.6-py2-none-any.whl new file mode 100644 index 00000000..a6dc8ab1 Binary files /dev/null and b/tests/pypi/typing/typing-3.6.6-py2-none-any.whl differ diff --git a/tests/pypi/typing/typing-3.6.6-py3-none-any.whl b/tests/pypi/typing/typing-3.6.6-py3-none-any.whl new file mode 100644 index 00000000..f38e8528 Binary files /dev/null and b/tests/pypi/typing/typing-3.6.6-py3-none-any.whl differ diff --git a/tests/pytest-pypi/pytest_pypi/app.py b/tests/pytest-pypi/pytest_pypi/app.py index 4e013ab5..fa494ea8 100644 --- a/tests/pytest-pypi/pytest_pypi/app.py +++ b/tests/pytest-pypi/pytest_pypi/app.py @@ -1,5 +1,6 @@ import os import json +import io import sys import requests @@ -29,7 +30,26 @@ class Package(object): with open(os.path.join(path, 'api.json')) as f: return json.load(f) except FileNotFoundError: - pass + r = session.get('https://pypi.org/pypi/{0}/json'.format(self.name)) + response = r.json() + releases = response["releases"] + files = { + pkg for pkg_dir in self._package_dirs + for pkg in os.listdir(pkg_dir) + } + for release in list(releases.keys()): + values = ( + r for r in releases[release] if r["filename"] in files + ) + values = list(values) + if values: + releases[release] = values + else: + del releases[release] + response["releases"] = releases + with io.open(os.path.join(path, "api.json"), "w") as fh: + json.dump(response, fh, indent=4) + return response def __repr__(self): return "/json') def json_for_package(package): - try: - return jsonify(packages[package].json) - except Exception: - pass - - r = session.get('https://pypi.org/pypi/{0}/json'.format(package)) - return jsonify(r.json()) + return jsonify(packages[package].json) + # try: + # except Exception: + # r = session.get('https://pypi.org/pypi/{0}/json'.format(package)) + # return jsonify(r.json()) if __name__ == '__main__': diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index ac5eb29b..a2f3dd56 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -75,12 +75,20 @@ DEP_PIP_PAIRS = [ ] +def mock_unpack(link, source_dir, download_dir, only_download=False, session=None, + hashes=None, progress_bar="off"): + return + + @pytest.mark.utils @pytest.mark.parametrize("deps, expected", DEP_PIP_PAIRS) -def test_convert_deps_to_pip(deps, expected): - if expected.startswith("Django"): - expected = expected.lower() - assert pipenv.utils.convert_deps_to_pip(deps, r=False) == [expected] +def test_convert_deps_to_pip(monkeypatch, deps, expected): + with monkeypatch.context() as m: + import pip_shims + m.setattr(pip_shims.shims, "unpack_url", mock_unpack) + if expected.startswith("Django"): + expected = expected.lower() + assert pipenv.utils.convert_deps_to_pip(deps, r=False) == [expected] @pytest.mark.utils @@ -121,8 +129,11 @@ def test_convert_deps_to_pip(deps, expected): ), ], ) -def test_convert_deps_to_pip_one_way(deps, expected): - assert pipenv.utils.convert_deps_to_pip(deps, r=False) == [expected.lower()] +def test_convert_deps_to_pip_one_way(monkeypatch, deps, expected): + with monkeypatch.context() as m: + import pip_shims + m.setattr(pip_shims.shims, "unpack_url", mock_unpack) + assert pipenv.utils.convert_deps_to_pip(deps, r=False) == [expected.lower()] @pytest.mark.skipif(isinstance(u"", str), reason="don't need to test if unicode is str")