diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 48175a72..46b2e799 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,4 +1,5 @@ steps: - label: ":python:" commands: + # - make - ./run-tests.sh \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index af7ab55b..d7fa45c6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -3,8 +3,8 @@ Before opening any issues or proposing any pull requests, please do the following: -1. Read our [Contributor's Guide](http://docs.pipenv.org/en/latest/dev/contributing/). -2. Understand our [development philosophy](http://docs.pipenv.org/en/latest/dev/philosophy/). +1. Read our [Contributor's Guide](https://docs.pipenv.org/dev/contributing/). +2. Understand our [development philosophy](https://docs.pipenv.org/dev/philosophy/). To get the greatest chance of helpful responses, please also observe the following additional notes. diff --git a/Dockerfile b/Dockerfile index 4bab57e2..ccf35a6c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,7 @@ -FROM ubuntu:17.10 +FROM ubuntu:18.04 # -- Install Pipenv: -RUN apt-get update \ - && apt-get install software-properties-common python-software-properties -y \ - && add-apt-repository ppa:pypa/ppa -y \ - && apt-get update \ - && apt-get install git pipenv -y +RUN apt update && apt install python3-pip -y && pip3 install pipenv ENV LC_ALL C.UTF-8 ENV LANG C.UTF-8 diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..51b267c5 --- /dev/null +++ b/Makefile @@ -0,0 +1,2 @@ +test: + docker-compose up \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..1fede25a --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,7 @@ +version: '3' +services: + pipenv-tests: + image: kennethreitz/pipenv-tests + command: bash /pipenv/run-tests.sh + volumes: + - .:/pipenv diff --git a/pipenv/_compat.py b/pipenv/_compat.py index 14326813..8ccf025a 100644 --- a/pipenv/_compat.py +++ b/pipenv/_compat.py @@ -5,6 +5,7 @@ Exposes a standard API that enables compatibility across python versions, operating systems, etc. """ import functools +import importlib import io import os import six @@ -50,6 +51,24 @@ if six.PY2: class ResourceWarning(Warning): pass +# -*- coding=utf-8 -*- + + +def pip_import(module_path, subimport=None, old_path=None): + internal = 'pip._internal.{0}'.format(module_path) + old_path = old_path or module_path + pip9 = 'pip.{0}'.format(old_path) + try: + _tmp = importlib.import_module(internal) + except ImportError: + _tmp = importlib.import_module(pip9) + if subimport: + return getattr(_tmp, subimport, _tmp) + return _tmp + + +vcs = pip_import('vcs', 'VcsSupport') + class TemporaryDirectory(object): """Create and return a temporary directory. This has the same diff --git a/pipenv/core.py b/pipenv/core.py index 6731de2c..5ad0cd25 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -16,6 +16,7 @@ import crayons import dotenv import delegator from .vendor import pexpect +from first import first import pipfile from blindspin import spinner from requests.packages import urllib3 @@ -46,9 +47,11 @@ from .utils import ( is_star, rmtree, split_argument, + extract_uri_from_vcs_dep, ) from ._compat import ( TemporaryDirectory, + vcs ) from .import pep508checker, progress from .environments import ( @@ -999,6 +1002,7 @@ def do_lock( ): """Executes the freeze functionality.""" from notpip._vendor.distlib.markers import Evaluator + from .utils import get_vcs_deps allowed_marker_keys = ['markers'] + [k for k in Evaluator.allowed_values.keys()] cached_lockfile = {} if not pre: @@ -1035,6 +1039,9 @@ def do_lock( if dev_package in project.packages: dev_packages[dev_package] = project.packages[dev_package] # Resolve dev-package dependencies, with pip-tools. + pip_freeze = delegator.run( + '{0} freeze'.format(escape_grouped_arguments(which_pip(allow_global=system))) + ).out deps = convert_deps_to_pip( dev_packages, project, r=False, include_index=True ) @@ -1066,24 +1073,14 @@ def do_lock( lockfile['develop'][dep['name']]['markers'] = dep['markers'] # Add refs for VCS installs. # TODO: be smarter about this. - vcs_deps = convert_deps_to_pip(project.vcs_dev_packages, project, r=False) - pip_freeze = delegator.run( - '{0} freeze'.format(escape_grouped_arguments(which_pip(allow_global=system))) - ).out - if vcs_deps: - for line in pip_freeze.strip().split('\n'): - # if the line doesn't match a vcs dependency in the Pipfile, - # ignore it - if not any(dep in line for dep in vcs_deps): - continue - - try: - installed = convert_deps_from_pip(line) - name = list(installed.keys())[0] - if is_vcs(installed[name]): - lockfile['develop'].update(installed) - except IndexError: - pass + vcs_dev_lines, vcs_dev_lockfiles = get_vcs_deps(project, pip_freeze, which=which, verbose=verbose, clear=clear, pre=pre, allow_global=system, dev=True) + for lf in vcs_dev_lockfiles: + try: + name = first(lf.keys()) + except AttributeError: + continue + if hasattr(lf[name], 'keys'): + lockfile['develop'].update(lf) if write: # Alert the user of progress. click.echo( @@ -1132,23 +1129,15 @@ def do_lock( lockfile['default'][dep['name']]['markers'] = dep['markers'] # Add refs for VCS installs. # TODO: be smarter about this. - vcs_deps = convert_deps_to_pip(project.vcs_packages, project, r=False) - if vcs_deps: - for line in pip_freeze.strip().split('\n'): - # if the line doesn't match a vcs dependency in the Pipfile, - # ignore it - if not any(dep in line for dep in vcs_deps): - continue + _vcs_deps, vcs_lockfiles = get_vcs_deps(project, pip_freeze, which=which, verbose=verbose, clear=clear, pre=pre, allow_global=system, dev=False) + for lf in vcs_lockfiles: + try: + name = first(lf.keys()) + except AttributeError: + continue + if hasattr(lf[name], 'keys'): + lockfile['default'].update(lf) - try: - installed = convert_deps_from_pip(line) - name = list(installed.keys())[0] - if is_vcs(installed[name]): - # Convert name to PEP 423 name. - installed = {pep423_name(name): installed[name]} - lockfile['default'].update(installed) - except IndexError: - pass # Support for --keep-outdated… if keep_outdated: for section_name, section in ( diff --git a/pipenv/patched/pew/pew.py b/pipenv/patched/pew/pew.py index 07dbdf4f..038ad886 100644 --- a/pipenv/patched/pew/pew.py +++ b/pipenv/patched/pew/pew.py @@ -184,7 +184,8 @@ def fork_bash(env, cwd): def fork_cmder(env, cwd): shell_cmd = ['cmd'] - cmderrc_path = r'%CMDER_ROOT%\vendor\init.bat' + escaped_cmder_root = os.environ['CMDER_ROOT'].replace(' ', '^ ') + cmderrc_path = r'{0}\vendor\init.bat'.format(escaped_cmder_root) if expandpath(cmderrc_path).exists(): shell_cmd += ['/k', cmderrc_path] if cwd: diff --git a/pipenv/patched/piptools/utils.py b/pipenv/patched/piptools/utils.py index d76695d4..1d732bf9 100644 --- a/pipenv/patched/piptools/utils.py +++ b/pipenv/patched/piptools/utils.py @@ -60,6 +60,36 @@ def make_install_requirement(name, version, extras, markers, constraint=False): constraint=constraint) +def _requirement_to_str_lowercase_name(requirement): + """ + Formats a packaging.requirements.Requirement with a lowercase name. + + This is simply a copy of + https://github.com/pypa/packaging/blob/16.8/packaging/requirements.py#L109-L124 + modified to lowercase the dependency name. + + Previously, we were invoking the original Requirement.__str__ method and + lowercasing the entire result, which would lowercase the name, *and* other, + important stuff that should not be lowercased (such as the marker). See + this issue for more information: https://github.com/pypa/pipenv/issues/2113. + """ + parts = [requirement.name.lower()] + + if requirement.extras: + parts.append("[{0}]".format(",".join(sorted(requirement.extras)))) + + if requirement.specifier: + parts.append(str(requirement.specifier)) + + if requirement.url: + parts.append("@ {0}".format(requirement.url)) + + if requirement.marker: + parts.append("; {0}".format(requirement.marker)) + + return "".join(parts) + + def format_requirement(ireq, marker=None): """ Generic formatter for pretty printing InstallRequirements to the terminal @@ -68,7 +98,7 @@ def format_requirement(ireq, marker=None): if ireq.editable: line = '-e {}'.format(ireq.link) else: - line = str(ireq.req).lower() + line = _requirement_to_str_lowercase_name(ireq.req) if marker: line = '{}; {}'.format(line, marker) diff --git a/pipenv/project.py b/pipenv/project.py index 2294d43f..76bcdadc 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -497,41 +497,41 @@ class Project(object): def lockfile_content(self): return self.load_lockfile() - @property - def editable_packages(self): + def _get_editable_packages(self, dev=False): + section = 'dev-packages' if dev else 'packages' packages = { k: v - for k, v in self.parsed_pipfile.get('packages', {}).items() + for k, v in self.parsed_pipfile.get(section, {}).items() if is_editable(v) } return packages - @property - def editable_dev_packages(self): + def _get_vcs_packages(self, dev=False): + section = 'dev-packages' if dev else 'packages' packages = { k: v - for k, v in self.parsed_pipfile.get('dev-packages', {}).items() - if is_editable(v) + for k, v in self.parsed_pipfile.get(section, {}).items() + if is_vcs(v) or is_vcs(k) } - return packages + return packages or {} + + @property + def editable_packages(self): + return self._get_editable_packages(dev=False) + + @property + def editable_dev_packages(self): + return self._get_editable_packages(dev=True) @property def vcs_packages(self): """Returns a list of VCS packages, for not pip-tools to consume.""" - ps = {} - for k, v in self.parsed_pipfile.get('packages', {}).items(): - if is_vcs(v) or is_vcs(k): - ps.update({k: v}) - return ps + return self._get_vcs_packages(dev=False) @property def vcs_dev_packages(self): """Returns a list of VCS packages, for not pip-tools to consume.""" - ps = {} - for k, v in self.parsed_pipfile.get('dev-packages', {}).items(): - if is_vcs(v) or is_vcs(k): - ps.update({k: v}) - return ps + return self._get_vcs_packages(dev=True) @property def all_packages(self): diff --git a/pipenv/utils.py b/pipenv/utils.py index 973b1c83..9ad18c7b 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -1358,3 +1358,71 @@ def safe_expandvars(value): if isinstance(value, six.string_types): return os.path.expandvars(value) return value + + +def extract_uri_from_vcs_dep(dep): + valid_keys = VCS_LIST + ('uri', 'file') + if hasattr(dep, 'keys'): + return first(dep[k] for k in valid_keys if k in dep) or None + return None + + +def install_or_update_vcs(vcs_obj, src_dir, name, rev=None): + target_dir = os.path.join(src_dir, name) + target_rev = vcs_obj.make_rev_options(rev) + if not os.path.exists(target_dir): + vcs_obj.obtain(target_dir) + vcs_obj.update(target_dir, target_rev) + return vcs_obj.get_revision(target_dir) + + +def get_vcs_deps(project, pip_freeze=None, which=None, verbose=False, clear=False, pre=False, allow_global=False, dev=False): + from ._compat import vcs + section = 'vcs_dev_packages' if dev else 'vcs_packages' + lines = [] + lockfiles = [] + try: + packages = getattr(project, section) + except AttributeError: + return [], [] + vcs_registry = vcs() + vcs_uri_map = { + extract_uri_from_vcs_dep(v): {'name': k, 'ref': v.get('ref')} + for k, v in packages.items() + } + for line in pip_freeze.strip().split('\n'): + # if the line doesn't match a vcs dependency in the Pipfile, + # ignore it + _vcs_match = first(_uri for _uri in vcs_uri_map.keys() if _uri in line) + if not _vcs_match: + continue + + pipfile_name = vcs_uri_map[_vcs_match]['name'] + pipfile_rev = vcs_uri_map[_vcs_match]['ref'] + src_dir = os.environ.get('PIP_SRC', os.path.join(project.virtualenv_location, 'src')) + mkdir_p(src_dir) + names = {pipfile_name} + _pip_uri = line.lstrip('-e ') + backend_name = str(_pip_uri.split('+', 1)[0]) + backend = vcs_registry._registry[first(b for b in vcs_registry if b == backend_name)] + __vcs = backend(url=_pip_uri) + + installed = convert_deps_from_pip(line) + if not hasattr(installed, 'keys'): + pass + lock_name = first(installed.keys()) + names.add(lock_name) + locked_rev = None + for _name in names: + locked_rev = install_or_update_vcs(__vcs, src_dir, _name, rev=pipfile_rev) + if is_vcs(installed[lock_name]): + installed[lock_name]['ref'] = locked_rev + lockfiles.append({pipfile_name: installed[lock_name]}) + pipfile_srcdir = os.path.join(src_dir, pipfile_name) + lockfile_srcdir = os.path.join(src_dir, lock_name) + lines.append(line) + if os.path.exists(pipfile_srcdir): + lockfiles.extend(venv_resolve_deps(['-e {0}'.format(pipfile_srcdir)], which=which, verbose=verbose, project=project, clear=clear, pre=pre, allow_global=allow_global)) + else: + lockfiles.extend(venv_resolve_deps(['-e {0}'.format(lockfile_srcdir)], which=which, verbose=verbose, project=project, clear=clear, pre=pre, allow_global=allow_global)) + return lines, lockfiles diff --git a/tasks/vendoring/patches/patched/pew-cmder-root-space-escape-fix.patch b/tasks/vendoring/patches/patched/pew-cmder-root-space-escape-fix.patch new file mode 100644 index 00000000..eb0e3d82 --- /dev/null +++ b/tasks/vendoring/patches/patched/pew-cmder-root-space-escape-fix.patch @@ -0,0 +1,14 @@ +diff --git a/pipenv/patched/pew/pew.py b/pipenv/patched/pew/pew.py +index 2d3889a0..91f313c1 100644 +--- a/pipenv/patched/pew/pew.py ++++ b/pipenv/patched/pew/pew.py +@@ -184,7 +184,8 @@ def fork_bash(env, cwd): + + def fork_cmder(env, cwd): + shell_cmd = ['cmd'] +- cmderrc_path = r'%CMDER_ROOT%\vendor\init.bat' ++ escaped_cmder_root = os.environ['CMDER_ROOT'].replace(' ', '^ ') ++ cmderrc_path = r'{0}\vendor\init.bat'.format(escaped_cmder_root) + if expandpath(cmderrc_path).exists(): + shell_cmd += ['/k', cmderrc_path] + if cwd: diff --git a/tests/integration/test_install_markers.py b/tests/integration/test_install_markers.py index e2f55415..6d048c8e 100644 --- a/tests/integration/test_install_markers.py +++ b/tests/integration/test_install_markers.py @@ -33,6 +33,26 @@ tablib = {version = "*", markers="os_name=='splashwear'"} c = p.pipenv('run python -c "import tablib;"') assert c.return_code == 1 +@pytest.mark.markers +@flaky +def test_platform_python_implementation_marker(PipenvInstance, pypi): + """Markers should be converted during locking to help users who input this incorrectly + """ + with PipenvInstance(pypi=pypi) as p: + with open(p.pipfile_path, 'w') as f: + contents = """ +[packages] +depends-on-marked-package = "*" + """.strip() + f.write(contents) + + c = p.pipenv('install') + assert c.return_code == 0 + + # depends-on-marked-package has an install_requires of 'pytz; platform_python_implementation=="CPython"' + # Verify that that marker shows up in our lockfile unaltered. + assert p.lockfile['default']['pytz']['markers'] == "platform_python_implementation == 'CPython'" + @pytest.mark.run @pytest.mark.alt diff --git a/tests/integration/test_install_uri.py b/tests/integration/test_install_uri.py index 646ca578..c187f217 100644 --- a/tests/integration/test_install_uri.py +++ b/tests/integration/test_install_uri.py @@ -2,6 +2,10 @@ import pytest import os from flaky import flaky import delegator +try: + from pathlib import Path +except ImportError: + from pathlib2 import Path @pytest.mark.vcs @@ -144,3 +148,25 @@ def test_install_local_vcs_not_in_lockfile(PipenvInstance, pip_src_dir): assert six_key in p.lockfile['default'] # Make sure we didn't put six in the lockfile by accident as a vcs ref assert 'six' not in p.lockfile['default'] + + +@pytest.mark.vcs +@pytest.mark.install +@pytest.mark.needs_internet +def test_get_vcs_refs(PipenvInstance, pip_src_dir): + with PipenvInstance(chdir=True) as p: + c = p.pipenv('install -e git+https://github.com/hynek/structlog.git@16.1.0#egg=structlog') + assert c.return_code == 0 + assert 'structlog' in p.pipfile['packages'] + assert 'structlog' in p.lockfile['default'] + assert 'six' in p.lockfile['default'] + assert p.lockfile['default']['structlog']['ref'] == 'a39f6906a268fb2f4c365042b31d0200468fb492' + pipfile = Path(p.pipfile_path) + new_content = pipfile.read_bytes().replace(b'16.1.0', b'18.1.0') + pipfile.write_bytes(new_content) + c = p.pipenv('lock') + assert c.return_code == 0 + assert p.lockfile['default']['structlog']['ref'] == 'a73fbd3a9c3cafb11f43168582083f839b883034' + assert 'structlog' in p.pipfile['packages'] + assert 'structlog' in p.lockfile['default'] + assert 'six' in p.lockfile['default'] diff --git a/tests/pypi/depends-on-marked-package/depends_on_marked_package-0.0.1-py2.py3-none-any.whl b/tests/pypi/depends-on-marked-package/depends_on_marked_package-0.0.1-py2.py3-none-any.whl new file mode 100644 index 00000000..d320deb3 Binary files /dev/null and b/tests/pypi/depends-on-marked-package/depends_on_marked_package-0.0.1-py2.py3-none-any.whl differ