diff --git a/HISTORY.txt b/HISTORY.txt index b6eb43b8..1ef0cdac 100644 --- a/HISTORY.txt +++ b/HISTORY.txt @@ -2,6 +2,8 @@ - Resolve editable packages on the local filesystem. - Ensure lock hash does not change based on injected env vars. - Fix bug in detecting .venv at project root when in subdirectories. + - Parse quoting in [scripts] section correctly + clearer run errors. + - Fix bug resolving & locking markers correctly 11.9.0: - Vastly improve markers capabilities. - Support for environment variables in Pipfiles. diff --git a/pipenv/core.py b/pipenv/core.py index b083ed9d..f7297ecc 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -4,6 +4,7 @@ import logging import os import sys import shutil +import shlex import signal import time import tempfile @@ -1007,6 +1008,8 @@ def do_lock( write=True, ): """Executes the freeze functionality.""" + from notpip._vendor.distlib.markers import Evaluator + allowed_marker_keys = ['markers'] + [k for k in Evaluator.allowed_values.keys()] cached_lockfile = {} if keep_outdated: if not project.lockfile_exists: @@ -1064,8 +1067,11 @@ def do_lock( # Add index metadata to lockfile. if 'index' in dep: lockfile['develop'][dep['name']]['index'] = dep['index'] - # Add PEP 508 specifier metadata to lockfile. + # Add PEP 508 specifier metadata to lockfile if dep isnt top level + # or top level dep doesn't itself have markers if 'markers' in dep: + if dep['name'] in dev_packages and not any(key in dev_packages[dep['name']] for key in allowed_marker_keys): + continue lockfile['develop'][dep['name']]['markers'] = dep['markers'] # Add refs for VCS installs. # TODO: be smarter about this. @@ -1120,8 +1126,11 @@ def do_lock( # Add index metadata to lockfile. if 'index' in dep: lockfile['default'][dep['name']]['index'] = dep['index'] - # Add PEP 508 specifier metadata to lockfile. + # Add PEP 508 specifier metadata to lockfile if dep isn't top level + # or top level dep has no specifiers itself if 'markers' in dep: + if dep['name'] in project.packages and not any(key in project.packages[dep['name']] for key in allowed_marker_keys): + continue lockfile['default'][dep['name']]['markers'] = dep['markers'] # Add refs for VCS installs. # TODO: be smarter about this. @@ -2184,34 +2193,52 @@ def inline_activate_virtualenv(): ) -def do_run(command, args, three=None, python=False): - # Ensure that virtualenv is available. - ensure_project(three=three, python=python, validate=False) - load_dot_env() +def do_run_nt(command, args): + """Run command by appending space-joined args to it!""" + import subprocess + command = project.scripts.get(command, command) + + # if you've passed something with crazy quoting... + # ...just don't. (or put it in a script!) + p = subprocess.Popen( + command + ' '.join(args), shell=True, universal_newlines=True + ) + p.communicate() + sys.exit(p.returncode) + + +def _get_command_posix(project, command, args): + """Fully bake command into executable and args, based upon project""" # Script was found… if command in project.scripts: - command = ' '.join(project.scripts[command]) - # Separate out things that were passed in as a string. - _c = list(command.split()) - command = _c.pop(0) - if _c: - args = list(args) - for __c in reversed(_c): - args.insert(0, __c) - # Activate virtualenv under the current interpreter's environment - inline_activate_virtualenv() - # Windows! - if os.name == 'nt': - import subprocess + command = project.scripts[command] + parsed_command = shlex.split(command) + executable = parsed_command[0] + # prepend arguments + args = list(parsed_command[1:]) + list(args) + return executable, args - p = subprocess.Popen( - [command] + list(args), shell=True, universal_newlines=True - ) - p.communicate() - sys.exit(p.returncode) - else: - command_path = system_which(command) - if not command_path: + +def do_run_posix(command, args): + """Attempt to run command either pulling from project or interpreting as executable. + + Args are appended to the command in [scripts] section of project if found. + """ + executable, args = _get_command_posix(project, command, args) + command_path = system_which(executable) + if not command_path: + if command in project.scripts: + click.echo( + '{0}: the command {1} (from {2}) could not be found within {3}.' + ''.format( + crayons.red('Error', bold=True), + crayons.red(executable), + crayons.normal(command, bold=True), + crayons.normal('PATH', bold=True), + ), + err=True, + ) + else: click.echo( '{0}: the command {1} could not be found within {2} or Pipfile\'s {3}.' ''.format( @@ -2222,10 +2249,20 @@ def do_run(command, args, three=None, python=False): ), err=True, ) - sys.exit(1) - # Execute the command. - os.execl(command_path, command_path, *args) - pass + sys.exit(1) + os.execl(command_path, command_path, *args) + + +def do_run(command, args, three=None, python=False): + # Ensure that virtualenv is available. + ensure_project(three=three, python=python, validate=False) + load_dot_env() + # Activate virtualenv under the current interpreter's environment + inline_activate_virtualenv() + if os.name == 'nt': + do_run_nt(command, args) + else: + do_run_posix(command, args) def do_check(three=None, python=False, system=False, unused=False, args=None): diff --git a/pipenv/patched/notpip/index.py b/pipenv/patched/notpip/index.py index f8f313e4..48aaa35f 100644 --- a/pipenv/patched/notpip/index.py +++ b/pipenv/patched/notpip/index.py @@ -653,7 +653,8 @@ class PackageFinder(object): if not ext: self._log_skipped_link(link, 'not a file') return - if ext not in SUPPORTED_EXTENSIONS and not ignore_compatibility: + # Always ignore unsupported extensions even when we ignore compatibility + if ext not in SUPPORTED_EXTENSIONS: self._log_skipped_link( link, 'unsupported archive format: %s' % ext) return diff --git a/pipenv/patched/piptools/resolver.py b/pipenv/patched/piptools/resolver.py index 0a4a02a9..acea5d9f 100755 --- a/pipenv/patched/piptools/resolver.py +++ b/pipenv/patched/piptools/resolver.py @@ -13,7 +13,7 @@ from . import click from .cache import DependencyCache from .exceptions import UnsupportedConstraint from .logging import log -from .utils import (format_requirement, format_specifier, full_groupby, +from .utils import (format_requirement, format_specifier, full_groupby, dedup, is_pinned_requirement, key_from_ireq, key_from_req, UNSAFE_PACKAGES) green = partial(click.style, fg='green') @@ -297,9 +297,9 @@ class Resolver(object): dependencies = self.repository.get_dependencies(ireq) import sys if sys.version_info[0] == 2: - self.dependency_cache[ireq] = sorted(str(ireq.req) for ireq in dependencies) + self.dependency_cache[ireq] = sorted(format_requirement(ireq) for ireq in dependencies) else: - self.dependency_cache[ireq] = sorted('{0}; {1}'.format(str(ireq.req), str(ireq.markers)) if ireq.markers else str(ireq.req) for ireq in dependencies) + self.dependency_cache[ireq] = sorted(format_requirement(ireq) for ireq in dependencies) # Example: ['Werkzeug>=0.9', 'Jinja2>=2.4'] dependency_strings = self.dependency_cache[ireq] @@ -310,28 +310,11 @@ class Resolver(object): for dependency_string in dependency_strings: try: - markers = None + _dependency_string = dependency_string if ';' in dependency_string: # split off markers and remove any duplicates by comparing against deps - dependencies, markers = dependency_string.rsplit(';', 1) - dependency_string = ';'.join([dep for dep in dependencies.split(';') if dep.strip() != markers.strip()]) - individual_dependencies = [dep.strip() for dep in dependency_string.split(', ')] - cleaned_deps = [] - for dep in individual_dependencies: - tokens = [token.strip() for token in dep.split(';')] - cleaned_tokens = [] - dep_markers = [] - if len(tokens) == 1: - cleaned_deps.append(tokens[0]) - continue - dep_markers = list(set(tokens[1:])) - cleaned_tokens.append(tokens[0]) - if dep_markers: - cleaned_tokens.extend(dep_markers) - cleaned_deps.append('; '.join(cleaned_tokens)) - _dependency_string = ', '.join(set(cleaned_deps)) - if markers: - _dependency_string += ';{0}'.format(markers) + _dependencies = [dep.strip() for dep in dependency_string.split(';')] + _dependency_string = '; '.join([dep for dep in dedup(_dependencies)]) yield InstallRequirement.from_line(_dependency_string, constraint=ireq.constraint) except InvalidMarker: diff --git a/pipenv/patched/piptools/utils.py b/pipenv/patched/piptools/utils.py index 44d2d336..26d418eb 100755 --- a/pipenv/patched/piptools/utils.py +++ b/pipenv/patched/piptools/utils.py @@ -87,7 +87,7 @@ def format_requirement(ireq, marker=None): line = str(ireq.req).lower() if marker: - line = '{} ; {}'.format(line, marker) + line = '{}; {}'.format(line, marker) return line diff --git a/pipenv/project.py b/pipenv/project.py index f349298c..4b8d1bb7 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -369,12 +369,7 @@ class Project(object): @property def scripts(self): - scripts = self.parsed_pipfile.get('scripts', {}) - posix = os.name == 'posix' - _scripts = {} - for (k, v) in scripts.items(): - _scripts[k] = shlex.split(str(v), posix=posix) - return _scripts + return dict(self.parsed_pipfile.get('scripts', {})) def update_settings(self, d): settings = self.settings diff --git a/tests/pypi/multidict/multidict-4.1.0-cp36-cp36m-win_amd64.whl b/tests/pypi/multidict/multidict-4.1.0-cp36-cp36m-win_amd64.whl new file mode 100644 index 00000000..6f8c38f3 Binary files /dev/null and b/tests/pypi/multidict/multidict-4.1.0-cp36-cp36m-win_amd64.whl differ diff --git a/tests/pypi/pyyaml/PyYAML-3.12-cp27-cp27m-win32.whl b/tests/pypi/pyyaml/PyYAML-3.12-cp27-cp27m-win32.whl new file mode 100644 index 00000000..d9f5e245 Binary files /dev/null and b/tests/pypi/pyyaml/PyYAML-3.12-cp27-cp27m-win32.whl differ diff --git a/tests/pypi/pyyaml/PyYAML-3.12-cp27-cp27m-win_amd64.whl b/tests/pypi/pyyaml/PyYAML-3.12-cp27-cp27m-win_amd64.whl new file mode 100644 index 00000000..2eb7094f Binary files /dev/null and b/tests/pypi/pyyaml/PyYAML-3.12-cp27-cp27m-win_amd64.whl differ diff --git a/tests/pypi/pyyaml/PyYAML-3.12-cp35-cp35m-win32.whl b/tests/pypi/pyyaml/PyYAML-3.12-cp35-cp35m-win32.whl new file mode 100644 index 00000000..f58c2f69 Binary files /dev/null and b/tests/pypi/pyyaml/PyYAML-3.12-cp35-cp35m-win32.whl differ diff --git a/tests/pypi/pyyaml/PyYAML-3.12-cp35-cp35m-win_amd64.whl b/tests/pypi/pyyaml/PyYAML-3.12-cp35-cp35m-win_amd64.whl new file mode 100644 index 00000000..c3d36a8b Binary files /dev/null and b/tests/pypi/pyyaml/PyYAML-3.12-cp35-cp35m-win_amd64.whl differ diff --git a/tests/pypi/randomwords/RandomWords-0.2.1-py2.7.egg b/tests/pypi/randomwords/RandomWords-0.2.1-py2.7.egg new file mode 100644 index 00000000..ef607e11 Binary files /dev/null and b/tests/pypi/randomwords/RandomWords-0.2.1-py2.7.egg differ diff --git a/tests/pypi/randomwords/RandomWords-0.2.1-py3.5.egg b/tests/pypi/randomwords/RandomWords-0.2.1-py3.5.egg new file mode 100644 index 00000000..e8fa3f78 Binary files /dev/null and b/tests/pypi/randomwords/RandomWords-0.2.1-py3.5.egg differ diff --git a/tests/pypi/randomwords/RandomWords-0.2.1-py3.6.egg b/tests/pypi/randomwords/RandomWords-0.2.1-py3.6.egg new file mode 100644 index 00000000..bd0b78c7 Binary files /dev/null and b/tests/pypi/randomwords/RandomWords-0.2.1-py3.6.egg differ diff --git a/tests/pypi/randomwords/RandomWords-0.2.1.tar.gz b/tests/pypi/randomwords/RandomWords-0.2.1.tar.gz new file mode 100644 index 00000000..3791a446 Binary files /dev/null and b/tests/pypi/randomwords/RandomWords-0.2.1.tar.gz differ diff --git a/tests/pypi/requests/requests-2.14.0-py2.py3-none-any.whl b/tests/pypi/requests/requests-2.14.0-py2.py3-none-any.whl new file mode 100644 index 00000000..1fb0b95a Binary files /dev/null and b/tests/pypi/requests/requests-2.14.0-py2.py3-none-any.whl differ diff --git a/tests/pypi/requests/requests-2.14.0.tar.gz b/tests/pypi/requests/requests-2.14.0.tar.gz new file mode 100644 index 00000000..33631280 Binary files /dev/null and b/tests/pypi/requests/requests-2.14.0.tar.gz differ diff --git a/tests/pypi/vcrpy/vcrpy-1.11.0-py2.py3-none-any.whl b/tests/pypi/vcrpy/vcrpy-1.11.0-py2.py3-none-any.whl new file mode 100644 index 00000000..2853aafc Binary files /dev/null and b/tests/pypi/vcrpy/vcrpy-1.11.0-py2.py3-none-any.whl differ diff --git a/tests/pypi/yarl/yarl-1.1.1-cp36-cp36m-manylinux1_i686.whl b/tests/pypi/yarl/yarl-1.1.1-cp36-cp36m-manylinux1_i686.whl new file mode 100644 index 00000000..ab6c0e0c Binary files /dev/null and b/tests/pypi/yarl/yarl-1.1.1-cp36-cp36m-manylinux1_i686.whl differ diff --git a/tests/pypi/yarl/yarl-1.1.1-cp36-cp36m-manylinux1_x86_64.whl b/tests/pypi/yarl/yarl-1.1.1-cp36-cp36m-manylinux1_x86_64.whl new file mode 100644 index 00000000..baeeb2c1 Binary files /dev/null and b/tests/pypi/yarl/yarl-1.1.1-cp36-cp36m-manylinux1_x86_64.whl differ diff --git a/tests/pypi/yarl/yarl-1.1.1-cp36-cp36m-win32.whl b/tests/pypi/yarl/yarl-1.1.1-cp36-cp36m-win32.whl new file mode 100644 index 00000000..45fe89ed Binary files /dev/null and b/tests/pypi/yarl/yarl-1.1.1-cp36-cp36m-win32.whl differ diff --git a/tests/pypi/yarl/yarl-1.1.1-cp36-cp36m-win_amd64.whl b/tests/pypi/yarl/yarl-1.1.1-cp36-cp36m-win_amd64.whl new file mode 100644 index 00000000..e11d5c62 Binary files /dev/null and b/tests/pypi/yarl/yarl-1.1.1-cp36-cp36m-win_amd64.whl differ diff --git a/tests/test_pipenv.py b/tests/test_pipenv.py index 0865556f..42865df2 100644 --- a/tests/test_pipenv.py +++ b/tests/test_pipenv.py @@ -1,10 +1,11 @@ import os +import sys import re import shutil import json import pytest import warnings -from pipenv.core import activate_virtualenv +from pipenv.core import activate_virtualenv, _get_command_posix from pipenv.utils import ( temp_environ, get_windows_path, mkdir_p, normalize_drive, TemporaryDirectory ) @@ -22,6 +23,9 @@ try: except ImportError: from pipenv.vendor.pathlib2 import Path +py3_only = pytest.mark.skipif(sys.version_info < (3, 0), reason="requires Python3") +nix_only = pytest.mark.skipif(os.name != 'nt', reason="doesn't run on windows") + os.environ['PIPENV_DONT_USE_PYENV'] = '1' os.environ['PIPENV_IGNORE_VIRTUALENVS'] = '1' os.environ['PIPENV_VENV_IN_PROJECT'] = '1' @@ -558,23 +562,23 @@ tpfd = "*" @pytest.mark.run @pytest.mark.markers @pytest.mark.install - def test_package_environment_markers(self): + @pytest.mark.failed + def test_package_environment_markers(self, pypi): - with PipenvInstance() as p: + with PipenvInstance(pypi=pypi) as p: with open(p.pipfile_path, 'w') as f: contents = """ [packages] -requests = {version = "*", markers="os_name=='splashwear'"} +tablib = {version = "*", markers="os_name=='splashwear'"} """.strip() f.write(contents) c = p.pipenv('install') assert c.return_code == 0 - assert 'Ignoring' in c.out - assert 'markers' in p.lockfile['default']['requests'] + assert 'markers' in p.lockfile['default']['tablib'] - c = p.pipenv('run python -c "import requests;"') + c = p.pipenv('run python -c "import tablib;"') assert c.return_code == 1 @pytest.mark.run @@ -873,6 +877,21 @@ import records assert command == '{0}/bin/activate'.format(venv) + @pytest.mark.lock + def test_lock_handle_eggs(self, pypi): + """Ensure locking works with packages provoding egg formats. + """ + with PipenvInstance() as p: + with open(p.pipfile_path, 'w') as f: + f.write(""" +[packages] +RandomWords = "*" + """) + c = p.pipenv('lock --verbose') + assert c.return_code == 0 + assert 'randomwords' in p.lockfile['default'] + assert p.lockfile['default']['randomwords']['version'] == '==0.2.1' + @pytest.mark.lock @pytest.mark.requirements def test_lock_requirements_file(self, pypi): @@ -1164,14 +1183,16 @@ flask = "==0.12.2" assert Project().get_lockfile_hash() != Project().calculate_pipfile_hash() @pytest.mark.run - def test_scripts_basic(self): + def test_scripts(self): with PipenvInstance(chdir=True) as p: with open(p.pipfile_path, 'w') as f: - f.write(""" + f.write(r""" [scripts] -printfoo = "python -c print('foo')" +printfoo = "python -c \"print('foo')\"" +notfoundscript = "randomthingtotally" +appendscript = "cmd arg1" +multicommand = "bash -c \"cd docs && make html\"" """) - c = p.pipenv('install') assert c.return_code == 0 @@ -1179,21 +1200,34 @@ printfoo = "python -c print('foo')" assert c.return_code == 0 assert c.out == 'foo\n' assert c.err == '' + if os.name != 'nt': + c = p.pipenv('run notfoundscript') + assert c.return_code == 1 + assert c.out == '' + assert 'Error' in c.err + assert 'randomthingtotally (from notfoundscript)' in c.err + executable, argv = _get_command_posix(Project(), 'multicommand', []) + assert executable == 'bash' + assert argv == ['-c', 'cd docs && make html'] + executable, argv = _get_command_posix(Project(), 'appendscript', ['a', 'b']) + assert executable == 'cmd' + assert argv == ['arg1', 'a', 'b'] - @pytest.mark.run - @pytest.mark.skip(reason='This fails on Windows (not sure about POSIX).') - def test_scripts_quoted(self): + @pytest.mark.lock + @pytest.mark.complex + @py3_only + def test_resolver_unique_markers(self, pypi): + """vcrpy has a dependency on `yarl` which comes with a marker + of 'python version in "3.4, 3.5, 3.6" - this marker duplicates itself: + + 'yarl; python version in "3.4, 3.5, 3.6"; python version in "3.4, 3.5, 3.6"' + + This verifies that we clean that successfully. + """ with PipenvInstance(chdir=True) as p: - with open(p.pipfile_path, 'w') as f: - f.write(""" -[scripts] -printfoo = "python -c print('foo')" - """) - - c = p.pipenv('install') + c = p.pipenv('install vcrpy==1.11.0') assert c.return_code == 0 - - c = p.pipenv('run printfoo') - assert c.return_code == 0 - assert c.out == 'foo\n' - assert c.err == '' + assert 'yarl' in p.lockfile['default'] + yarl = p.lockfile['default']['yarl'] + assert 'markers' in yarl + assert yarl['markers'] == "python_version in '3.4, 3.5, 3.6'"