Merge branch 'master' into patch-2

HISTORY.txt

@@ -2,6 +2,8 @@
  - Resolve editable packages on the local filesystem.
  - Ensure lock hash does not change based on injected env vars.
  - Fix bug in detecting .venv at project root when in subdirectories.
+ - Parse quoting in [scripts] section correctly + clearer run errors.
+ - Fix bug resolving & locking markers correctly 
 11.9.0:
  - Vastly improve markers capabilities.
  - Support for environment variables in Pipfiles.

pipenv/core.py

@@ -4,6 +4,7 @@ import logging
 import os
 import sys
 import shutil
+import shlex
 import signal
 import time
 import tempfile
@@ -1007,6 +1008,8 @@ def do_lock(
     write=True,
 ):
     """Executes the freeze functionality."""
+    from notpip._vendor.distlib.markers import Evaluator
+    allowed_marker_keys = ['markers'] + [k for k in Evaluator.allowed_values.keys()]
     cached_lockfile = {}
     if keep_outdated:
         if not project.lockfile_exists:
@@ -1064,8 +1067,11 @@ def do_lock(
         # Add index metadata to lockfile.
         if 'index' in dep:
             lockfile['develop'][dep['name']]['index'] = dep['index']
-        # Add PEP 508 specifier metadata to lockfile.
+        # Add PEP 508 specifier metadata to lockfile if dep isnt top level
+        # or top level dep doesn't itself have markers
         if 'markers' in dep:
+            if dep['name'] in dev_packages and not any(key in dev_packages[dep['name']] for key in allowed_marker_keys):
+                continue
             lockfile['develop'][dep['name']]['markers'] = dep['markers']
     # Add refs for VCS installs.
     # TODO: be smarter about this.
@@ -1120,8 +1126,11 @@ def do_lock(
         # Add index metadata to lockfile.
         if 'index' in dep:
             lockfile['default'][dep['name']]['index'] = dep['index']
-        # Add PEP 508 specifier metadata to lockfile.
+        # Add PEP 508 specifier metadata to lockfile if dep isn't top level
+        # or top level dep has no specifiers itself
         if 'markers' in dep:
+            if dep['name'] in project.packages and not any(key in project.packages[dep['name']] for key in allowed_marker_keys):
+                continue
             lockfile['default'][dep['name']]['markers'] = dep['markers']
     # Add refs for VCS installs.
     # TODO: be smarter about this.
@@ -2184,34 +2193,52 @@ def inline_activate_virtualenv():
         )
 
 
-def do_run(command, args, three=None, python=False):
-    # Ensure that virtualenv is available.
-    ensure_project(three=three, python=python, validate=False)
-    load_dot_env()
+def do_run_nt(command, args):
+    """Run command by appending space-joined args to it!"""
+    import subprocess
+    command = project.scripts.get(command, command)
+
+    # if you've passed something with crazy quoting...
+    # ...just don't. (or put it in a script!)
+    p = subprocess.Popen(
+        command + ' '.join(args), shell=True, universal_newlines=True
+    )
+    p.communicate()
+    sys.exit(p.returncode)
+
+
+def _get_command_posix(project, command, args):
+    """Fully bake command into executable and args, based upon project"""
     # Script was found…
     if command in project.scripts:
-        command = ' '.join(project.scripts[command])
-    # Separate out things that were passed in as a string.
-    _c = list(command.split())
-    command = _c.pop(0)
-    if _c:
-        args = list(args)
-        for __c in reversed(_c):
-            args.insert(0, __c)
-    # Activate virtualenv under the current interpreter's environment
-    inline_activate_virtualenv()
-    # Windows!
-    if os.name == 'nt':
-        import subprocess
+        command = project.scripts[command]
+    parsed_command = shlex.split(command)
+    executable = parsed_command[0]
+    # prepend arguments
+    args = list(parsed_command[1:]) + list(args)
+    return executable, args
 
-        p = subprocess.Popen(
-            [command] + list(args), shell=True, universal_newlines=True
-        )
-        p.communicate()
-        sys.exit(p.returncode)
-    else:
-        command_path = system_which(command)
-        if not command_path:
+
+def do_run_posix(command, args):
+    """Attempt to run command either pulling from project or interpreting as executable.
+
+    Args are appended to the command in [scripts] section of project if found.
+    """
+    executable, args = _get_command_posix(project, command, args)
+    command_path = system_which(executable)
+    if not command_path:
+        if command in project.scripts:
+            click.echo(
+                '{0}: the command {1} (from {2}) could not be found within {3}.'
+                ''.format(
+                    crayons.red('Error', bold=True),
+                    crayons.red(executable),
+                    crayons.normal(command, bold=True),
+                    crayons.normal('PATH', bold=True),
+                ),
+                err=True,
+            )
+        else:
             click.echo(
                 '{0}: the command {1} could not be found within {2} or Pipfile\'s {3}.'
                 ''.format(
@@ -2222,10 +2249,20 @@ def do_run(command, args, three=None, python=False):
                 ),
                 err=True,
             )
-            sys.exit(1)
-        # Execute the command.
-        os.execl(command_path, command_path, *args)
-        pass
+        sys.exit(1)
+    os.execl(command_path, command_path, *args)
+
+
+def do_run(command, args, three=None, python=False):
+    # Ensure that virtualenv is available.
+    ensure_project(three=three, python=python, validate=False)
+    load_dot_env()
+    # Activate virtualenv under the current interpreter's environment
+    inline_activate_virtualenv()
+    if os.name == 'nt':
+        do_run_nt(command, args)
+    else:
+        do_run_posix(command, args)
 
 
 def do_check(three=None, python=False, system=False, unused=False, args=None):

pipenv/patched/notpip/index.py

@@ -653,7 +653,8 @@ class PackageFinder(object):
             if not ext:
                 self._log_skipped_link(link, 'not a file')
                 return
-            if ext not in SUPPORTED_EXTENSIONS and not ignore_compatibility:
+            # Always ignore unsupported extensions even when we ignore compatibility
+            if ext not in SUPPORTED_EXTENSIONS:
                 self._log_skipped_link(
                     link, 'unsupported archive format: %s' % ext)
                 return

pipenv/patched/piptools/resolver.py

@@ -13,7 +13,7 @@ from . import click
 from .cache import DependencyCache
 from .exceptions import UnsupportedConstraint
 from .logging import log
-from .utils import (format_requirement, format_specifier, full_groupby,
+from .utils import (format_requirement, format_specifier, full_groupby, dedup,
                     is_pinned_requirement, key_from_ireq, key_from_req, UNSAFE_PACKAGES)
 
 green = partial(click.style, fg='green')
@@ -297,9 +297,9 @@ class Resolver(object):
             dependencies = self.repository.get_dependencies(ireq)
             import sys
             if sys.version_info[0] == 2:
-                self.dependency_cache[ireq] = sorted(str(ireq.req) for ireq in dependencies)
+                self.dependency_cache[ireq] = sorted(format_requirement(ireq) for ireq in dependencies)
             else:
-                self.dependency_cache[ireq] = sorted('{0}; {1}'.format(str(ireq.req), str(ireq.markers)) if ireq.markers else str(ireq.req) for ireq in dependencies)
+                self.dependency_cache[ireq] = sorted(format_requirement(ireq) for ireq in dependencies)
 
         # Example: ['Werkzeug>=0.9', 'Jinja2>=2.4']
         dependency_strings = self.dependency_cache[ireq]
@@ -310,28 +310,11 @@ class Resolver(object):
         for dependency_string in dependency_strings:
 
             try:
-                markers = None
+                _dependency_string = dependency_string
                 if ';' in dependency_string:
                     # split off markers and remove any duplicates by comparing against deps 
-                    dependencies, markers = dependency_string.rsplit(';', 1)
-                    dependency_string = ';'.join([dep for dep in dependencies.split(';') if dep.strip() != markers.strip()])
-                individual_dependencies = [dep.strip() for dep in dependency_string.split(', ')]
-                cleaned_deps = []
-                for dep in individual_dependencies:
-                    tokens = [token.strip() for token in dep.split(';')]
-                    cleaned_tokens = []
-                    dep_markers = []
-                    if len(tokens) == 1:
-                        cleaned_deps.append(tokens[0])
-                        continue
-                    dep_markers = list(set(tokens[1:]))
-                    cleaned_tokens.append(tokens[0])
-                    if dep_markers:
-                        cleaned_tokens.extend(dep_markers)
-                    cleaned_deps.append('; '.join(cleaned_tokens))
-                _dependency_string = ', '.join(set(cleaned_deps))
-                if markers:
-                    _dependency_string += ';{0}'.format(markers)
+                    _dependencies = [dep.strip() for dep in dependency_string.split(';')]
+                    _dependency_string = '; '.join([dep for dep in dedup(_dependencies)])
 
                 yield InstallRequirement.from_line(_dependency_string, constraint=ireq.constraint)
             except InvalidMarker:

pipenv/patched/piptools/utils.py

@@ -87,7 +87,7 @@ def format_requirement(ireq, marker=None):
         line = str(ireq.req).lower()
 
     if marker:
-        line = '{} ; {}'.format(line, marker)
+        line = '{}; {}'.format(line, marker)
 
     return line
 

pipenv/project.py

@@ -369,12 +369,7 @@ class Project(object):
 
     @property
     def scripts(self):
-        scripts = self.parsed_pipfile.get('scripts', {})
-        posix = os.name == 'posix'
-        _scripts = {}
-        for (k, v) in scripts.items():
-            _scripts[k] = shlex.split(str(v), posix=posix)
-        return _scripts
+        return dict(self.parsed_pipfile.get('scripts', {}))
 
     def update_settings(self, d):
         settings = self.settings

tests/test_pipenv.py

@@ -1,10 +1,11 @@
 import os
+import sys
 import re
 import shutil
 import json
 import pytest
 import warnings
-from pipenv.core import activate_virtualenv
+from pipenv.core import activate_virtualenv, _get_command_posix
 from pipenv.utils import (
     temp_environ, get_windows_path, mkdir_p, normalize_drive, TemporaryDirectory
 )
@@ -22,6 +23,9 @@ try:
 except ImportError:
     from pipenv.vendor.pathlib2 import Path
 
+py3_only = pytest.mark.skipif(sys.version_info < (3, 0), reason="requires Python3")
+nix_only = pytest.mark.skipif(os.name != 'nt', reason="doesn't run on windows")
+
 os.environ['PIPENV_DONT_USE_PYENV'] = '1'
 os.environ['PIPENV_IGNORE_VIRTUALENVS'] = '1'
 os.environ['PIPENV_VENV_IN_PROJECT'] = '1'
@@ -558,23 +562,23 @@ tpfd = "*"
     @pytest.mark.run
     @pytest.mark.markers
     @pytest.mark.install
-    def test_package_environment_markers(self):
+    @pytest.mark.failed
+    def test_package_environment_markers(self, pypi):
 
-        with PipenvInstance() as p:
+        with PipenvInstance(pypi=pypi) as p:
             with open(p.pipfile_path, 'w') as f:
                 contents = """
 [packages]
-requests = {version = "*", markers="os_name=='splashwear'"}
+tablib = {version = "*", markers="os_name=='splashwear'"}
                 """.strip()
                 f.write(contents)
 
             c = p.pipenv('install')
             assert c.return_code == 0
-
             assert 'Ignoring' in c.out
-            assert 'markers' in p.lockfile['default']['requests']
+            assert 'markers' in p.lockfile['default']['tablib']
 
-            c = p.pipenv('run python -c "import requests;"')
+            c = p.pipenv('run python -c "import tablib;"')
             assert c.return_code == 1
 
     @pytest.mark.run
@@ -873,6 +877,21 @@ import records
 
         assert command == '{0}/bin/activate'.format(venv)
 
+    @pytest.mark.lock
+    def test_lock_handle_eggs(self, pypi):
+        """Ensure locking works with packages provoding egg formats.
+        """
+        with PipenvInstance() as p:
+            with open(p.pipfile_path, 'w') as f:
+                f.write("""
+[packages]
+RandomWords = "*"
+                """)
+            c = p.pipenv('lock --verbose')
+            assert c.return_code == 0
+            assert 'randomwords' in p.lockfile['default']
+            assert p.lockfile['default']['randomwords']['version'] == '==0.2.1'
+
     @pytest.mark.lock
     @pytest.mark.requirements
     def test_lock_requirements_file(self, pypi):
@@ -1164,14 +1183,16 @@ flask = "==0.12.2"
                 assert Project().get_lockfile_hash() != Project().calculate_pipfile_hash()
 
     @pytest.mark.run
-    def test_scripts_basic(self):
+    def test_scripts(self):
         with PipenvInstance(chdir=True) as p:
             with open(p.pipfile_path, 'w') as f:
-                f.write("""
+                f.write(r"""
 [scripts]
-printfoo = "python -c print('foo')"
+printfoo = "python -c \"print('foo')\""
+notfoundscript = "randomthingtotally"
+appendscript = "cmd arg1"
+multicommand = "bash -c \"cd docs && make html\""
                 """)
-
             c = p.pipenv('install')
             assert c.return_code == 0
 
@@ -1179,21 +1200,34 @@ printfoo = "python -c print('foo')"
             assert c.return_code == 0
             assert c.out == 'foo\n'
             assert c.err == ''
+            if os.name != 'nt':
+                c = p.pipenv('run notfoundscript')
+                assert c.return_code == 1
+                assert c.out == ''
+                assert 'Error' in c.err
+                assert 'randomthingtotally (from notfoundscript)' in c.err
+            executable, argv = _get_command_posix(Project(), 'multicommand', [])
+            assert executable == 'bash'
+            assert argv == ['-c', 'cd docs && make html']
+            executable, argv = _get_command_posix(Project(), 'appendscript', ['a', 'b'])
+            assert executable == 'cmd'
+            assert argv == ['arg1', 'a', 'b']
 
-    @pytest.mark.run
-    @pytest.mark.skip(reason='This fails on Windows (not sure about POSIX).')
-    def test_scripts_quoted(self):
+    @pytest.mark.lock
+    @pytest.mark.complex
+    @py3_only
+    def test_resolver_unique_markers(self, pypi):
+        """vcrpy has a dependency on `yarl` which comes with a marker
+        of 'python version in "3.4, 3.5, 3.6" - this marker duplicates itself:
+
+        'yarl; python version in "3.4, 3.5, 3.6"; python version in "3.4, 3.5, 3.6"'
+
+        This verifies that we clean that successfully.
+        """
         with PipenvInstance(chdir=True) as p:
-            with open(p.pipfile_path, 'w') as f:
-                f.write("""
-[scripts]
-printfoo = "python -c print('foo')"
-                """)
-
-            c = p.pipenv('install')
+            c = p.pipenv('install vcrpy==1.11.0')
             assert c.return_code == 0
-
-            c = p.pipenv('run printfoo')
-            assert c.return_code == 0
-            assert c.out == 'foo\n'
-            assert c.err == ''
+            assert 'yarl' in p.lockfile['default']
+            yarl = p.lockfile['default']['yarl']
+            assert 'markers' in yarl
+            assert yarl['markers'] == "python_version in '3.4, 3.5, 3.6'"