Merge pull request #1980 from pypa/1973-extra-index-urls

Add `--extra-index-url` from all extra indexes

pipenv/core.py

@@ -18,12 +18,12 @@ import delegator
 from .vendor import pexpect
 import pipfile
 from blindspin import spinner
-
 from requests.packages import urllib3
 from requests.packages.urllib3.exceptions import InsecureRequestWarning
+import six
 
 from .cmdparse import ScriptEmptyError
-from .project import Project
+from .project import Project, SourceNotFound
 from .utils import (
     convert_deps_from_pip,
     convert_deps_to_pip,
@@ -46,6 +46,7 @@ from .utils import (
     is_star,
     TemporaryDirectory,
     rmtree,
+    split_argument,
 )
 from .import pep508checker, progress
 from .environments import (
@@ -775,7 +776,10 @@ def do_install_dependencies(
                 l[i] = list(l[i])
                 if '--hash' in l[i][0]:
                     l[i][0] = (l[i][0].split('--hash')[0].strip())
+        index_args = prepare_pip_source_args(project.sources)
+        index_args = ' '.join(index_args).replace(' -', '\n-')
         # Output only default dependencies
+        click.echo(index_args)
         if not dev:
             click.echo('\n'.join(d[0] for d in sorted(deps_list)))
             sys.exit(0)
@@ -790,12 +794,8 @@ def do_install_dependencies(
     for dep, ignore_hash, block in deps_list_bar:
         if len(procs) < PIPENV_MAX_SUBPROCESS:
             # Use a specific index, if specified.
-            index = None
-            if ' -i ' in dep:
-                dep, index = dep.split(' -i ')
-                dep = '{0} {1}'.format(dep, ' '.join(index.split()[1:])).strip(
-                )
-                index = index.split()[0]
+            dep, index = split_argument(dep, short='i', long_='index')
+            dep, extra_index = split_argument(dep, long_='extra-index-url')
             # Install the module.
             c = pip_install(
                 dep,
@@ -806,6 +806,7 @@ def do_install_dependencies(
                 block=block,
                 index=index,
                 requirements_dir=requirements_dir,
+                extra_indexes=extra_index,
             )
             c.dep = dep
             c.ignore_hash = ignore_hash
@@ -824,12 +825,9 @@ def do_install_dependencies(
         for dep, ignore_hash in progress.bar(
             failed_deps_list, label=INSTALL_LABEL2
         ):
-            index = None
-            if ' -i ' in dep:
-                dep, index = dep.split(' -i ')
-                dep = '{0} {1}'.format(dep, ' '.join(index.split()[1:])).strip(
-                )
-                index = index.split()[0]
+            # Use a specific index, if specified.
+            dep, index = split_argument(dep, short='i', long_='index')
+            dep, extra_index = split_argument(dep, long_='extra-index-url')
             # Install the module.
             c = pip_install(
                 dep,
@@ -839,6 +837,7 @@ def do_install_dependencies(
                 verbose=verbose,
                 index=index,
                 requirements_dir=requirements_dir,
+                extra_indexes=extra_index,
             )
             # The Installation failed...
             if c.return_code != 0:
@@ -1367,6 +1366,7 @@ def pip_install(
     pre=False,
     selective_upgrade=False,
     requirements_dir=None,
+    extra_indexes=None,
 ):
     import pip9
 
@@ -1415,51 +1415,63 @@ def pip_install(
             src = ''
     else:
         src = ''
+
     # Try installing for each source in project.sources.
     if index:
+        if not is_valid_url(index):
+            index = project.find_source(index).get('url')
         sources = [{'url': index}]
+        if extra_indexes:
+            if isinstance(extra_indexes, six.string_types):
+                extra_indexes = [extra_indexes,]
+            for idx in extra_indexes:
+                try:
+                    extra_src = project.find_source(idx).get('url')
+                except SourceNotFound:
+                    extra_src = idx
+                if extra_src != index:
+                    sources.append({'url': extra_src})
+        else:
+            for idx in project.pipfile_sources:
+                if idx['url'] != sources[0]['url']:
+                    sources.append({'url': idx['url']})
     else:
-        sources = project.sources
-    for source in sources:
-        if package_name.startswith('-e '):
-            install_reqs = ' -e "{0}"'.format(package_name.split('-e ')[1])
-        elif r:
-            install_reqs = ' -r {0}'.format(r)
-        else:
-            install_reqs = ' "{0}"'.format(package_name)
-        # Skip hash-checking mode, when appropriate.
-        if r:
-            with open(r) as f:
-                if '--hash' not in f.read():
-                    ignore_hashes = True
-        else:
-            if '--hash' not in install_reqs:
+        sources = project.pipfile_sources
+    if package_name.startswith('-e '):
+        install_reqs = ' -e "{0}"'.format(package_name.split('-e ')[1])
+    elif r:
+        install_reqs = ' -r {0}'.format(r)
+    else:
+        install_reqs = ' "{0}"'.format(package_name)
+    # Skip hash-checking mode, when appropriate.
+    if r:
+        with open(r) as f:
+            if '--hash' not in f.read():
                 ignore_hashes = True
-        verbose_flag = '--verbose' if verbose else ''
-        if not ignore_hashes:
-            install_reqs += ' --require-hashes'
-        no_deps = '--no-deps' if no_deps else ''
-        pre = '--pre' if pre else ''
-        quoted_python = which('python', allow_global=allow_global)
-        quoted_python = escape_grouped_arguments(quoted_python)
-        upgrade_strategy = '--upgrade --upgrade-strategy=only-if-needed' if selective_upgrade else ''
-        pip_command = '{0} -m pipenv.vendor.pip9 install {4} {5} {6} {7} {3} {1} {2} --exists-action w'.format(
-            quoted_python,
-            install_reqs,
-            ' '.join(prepare_pip_source_args([source])),
-            no_deps,
-            pre,
-            src,
-            verbose_flag,
-            upgrade_strategy,
-        )
-        if verbose:
-            click.echo('$ {0}'.format(pip_command), err=True)
-        c = delegator.run(pip_command, block=block)
-        if c.return_code == 0:
-            break
-
-    # Return the result of the first one that runs ok, or the last one that didn't work.
+    else:
+        if '--hash' not in install_reqs:
+            ignore_hashes = True
+    verbose_flag = '--verbose' if verbose else ''
+    if not ignore_hashes:
+        install_reqs += ' --require-hashes'
+    no_deps = '--no-deps' if no_deps else ''
+    pre = '--pre' if pre else ''
+    quoted_python = which('python', allow_global=allow_global)
+    quoted_python = escape_grouped_arguments(quoted_python)
+    upgrade_strategy = '--upgrade --upgrade-strategy=only-if-needed' if selective_upgrade else ''
+    pip_command = '{0} -m pipenv.vendor.pip9 install {4} {5} {6} {7} {3} {1} {2} --exists-action w'.format(
+        quoted_python,
+        install_reqs,
+        ' '.join(prepare_pip_source_args(sources)),
+        no_deps,
+        pre,
+        src,
+        verbose_flag,
+        upgrade_strategy,
+    )
+    if verbose:
+        click.echo('$ {0}'.format(pip_command), err=True)
+    c = delegator.run(pip_command, block=block)
     return c
 
 
@@ -1841,6 +1853,19 @@ def do_install(
     more_packages = list(more_packages)
     if package_name == '-e':
         package_name = ' '.join([package_name, more_packages.pop(0)])
+    # capture indexes and extra indexes
+    line = [package_name] + more_packages
+    index_indicators = ['-i', '--index', '--extra-index-url']
+    index, extra_indexes = None, None
+    if more_packages and any(more_packages[0].startswith(s) for s in index_indicators):
+        line, index = split_argument(' '.join(line), short='i', long_='index')
+        line, extra_indexes = split_argument(line, long_='extra-index-url')
+        package_names = line.split()
+        package_name = package_names[0]
+        if len(package_names) > 1:
+            more_packages = package_names[1:]
+        else:
+            more_packages = []
     # Capture . argument and assign it to nothing
     if package_name == '.':
         package_name = False
@@ -1920,6 +1945,8 @@ def do_install(
                 verbose=verbose,
                 pre=pre,
                 requirements_dir=requirements_directory.name,
+                index=index,
+                extra_indexes=extra_indexes,
             )
             # Warn if --editable wasn't passed.
             try:

pipenv/project.py

@@ -6,8 +6,8 @@ import re
 import sys
 import base64
 import hashlib
-
 import contoml
+from first import first
 import pipfile
 import pipfile.api
 import toml
@@ -611,6 +611,16 @@ class Project(object):
         # pipfile is mutated!
         self.clear_pipfile_cache()
 
+    @property
+    def pipfile_sources(self):
+        if 'source' in self.parsed_pipfile:
+            sources = []
+            for s in self.parsed_pipfile['source']:
+                s['url'] = os.path.expandvars(s['url'])
+                sources.append(s)
+            return sources
+        return [DEFAULT_SOURCE]
+
     @property
     def sources(self):
         if self.lockfile_exists:
@@ -624,15 +634,36 @@ class Project(object):
         else:
             return [DEFAULT_SOURCE]
 
-    def get_source(self, name=None, url=None):
-        for source in self.sources:
-            if name:
-                if source.get('name') == name:
-                    return source
+    def find_source(self, source):
+        """given a source, find it.
 
+        source can be a url or an index name.
+        """
+        if not is_valid_url(source):
+            try:
+                source = self.get_source(name=source)
+            except SourceNotFound:
+                source = self.get_source(url=source)
+        else:
+            source = self.get_source(url=source)
+        return source
+
+    def get_source(self, name=None, url=None):
+        def find_source(sources, name=None, url=None):
+            source = None
+            if name:
+                source = [s for s in sources if s.get('name') == name]
             elif url:
-                if source.get('url') in url:
-                    return source
+                source = [s for s in sources if url.startswith(s.get('url'))]
+            if source:
+                return first(source)
+
+        found_source = find_source(self.sources, name=name, url=url)
+        if found_source:
+            return found_source
+        found_source = find_source(self.pipfile_sources, name=name, url=url)
+        if found_source:
+            return found_source
         raise SourceNotFound(name or url)
 
     def destroy_lockfile(self):
@@ -668,7 +699,7 @@ class Project(object):
         p = self.parsed_pipfile
         # Don't re-capitalize file URLs or VCSs.
         converted = convert_deps_from_pip(package_name)
-        converted = converted[[k for k in converted.keys()][0]]
+        converted = converted[first(k for k in converted.keys())]
         if not (
             is_file(package_name) or is_vcs(converted) or 'path' in converted
         ):
@@ -678,7 +709,7 @@ class Project(object):
         if key not in p:
             p[key] = {}
         package = convert_deps_from_pip(package_name)
-        package_name = [k for k in package.keys()][0]
+        package_name = first(k for k in package.keys())
         name = self.get_package_name_in_pipfile(package_name, dev)
         if name and converted == '*':
             # Skip for wildcard version

pipenv/utils.py

@@ -14,6 +14,7 @@ import stat
 import warnings
 from click import echo as click_echo
 
+from first import first
 try:
     from weakref import finalize
 except ImportError:
@@ -685,6 +686,9 @@ def convert_deps_to_pip(deps, project=None, r=True, include_index=False):
             pip_src_args = []
             if 'index' in deps[dep]:
                 pip_src_args = [project.get_source(deps[dep]['index'])]
+                for idx in project.sources:
+                    if idx['url'] != pip_src_args[0]['url']:
+                        pip_src_args.append(idx)
             else:
                 pip_src_args = project.sources
             pip_args = prepare_pip_source_args(pip_src_args)
@@ -1291,3 +1295,27 @@ class TemporaryDirectory(object):
     def cleanup(self):
         if self._finalizer.detach():
             rmtree(self.name)
+
+
+def split_argument(req, short=None, long_=None):
+    """Split an argument from a string (finds None if not present).
+
+    Uses -short <arg>, --long <arg>, and --long=arg as permutations.
+
+    returns string, index
+    """
+    index_entries = []
+    if long_:
+        long_ = ' --{0}'.format(long_)
+        index_entries.extend(['{0}{1}'.format(long_, s) for s in [' ', '=']])
+    if short:
+        index_entries.append(' -{0} '.format(short))
+    index = None
+    index_entry = first([entry for entry in index_entries if entry in req])
+    if index_entry:
+        req, index = req.split(index_entry)
+        remaining_line = index.split()
+        if len(remaining_line) > 1:
+            index, more_req = remaining_line[0], ' '.join(remaining_line[1:])
+            req = '{0} {1}'.format(req, more_req)
+    return req, index

run-tests.sh

@@ -27,7 +27,7 @@ if [[ ! -z "$CI" ]]; then
 
 	echo "Installing Pipenv…"
 
-
+	pip uninstall -y pipenv
 	pip install -e "$(pwd)" --upgrade
 	pipenv install --deploy --system --dev
 
@@ -62,8 +62,8 @@ fi
 
 # Use tap output if in a CI environment, otherwise just run the tests.
 if [[ "$TAP_OUTPUT" ]]; then
-	echo "$ pipenv run time pytest -v -n auto tests -m \"$TEST_SUITE\" --tap-stream | tee report-$PYTHON.tap"
-	pipenv run time pytest -v -n auto tests -m "$TEST_SUITE"  --tap-stream | tee report.tap
+	echo "$ pipenv run time python -m pytest -v -n auto -m \"$TEST_SUITE\" --tap-stream tests/ | tee report-$PYTHON.tap"
+	pipenv run time python -m pytest -v -n auto -m "$TEST_SUITE"  --tap-stream tests/ | tee report.tap
 
 else
 	echo "$ pipenv run time pytest -v -n auto tests -m \"$TEST_SUITE\""

tests/integration/test_install_uri.py

@@ -1,5 +1,5 @@
 import pytest
-
+import os
 from flaky import flaky
 
 
@@ -92,3 +92,30 @@ def test_install_editable_git_tag(PipenvInstance, pip_src_dir, pypi):
         assert 'git' in p.lockfile['default']['six']
         assert p.lockfile['default']['six']['git'] == 'https://github.com/benjaminp/six.git'
         assert 'ref' in p.lockfile['default']['six']
+
+
+@pytest.mark.install
+@pytest.mark.index
+@pytest.mark.needs_internet
+def test_install_named_index_alias(PipenvInstance, pypi):
+    with PipenvInstance(pypi=pypi) as p:
+        with open(p.pipfile_path, 'w') as f:
+            contents = """
+[[source]]
+url = "https://pypi.python.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[[source]]
+url = "https://test.pypi.org/simple"
+verify_ssl = true
+name = "testpypi"
+
+[packages]
+six = "*"
+
+[dev-packages]
+            """.strip()
+            f.write(contents)
+        c = p.pipenv('install pipenv-test-private-package --index testpypi')
+        assert c.return_code == 0

tests/integration/test_lock.py

@@ -150,3 +150,63 @@ records = {extras = ["pandas"], version = "==0.5.2"}
         assert c.return_code == 0
         assert 'tablib' in p.lockfile['default']
         assert 'pandas' in p.lockfile['default']
+
+
+@pytest.mark.skip_lock
+@pytest.mark.index
+@pytest.mark.needs_internet
+@pytest.mark.install  # private indexes need to be uncached for resolution
+def test_private_index_skip_lock(PipenvInstance):
+    with PipenvInstance() as p:
+        with open(p.pipfile_path, 'w') as f:
+            contents = """
+[[source]]
+url = "https://pypi.python.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[[source]]
+url = "https://test.pypi.org/simple"
+verify_ssl = true
+name = "testpypi"
+
+[packages]
+pipenv-test-private-package = {version = "*", index = "testpypi"}
+requests = "*"
+            """.strip()
+            f.write(contents)
+        c = p.pipenv('install --skip-lock')
+        assert c.return_code == 0
+
+
+@pytest.mark.requirements
+@pytest.mark.lock
+@pytest.mark.index
+@pytest.mark.install  # private indexes need to be uncached for resolution
+@pytest.mark.needs_internet
+def test_private_index_lock_requirements(PipenvInstance):
+    # Don't use the local fake pypi
+    with PipenvInstance() as p:
+        with open(p.pipfile_path, 'w') as f:
+            contents = """
+[[source]]
+url = "https://pypi.python.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[[source]]
+url = "https://test.pypi.org/simple"
+verify_ssl = true
+name = "testpypi"
+
+[packages]
+pipenv-test-private-package = {version = "*", index = "testpypi"}
+requests = "*"
+            """.strip()
+            f.write(contents)
+        c = p.pipenv('install')
+        assert c.return_code == 0
+        c = p.pipenv('lock -r')
+        assert c.return_code == 0
+        assert '-i https://pypi.python.org/simple' in c.out.strip()
+        assert '--extra-index-url https://test.pypi.org/simple' in c.out.strip()

tests/integration/test_project.py

@@ -0,0 +1,51 @@
+# -*- coding=utf-8 -*-
+import pytest
+import os
+from pipenv.project import Project
+
+
+@pytest.mark.project
+@pytest.mark.sources
+@pytest.mark.parametrize('lock_first', [True, False])
+def test_get_source(PipenvInstance, pypi, lock_first):
+    with PipenvInstance(pypi=pypi, chdir=True) as p:
+        with open(p.pipfile_path, 'w') as f:
+            contents = """
+[[source]]
+url = "{0}"
+verify_ssl = false
+name = "testindex"
+
+[[source]]
+url = "https://pypi.python.org/simple"
+verify_ssl = "true"
+name = "pypi"
+
+[packages]
+pytz = "*"
+six = {{version = "*", index = "pypi"}}
+
+[dev-packages]
+            """.format(os.environ['PIPENV_TEST_INDEX']).strip()
+            f.write(contents)
+
+        if lock_first:
+            # force source to be cached
+            c = p.pipenv('lock')
+            assert c.return_code == 0
+        project = Project()
+        sources = [
+            ['pypi', 'https://pypi.python.org/simple'],
+            ['testindex', os.environ.get('PIPENV_TEST_INDEX')]
+        ]
+        for src in sources:
+            name, url = src
+            source = [s for s in project.pipfile_sources if s.get('name') == name]
+            assert source
+            source = source[0]
+            assert source['name'] == name
+            assert source['url'] == url
+            assert sorted(source.items()) == sorted(project.get_source(name=name).items())
+            assert sorted(source.items()) == sorted(project.get_source(url=url).items())
+            assert sorted(source.items()) == sorted(project.find_source(name).items())
+            assert sorted(source.items()) == sorted(project.find_source(url).items())