diff --git a/pipenv/core.py b/pipenv/core.py index f58db399..0a960668 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -18,12 +18,12 @@ import delegator from .vendor import pexpect import pipfile from blindspin import spinner - from requests.packages import urllib3 from requests.packages.urllib3.exceptions import InsecureRequestWarning +import six from .cmdparse import ScriptEmptyError -from .project import Project +from .project import Project, SourceNotFound from .utils import ( convert_deps_from_pip, convert_deps_to_pip, @@ -46,6 +46,7 @@ from .utils import ( is_star, TemporaryDirectory, rmtree, + split_argument, ) from .import pep508checker, progress from .environments import ( @@ -775,7 +776,10 @@ def do_install_dependencies( l[i] = list(l[i]) if '--hash' in l[i][0]: l[i][0] = (l[i][0].split('--hash')[0].strip()) + index_args = prepare_pip_source_args(project.sources) + index_args = ' '.join(index_args).replace(' -', '\n-') # Output only default dependencies + click.echo(index_args) if not dev: click.echo('\n'.join(d[0] for d in sorted(deps_list))) sys.exit(0) @@ -790,12 +794,8 @@ def do_install_dependencies( for dep, ignore_hash, block in deps_list_bar: if len(procs) < PIPENV_MAX_SUBPROCESS: # Use a specific index, if specified. - index = None - if ' -i ' in dep: - dep, index = dep.split(' -i ') - dep = '{0} {1}'.format(dep, ' '.join(index.split()[1:])).strip( - ) - index = index.split()[0] + dep, index = split_argument(dep, short='i', long_='index') + dep, extra_index = split_argument(dep, long_='extra-index-url') # Install the module. c = pip_install( dep, @@ -806,6 +806,7 @@ def do_install_dependencies( block=block, index=index, requirements_dir=requirements_dir, + extra_indexes=extra_index, ) c.dep = dep c.ignore_hash = ignore_hash @@ -824,12 +825,9 @@ def do_install_dependencies( for dep, ignore_hash in progress.bar( failed_deps_list, label=INSTALL_LABEL2 ): - index = None - if ' -i ' in dep: - dep, index = dep.split(' -i ') - dep = '{0} {1}'.format(dep, ' '.join(index.split()[1:])).strip( - ) - index = index.split()[0] + # Use a specific index, if specified. + dep, index = split_argument(dep, short='i', long_='index') + dep, extra_index = split_argument(dep, long_='extra-index-url') # Install the module. c = pip_install( dep, @@ -839,6 +837,7 @@ def do_install_dependencies( verbose=verbose, index=index, requirements_dir=requirements_dir, + extra_indexes=extra_index, ) # The Installation failed... if c.return_code != 0: @@ -1367,6 +1366,7 @@ def pip_install( pre=False, selective_upgrade=False, requirements_dir=None, + extra_indexes=None, ): import pip9 @@ -1415,51 +1415,63 @@ def pip_install( src = '' else: src = '' + # Try installing for each source in project.sources. if index: + if not is_valid_url(index): + index = project.find_source(index).get('url') sources = [{'url': index}] + if extra_indexes: + if isinstance(extra_indexes, six.string_types): + extra_indexes = [extra_indexes,] + for idx in extra_indexes: + try: + extra_src = project.find_source(idx).get('url') + except SourceNotFound: + extra_src = idx + if extra_src != index: + sources.append({'url': extra_src}) + else: + for idx in project.pipfile_sources: + if idx['url'] != sources[0]['url']: + sources.append({'url': idx['url']}) else: - sources = project.sources - for source in sources: - if package_name.startswith('-e '): - install_reqs = ' -e "{0}"'.format(package_name.split('-e ')[1]) - elif r: - install_reqs = ' -r {0}'.format(r) - else: - install_reqs = ' "{0}"'.format(package_name) - # Skip hash-checking mode, when appropriate. - if r: - with open(r) as f: - if '--hash' not in f.read(): - ignore_hashes = True - else: - if '--hash' not in install_reqs: + sources = project.pipfile_sources + if package_name.startswith('-e '): + install_reqs = ' -e "{0}"'.format(package_name.split('-e ')[1]) + elif r: + install_reqs = ' -r {0}'.format(r) + else: + install_reqs = ' "{0}"'.format(package_name) + # Skip hash-checking mode, when appropriate. + if r: + with open(r) as f: + if '--hash' not in f.read(): ignore_hashes = True - verbose_flag = '--verbose' if verbose else '' - if not ignore_hashes: - install_reqs += ' --require-hashes' - no_deps = '--no-deps' if no_deps else '' - pre = '--pre' if pre else '' - quoted_python = which('python', allow_global=allow_global) - quoted_python = escape_grouped_arguments(quoted_python) - upgrade_strategy = '--upgrade --upgrade-strategy=only-if-needed' if selective_upgrade else '' - pip_command = '{0} -m pipenv.vendor.pip9 install {4} {5} {6} {7} {3} {1} {2} --exists-action w'.format( - quoted_python, - install_reqs, - ' '.join(prepare_pip_source_args([source])), - no_deps, - pre, - src, - verbose_flag, - upgrade_strategy, - ) - if verbose: - click.echo('$ {0}'.format(pip_command), err=True) - c = delegator.run(pip_command, block=block) - if c.return_code == 0: - break - - # Return the result of the first one that runs ok, or the last one that didn't work. + else: + if '--hash' not in install_reqs: + ignore_hashes = True + verbose_flag = '--verbose' if verbose else '' + if not ignore_hashes: + install_reqs += ' --require-hashes' + no_deps = '--no-deps' if no_deps else '' + pre = '--pre' if pre else '' + quoted_python = which('python', allow_global=allow_global) + quoted_python = escape_grouped_arguments(quoted_python) + upgrade_strategy = '--upgrade --upgrade-strategy=only-if-needed' if selective_upgrade else '' + pip_command = '{0} -m pipenv.vendor.pip9 install {4} {5} {6} {7} {3} {1} {2} --exists-action w'.format( + quoted_python, + install_reqs, + ' '.join(prepare_pip_source_args(sources)), + no_deps, + pre, + src, + verbose_flag, + upgrade_strategy, + ) + if verbose: + click.echo('$ {0}'.format(pip_command), err=True) + c = delegator.run(pip_command, block=block) return c @@ -1841,6 +1853,19 @@ def do_install( more_packages = list(more_packages) if package_name == '-e': package_name = ' '.join([package_name, more_packages.pop(0)]) + # capture indexes and extra indexes + line = [package_name] + more_packages + index_indicators = ['-i', '--index', '--extra-index-url'] + index, extra_indexes = None, None + if more_packages and any(more_packages[0].startswith(s) for s in index_indicators): + line, index = split_argument(' '.join(line), short='i', long_='index') + line, extra_indexes = split_argument(line, long_='extra-index-url') + package_names = line.split() + package_name = package_names[0] + if len(package_names) > 1: + more_packages = package_names[1:] + else: + more_packages = [] # Capture . argument and assign it to nothing if package_name == '.': package_name = False @@ -1920,6 +1945,8 @@ def do_install( verbose=verbose, pre=pre, requirements_dir=requirements_directory.name, + index=index, + extra_indexes=extra_indexes, ) # Warn if --editable wasn't passed. try: diff --git a/pipenv/project.py b/pipenv/project.py index bf0c50fe..effaa17e 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -6,8 +6,8 @@ import re import sys import base64 import hashlib - import contoml +from first import first import pipfile import pipfile.api import toml @@ -611,6 +611,16 @@ class Project(object): # pipfile is mutated! self.clear_pipfile_cache() + @property + def pipfile_sources(self): + if 'source' in self.parsed_pipfile: + sources = [] + for s in self.parsed_pipfile['source']: + s['url'] = os.path.expandvars(s['url']) + sources.append(s) + return sources + return [DEFAULT_SOURCE] + @property def sources(self): if self.lockfile_exists: @@ -624,15 +634,36 @@ class Project(object): else: return [DEFAULT_SOURCE] - def get_source(self, name=None, url=None): - for source in self.sources: - if name: - if source.get('name') == name: - return source + def find_source(self, source): + """given a source, find it. + source can be a url or an index name. + """ + if not is_valid_url(source): + try: + source = self.get_source(name=source) + except SourceNotFound: + source = self.get_source(url=source) + else: + source = self.get_source(url=source) + return source + + def get_source(self, name=None, url=None): + def find_source(sources, name=None, url=None): + source = None + if name: + source = [s for s in sources if s.get('name') == name] elif url: - if source.get('url') in url: - return source + source = [s for s in sources if url.startswith(s.get('url'))] + if source: + return first(source) + + found_source = find_source(self.sources, name=name, url=url) + if found_source: + return found_source + found_source = find_source(self.pipfile_sources, name=name, url=url) + if found_source: + return found_source raise SourceNotFound(name or url) def destroy_lockfile(self): @@ -668,7 +699,7 @@ class Project(object): p = self.parsed_pipfile # Don't re-capitalize file URLs or VCSs. converted = convert_deps_from_pip(package_name) - converted = converted[[k for k in converted.keys()][0]] + converted = converted[first(k for k in converted.keys())] if not ( is_file(package_name) or is_vcs(converted) or 'path' in converted ): @@ -678,7 +709,7 @@ class Project(object): if key not in p: p[key] = {} package = convert_deps_from_pip(package_name) - package_name = [k for k in package.keys()][0] + package_name = first(k for k in package.keys()) name = self.get_package_name_in_pipfile(package_name, dev) if name and converted == '*': # Skip for wildcard version diff --git a/pipenv/utils.py b/pipenv/utils.py index 59e8f5b2..8f43c788 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -14,6 +14,7 @@ import stat import warnings from click import echo as click_echo +from first import first try: from weakref import finalize except ImportError: @@ -685,6 +686,9 @@ def convert_deps_to_pip(deps, project=None, r=True, include_index=False): pip_src_args = [] if 'index' in deps[dep]: pip_src_args = [project.get_source(deps[dep]['index'])] + for idx in project.sources: + if idx['url'] != pip_src_args[0]['url']: + pip_src_args.append(idx) else: pip_src_args = project.sources pip_args = prepare_pip_source_args(pip_src_args) @@ -1291,3 +1295,27 @@ class TemporaryDirectory(object): def cleanup(self): if self._finalizer.detach(): rmtree(self.name) + + +def split_argument(req, short=None, long_=None): + """Split an argument from a string (finds None if not present). + + Uses -short , --long , and --long=arg as permutations. + + returns string, index + """ + index_entries = [] + if long_: + long_ = ' --{0}'.format(long_) + index_entries.extend(['{0}{1}'.format(long_, s) for s in [' ', '=']]) + if short: + index_entries.append(' -{0} '.format(short)) + index = None + index_entry = first([entry for entry in index_entries if entry in req]) + if index_entry: + req, index = req.split(index_entry) + remaining_line = index.split() + if len(remaining_line) > 1: + index, more_req = remaining_line[0], ' '.join(remaining_line[1:]) + req = '{0} {1}'.format(req, more_req) + return req, index diff --git a/run-tests.sh b/run-tests.sh index 8e3eddde..37d39cf1 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -27,7 +27,7 @@ if [[ ! -z "$CI" ]]; then echo "Installing Pipenv…" - + pip uninstall -y pipenv pip install -e "$(pwd)" --upgrade pipenv install --deploy --system --dev @@ -62,8 +62,8 @@ fi # Use tap output if in a CI environment, otherwise just run the tests. if [[ "$TAP_OUTPUT" ]]; then - echo "$ pipenv run time pytest -v -n auto tests -m \"$TEST_SUITE\" --tap-stream | tee report-$PYTHON.tap" - pipenv run time pytest -v -n auto tests -m "$TEST_SUITE" --tap-stream | tee report.tap + echo "$ pipenv run time python -m pytest -v -n auto -m \"$TEST_SUITE\" --tap-stream tests/ | tee report-$PYTHON.tap" + pipenv run time python -m pytest -v -n auto -m "$TEST_SUITE" --tap-stream tests/ | tee report.tap else echo "$ pipenv run time pytest -v -n auto tests -m \"$TEST_SUITE\"" diff --git a/tests/integration/test_install_uri.py b/tests/integration/test_install_uri.py index 50b56d10..b664718d 100644 --- a/tests/integration/test_install_uri.py +++ b/tests/integration/test_install_uri.py @@ -1,5 +1,5 @@ import pytest - +import os from flaky import flaky @@ -92,3 +92,30 @@ def test_install_editable_git_tag(PipenvInstance, pip_src_dir, pypi): assert 'git' in p.lockfile['default']['six'] assert p.lockfile['default']['six']['git'] == 'https://github.com/benjaminp/six.git' assert 'ref' in p.lockfile['default']['six'] + + +@pytest.mark.install +@pytest.mark.index +@pytest.mark.needs_internet +def test_install_named_index_alias(PipenvInstance, pypi): + with PipenvInstance(pypi=pypi) as p: + with open(p.pipfile_path, 'w') as f: + contents = """ +[[source]] +url = "https://pypi.python.org/simple" +verify_ssl = true +name = "pypi" + +[[source]] +url = "https://test.pypi.org/simple" +verify_ssl = true +name = "testpypi" + +[packages] +six = "*" + +[dev-packages] + """.strip() + f.write(contents) + c = p.pipenv('install pipenv-test-private-package --index testpypi') + assert c.return_code == 0 diff --git a/tests/integration/test_lock.py b/tests/integration/test_lock.py index b157e12a..29d472e5 100644 --- a/tests/integration/test_lock.py +++ b/tests/integration/test_lock.py @@ -150,3 +150,63 @@ records = {extras = ["pandas"], version = "==0.5.2"} assert c.return_code == 0 assert 'tablib' in p.lockfile['default'] assert 'pandas' in p.lockfile['default'] + + +@pytest.mark.skip_lock +@pytest.mark.index +@pytest.mark.needs_internet +@pytest.mark.install # private indexes need to be uncached for resolution +def test_private_index_skip_lock(PipenvInstance): + with PipenvInstance() as p: + with open(p.pipfile_path, 'w') as f: + contents = """ +[[source]] +url = "https://pypi.python.org/simple" +verify_ssl = true +name = "pypi" + +[[source]] +url = "https://test.pypi.org/simple" +verify_ssl = true +name = "testpypi" + +[packages] +pipenv-test-private-package = {version = "*", index = "testpypi"} +requests = "*" + """.strip() + f.write(contents) + c = p.pipenv('install --skip-lock') + assert c.return_code == 0 + + +@pytest.mark.requirements +@pytest.mark.lock +@pytest.mark.index +@pytest.mark.install # private indexes need to be uncached for resolution +@pytest.mark.needs_internet +def test_private_index_lock_requirements(PipenvInstance): + # Don't use the local fake pypi + with PipenvInstance() as p: + with open(p.pipfile_path, 'w') as f: + contents = """ +[[source]] +url = "https://pypi.python.org/simple" +verify_ssl = true +name = "pypi" + +[[source]] +url = "https://test.pypi.org/simple" +verify_ssl = true +name = "testpypi" + +[packages] +pipenv-test-private-package = {version = "*", index = "testpypi"} +requests = "*" + """.strip() + f.write(contents) + c = p.pipenv('install') + assert c.return_code == 0 + c = p.pipenv('lock -r') + assert c.return_code == 0 + assert '-i https://pypi.python.org/simple' in c.out.strip() + assert '--extra-index-url https://test.pypi.org/simple' in c.out.strip() diff --git a/tests/integration/test_project.py b/tests/integration/test_project.py new file mode 100644 index 00000000..4e8fe44a --- /dev/null +++ b/tests/integration/test_project.py @@ -0,0 +1,51 @@ +# -*- coding=utf-8 -*- +import pytest +import os +from pipenv.project import Project + + +@pytest.mark.project +@pytest.mark.sources +@pytest.mark.parametrize('lock_first', [True, False]) +def test_get_source(PipenvInstance, pypi, lock_first): + with PipenvInstance(pypi=pypi, chdir=True) as p: + with open(p.pipfile_path, 'w') as f: + contents = """ +[[source]] +url = "{0}" +verify_ssl = false +name = "testindex" + +[[source]] +url = "https://pypi.python.org/simple" +verify_ssl = "true" +name = "pypi" + +[packages] +pytz = "*" +six = {{version = "*", index = "pypi"}} + +[dev-packages] + """.format(os.environ['PIPENV_TEST_INDEX']).strip() + f.write(contents) + + if lock_first: + # force source to be cached + c = p.pipenv('lock') + assert c.return_code == 0 + project = Project() + sources = [ + ['pypi', 'https://pypi.python.org/simple'], + ['testindex', os.environ.get('PIPENV_TEST_INDEX')] + ] + for src in sources: + name, url = src + source = [s for s in project.pipfile_sources if s.get('name') == name] + assert source + source = source[0] + assert source['name'] == name + assert source['url'] == url + assert sorted(source.items()) == sorted(project.get_source(name=name).items()) + assert sorted(source.items()) == sorted(project.get_source(url=url).items()) + assert sorted(source.items()) == sorted(project.find_source(name).items()) + assert sorted(source.items()) == sorted(project.find_source(url).items())