From 4a8086dd1ee2f59b7e535807523622b1e05a9164 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 10 Oct 2018 00:59:24 -0400 Subject: [PATCH 1/6] Clean up markers when formatting pipfiles Signed-off-by: Dan Ryan --- pipenv/utils.py | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/pipenv/utils.py b/pipenv/utils.py index 371a550a..88622b68 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -397,6 +397,7 @@ def resolve_deps( using pip-tools -- and their hashes, using the warehouse API / pip. """ from .patched.notpip._vendor.requests.exceptions import ConnectionError + from .vendor.requirementslib.models.requirements import Requirement from ._compat import TemporaryDirectory index_lookup = {} @@ -447,17 +448,11 @@ def resolve_deps( sys.exit(1) for result in resolved_tree: if not result.editable: - name = pep423_name(result.name) - version = clean_pkg_version(result.specifier) + req = Requirement.from_ireq(result) + name = pep423_name(req.name) + version = str(req.get_version()) index = index_lookup.get(result.name) - if not markers_lookup.get(result.name): - markers = ( - str(result.markers) - if result.markers and "extra" not in str(result.markers) - else None - ) - else: - markers = markers_lookup.get(result.name) + req.index = index collected_hashes = [] if result in hashes: collected_hashes = list(hashes.get(result)) @@ -493,13 +488,16 @@ def resolve_deps( # except (ValueError, KeyError, ConnectionError, IndexError): # if verbose: # print('Error generating hash for {}'.format(name)) - collected_hashes = sorted(set(collected_hashes)) - d = {"name": name, "version": version, "hashes": collected_hashes} - if index: - d.update({"index": index}) - if markers: - d.update({"markers": markers.replace('"', "'")}) - results.append(d) + req.hashes = sorted(set(collected_hashes)) + name, entry = req.pipfile_entry + entry["name"] = name + entry["version"] = version + # if index: + # d.update({"index": index}) + if markers_lookup.get(result.name): + entry.update({"markers": markers_lookup.get(result.name)}) + entry = translate_markers(entry) + results.append(entry) req_dir.cleanup() return results From a31f0c34a3bb288fecc9e6347a4a8f7183a683ea Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 10 Oct 2018 10:06:21 -0400 Subject: [PATCH 2/6] Fix trusted-host passthru - Fix marker cleaning - Fixes #2979 Signed-off-by: Dan Ryan --- news/2979.bugfix | 1 + pipenv/core.py | 25 +++++++++++++++++++------ pipenv/project.py | 27 +++++++++++++++------------ 3 files changed, 35 insertions(+), 18 deletions(-) create mode 100644 news/2979.bugfix diff --git a/news/2979.bugfix b/news/2979.bugfix new file mode 100644 index 00000000..67aae073 --- /dev/null +++ b/news/2979.bugfix @@ -0,0 +1 @@ +Fixed a bug which caused ``verify_ssl`` to fail to drop through to ``pip install`` correctly as ``trusted-host``. diff --git a/pipenv/core.py b/pipenv/core.py index 5a1a9979..aade28f7 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -780,6 +780,7 @@ def do_install_dependencies( requirements_dir=requirements_dir, extra_indexes=extra_indexes, pypi_mirror=pypi_mirror, + trusted_hosts=trusted_hosts ) c.dep = dep c.ignore_hash = ignore_hash @@ -1307,10 +1308,13 @@ def pip_install( requirements_dir=None, extra_indexes=None, pypi_mirror=None, + trusted_hosts=None ): from notpip._internal import logger as piplogger src = [] + if not trusted_hosts: + trusted_hosts = [] if environments.is_verbose(): piplogger.setLevel(logging.INFO) @@ -1335,23 +1339,28 @@ def pip_install( # Try installing for each source in project.sources. if index: - if not is_valid_url(index): - index = project.find_source(index).get("url") - sources = [{"url": index}] + try: + index_source = project.find_source(index) + index_source = index_source.copy() + except SourceNotFound: + src_name = project.src_name_from_url(index) + index_source = {"url": index, "verify_ssl": True, "name": src_name} + sources = [index_source.copy(),] if extra_indexes: if isinstance(extra_indexes, six.string_types): extra_indexes = [extra_indexes] for idx in extra_indexes: try: - extra_src = project.find_source(idx).get("url") + extra_src = project.find_source(idx) except SourceNotFound: extra_src = idx if extra_src != index: - sources.append({"url": extra_src}) + src_name = project.src_name_from_url(idx) + sources.append({"url": extra_src, "verify_ssl": True, "name": src_name}) else: for idx in project.pipfile_sources: if idx["url"] != sources[0]["url"]: - sources.append({"url": idx["url"]}) + sources.append(idx) else: sources = project.pipfile_sources if pypi_mirror: @@ -1372,6 +1381,10 @@ def pip_install( with open(r) as f: if "--hash" not in f.read(): ignore_hashes = True + # trusted_hosts = [ + # "--trusted-host={0}".format(source.get("url")) for source in sources + # if not source.get("verify_ssl", True) + # ] pip_command = [which_pip(allow_global=allow_global), "install"] if pre: pip_command.append("--pre") diff --git a/pipenv/project.py b/pipenv/project.py index 7fdf48cc..addc8ada 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -823,6 +823,20 @@ class Project(object): # Write Pipfile. self.write_toml(p) + def src_name_from_url(self, index_url): + name, _, tld_guess = six.moves.urllib.parse.urlsplit(index).netloc.rpartition( + "." + ) + src_name = name.replace(".", "") + try: + self.get_source(name=src_name) + except SourceNotFound: + name = src_name + else: + from random import randint + name = "{0}-{1}".format(src_name, randint(1, 1000)) + return name + def add_index_to_pipfile(self, index, verify_ssl=True): """Adds a given index to the Pipfile.""" # Read and append Pipfile. @@ -833,18 +847,7 @@ class Project(object): source = {"url": index, "verify_ssl": verify_ssl} else: return - name, _, tld_guess = six.moves.urllib.parse.urlsplit(index).netloc.rpartition( - "." - ) - src_name = name.replace(".", "") - try: - self.get_source(name=src_name) - except SourceNotFound: - source[name] = src_name - else: - from random import randint - - source[name] = "{0}-{1}".format(src_name, randint(1, 1000)) + source["name"] = self.src_name_from_url(index) # Add the package to the group. if "source" not in p: p["source"] = [source] From 3ee8e1159acbad2a1463283dccaefdab6232f9c4 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 10 Oct 2018 11:27:44 -0400 Subject: [PATCH 3/6] Fix source naming Signed-off-by: Dan Ryan --- pipenv/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/project.py b/pipenv/project.py index addc8ada..3a4d3cce 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -824,7 +824,7 @@ class Project(object): self.write_toml(p) def src_name_from_url(self, index_url): - name, _, tld_guess = six.moves.urllib.parse.urlsplit(index).netloc.rpartition( + name, _, tld_guess = six.moves.urllib.parse.urlsplit(index_url).netloc.rpartition( "." ) src_name = name.replace(".", "") From 54a2f7b977aeef612d5de6cf622423882c20965c Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 10 Oct 2018 16:53:00 -0400 Subject: [PATCH 4/6] Fix trusted host paramaterization Signed-off-by: Dan Ryan --- pipenv/core.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index aade28f7..56616623 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1315,7 +1315,7 @@ def pip_install( src = [] if not trusted_hosts: trusted_hosts = [] - + trusted_hosts.extend(os.environ.get("PIP_TRUSTED_HOSTS", [])) if environments.is_verbose(): piplogger.setLevel(logging.INFO) if requirement: @@ -1344,19 +1344,21 @@ def pip_install( index_source = index_source.copy() except SourceNotFound: src_name = project.src_name_from_url(index) - index_source = {"url": index, "verify_ssl": True, "name": src_name} + verify_ssl = True if index not in trusted_hosts else False + index_source = {"url": index, "verify_ssl": verify_ssl, "name": src_name} sources = [index_source.copy(),] if extra_indexes: if isinstance(extra_indexes, six.string_types): - extra_indexes = [extra_indexes] + extra_indexes = [extra_indexes,] for idx in extra_indexes: try: extra_src = project.find_source(idx) except SourceNotFound: - extra_src = idx - if extra_src != index: src_name = project.src_name_from_url(idx) - sources.append({"url": extra_src, "verify_ssl": True, "name": src_name}) + verify_ssl = True if idx not in trusted_hosts else False + extra_src = {"url": idx, "verify_ssl": verify_ssl, "name": extra_src} + if extra_src["url"] != index_source["url"]: + sources.append(extra_src) else: for idx in project.pipfile_sources: if idx["url"] != sources[0]["url"]: From a82f429fe9e9669389e996b901346c6278804fba Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 10 Oct 2018 17:21:25 -0400 Subject: [PATCH 5/6] Fix pipfile entry formation Signed-off-by: Dan Ryan --- pipenv/utils.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pipenv/utils.py b/pipenv/utils.py index 88622b68..4c65f363 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -489,9 +489,14 @@ def resolve_deps( # if verbose: # print('Error generating hash for {}'.format(name)) req.hashes = sorted(set(collected_hashes)) - name, entry = req.pipfile_entry + name, _entry = req.pipfile_entry + entry = {} + if isinstance(_entry, six.string_types): + entry["version"] = _entry + else: + entry["version"] = version + entry.update(_entry) entry["name"] = name - entry["version"] = version # if index: # d.update({"index": index}) if markers_lookup.get(result.name): From b34b5fd95135bd9106eab6376bdd5a13165faf45 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 10 Oct 2018 17:36:14 -0400 Subject: [PATCH 6/6] Fix versions Signed-off-by: Dan Ryan --- pipenv/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pipenv/utils.py b/pipenv/utils.py index 4c65f363..f3599301 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -492,10 +492,10 @@ def resolve_deps( name, _entry = req.pipfile_entry entry = {} if isinstance(_entry, six.string_types): - entry["version"] = _entry + entry["version"] = _entry.lstrip("=") else: - entry["version"] = version entry.update(_entry) + entry["version"] = version entry["name"] = name # if index: # d.update({"index": index})