Safer open_file for remote files and don't collect bad hashes

pipenv/project.py

@@ -317,7 +317,9 @@ class Project:
                     else:  # Fallback to downloading the file to obtain hash
                         package_url = urljoin(source["url"], package_url)
                         link = Link(package_url)
-                        collected_hashes.add(self.get_file_hash(session, link))
+                        file_hash = self.get_file_hash(session, link)
+                        if file_hash:
+                            collected_hashes.add(file_hash)
             return self.prepend_hash_types(collected_hashes, FAVORITE_HASH)
         except (ValueError, KeyError, ConnectionError):
             if self.s.is_verbose():
@@ -333,6 +335,8 @@ class Project:
         h = hashlib.new(FAVORITE_HASH)
         err.print(f"Downloading file {link.filename} to obtain hash...")
         with open_file(link.url, session) as fp:
+            if fp is None:
+                return None
             for chunk in iter(lambda: fp.read(8096), b""):
                 h.update(chunk)
         return f"{h.name}:{h.hexdigest()}"

pipenv/utils/fileutils.py

@@ -1,22 +1,20 @@
 """A collection for utilities for working with files and paths."""
 import atexit
+import io
 import os
 import sys
 import warnings
 from contextlib import contextmanager
 from pathlib import Path
 from tempfile import TemporaryDirectory
-from typing import IO, Any, ContextManager, Optional, TypeVar, Union
+from typing import Any, Optional
 from urllib import parse as urllib_parse
 from urllib import request as urllib_request
 from urllib.parse import quote, urlparse
 
-from pipenv.patched.pip._internal.vcs import RemoteNotFoundError
 from pipenv.patched.pip._vendor.requests import Session
 from pipenv.utils import err
 
-_T = TypeVar("_T")
-
 
 def is_file_url(url: Any) -> bool:
     """Returns true if the given url is a file url."""
@@ -116,27 +114,16 @@ def path_to_url(path):
 
 
 @contextmanager
-def open_file(
-    link: Union[_T, str], session: Optional[Session] = None, stream: bool = True
-) -> ContextManager[IO[bytes]]:
+def open_file(link, session: Optional[Session] = None, stream: bool = True):
     """Open local or remote file for reading.
 
-    :param pipenv.patched.pip._internal.index.Link link: A link object from resolving dependencies with
-        pip, or else a URL.
-    :param Optional[Session] session: A :class:`~requests.Session` instance
-    :param bool stream: Whether to stream the content if remote, default True
-    :raises ValueError: If link points to a local directory.
-    :return: a context manager to the opened file-like object
+    Other details...
     """
     if not isinstance(link, str):
         try:
             link = link.url_without_fragment
         except AttributeError:
-            err.print(
-                f"Cannot parse url from unknown type: {link!r}; Skipping ...",
-                style="bold red",
-            )
-            return None
+            raise ValueError(f"Cannot parse url from unknown type: {link!r}")
 
     if not is_valid_url(link) and os.path.exists(link):
         link = path_to_url(link)
@@ -154,21 +141,14 @@ def open_file(
         headers = {"Accept-Encoding": "identity"}
         if not session:
             session = Session()
-            with session.get(link, headers=headers, stream=stream) as resp:
-                if resp.status_code != 200:
-                    raise RemoteNotFoundError(
-                        f"HTTP error {resp.status_code} while getting {link}"
-                    )
-                try:
-                    raw = getattr(resp, "raw", None)
-                    result = raw if raw else resp
-                    yield result
-                finally:
-                    if raw:
-                        conn = raw._connection
-                        if conn is not None:
-                            conn.close()
-                    result.close()
+        resp = session.get(link, headers=headers, stream=stream)
+        if resp.status_code != 200:
+            err.print(f"HTTP error {resp.status_code} while getting {link}")
+            yield None
+        else:
+            # Creating a buffer-like object
+            buffer = io.BytesIO(resp.content)
+            yield buffer
 
 
 @contextmanager

tasks/vendoring/__init__.py

@@ -795,4 +795,7 @@ def vendor_artifact(ctx, package, version=None):
         dest_file = dest_dir / dest_path
         with open(dest_file.as_posix(), "wb") as target_handle:
             with open_file(link) as fp:
+                if fp is None:
+                    print(f"Error downloading {link}")
+                    continue
                 shutil.copyfileobj(fp, target_handle)