From 17888fb1319e3ed9b3e2e720d89e6e750f0ef1df Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 2 May 2018 02:00:28 -0400 Subject: [PATCH] Auto-download pip licenses Signed-off-by: Dan Ryan --- pipenv/vendor/vendor_pip.txt | 21 ++++ tasks/vendoring/__init__.py | 100 +++++++----------- .../patched/pipfile-update-pypi-uri.patch | 13 --- 3 files changed, 57 insertions(+), 77 deletions(-) create mode 100644 pipenv/vendor/vendor_pip.txt delete mode 100644 tasks/vendoring/patches/patched/pipfile-update-pypi-uri.patch diff --git a/pipenv/vendor/vendor_pip.txt b/pipenv/vendor/vendor_pip.txt new file mode 100644 index 00000000..53181c39 --- /dev/null +++ b/pipenv/vendor/vendor_pip.txt @@ -0,0 +1,21 @@ +setuptools==39.1.0 +appdirs==1.4.0 +distlib==0.2.4 +distro==1.2.0 +html5lib==1.0b10 +six==1.10.0 +colorama==0.3.7 +requests==2.18.4 +chardet==3.0.4 +idna==2.6 +urllib3==1.22 +certifi==2018.1.18 +CacheControl==0.11.7 +lockfile==0.12.2 +ordereddict==1.1 +progress==1.2 +ipaddress==1.0.17 +packaging==16.8 +pyparsing==2.1.10 +retrying==1.3.3 +webencodings==0.5 diff --git a/tasks/vendoring/__init__.py b/tasks/vendoring/__init__.py index 37db03fc..5280b799 100644 --- a/tasks/vendoring/__init__.py +++ b/tasks/vendoring/__init__.py @@ -17,13 +17,14 @@ import requests TASK_NAME = 'update' -LIBRARY_OVERRIDES = { +LIBRARY_DIRNAMES = { 'requirements-parser': 'requirements', 'backports.shutil_get_terminal_size': 'backports/shutil_get_terminal_size', 'backports.weakref': 'backports/weakref', 'shutil_backports': 'backports/shutil_get_terminal_size', 'python-dotenv': 'dotenv', - 'pip-tools': 'piptools' + 'pip-tools': 'piptools', + 'setuptools': 'pkg_resources', } # from time to time, remove the no longer needed ones @@ -38,7 +39,10 @@ HARDCODED_LICENSE_URLS = { 'semver': 'https://raw.githubusercontent.com/k-bx/python-semver/master/LICENSE.txt', 'crayons': 'https://raw.githubusercontent.com/kennethreitz/crayons/master/LICENSE', 'pip-tools': 'https://raw.githubusercontent.com/jazzband/pip-tools/master/LICENSE', - 'pew': 'https://raw.githubusercontent.com/berdario/pew/master/LICENSE' + 'pew': 'https://raw.githubusercontent.com/berdario/pew/master/LICENSE', + 'pytoml': 'https://github.com/avakar/pytoml/raw/master/LICENSE', + 'webencodings': 'https://github.com/SimonSapin/python-webencodings/raw/' + 'master/LICENSE', } FILE_WHITE_LIST = ( @@ -50,7 +54,8 @@ FILE_WHITE_LIST = ( 'README.md', 'appdirs.py', 'safety.zip', - 'cacert.pem' + 'cacert.pem', + 'vendor_pip.txt', ) LIBRARY_RENAMES = { @@ -212,14 +217,6 @@ cli(prog_name="safety") else: lib = yaml_build_dir / 'lib3' / 'yaml' shutil.copytree(str(lib.absolute()), str(safety_dir / 'yaml{0}'.format(version_choices[0]))) -# yaml_init = yaml_dir / '__init__.py' -# yaml_init.write_text(""" -# import sys -# if sys.version_info[0] == 3: -# from .yaml3 import * -# else: -# from .yaml2 import * -# """.strip()) requests_dir = safety_dir / 'requests' cacert = vendor_dir / 'requests' / 'cacert.pem' if not cacert.exists(): @@ -247,8 +244,8 @@ cli(prog_name="safety") def rename_if_needed(ctx, vendor_dir, item): rename_dict = LIBRARY_RENAMES if vendor_dir.name != 'patched' else PATCHED_RENAMES new_path = None - if item.name in rename_dict or item.name in LIBRARY_OVERRIDES: - new_name = rename_dict.get(item.name, LIBRARY_OVERRIDES.get(item.name)) + if item.name in rename_dict or item.name in LIBRARY_DIRNAMES: + new_name = rename_dict.get(item.name, LIBRARY_DIRNAMES.get(item.name)) new_path = item.parent / new_name log('Renaming %s => %s' % (item.name, new_path)) # handle existing directories @@ -307,7 +304,6 @@ def vendor(ctx, vendor_dir, rewrite=True): apply_patch(ctx, patch) # Global import rewrites - # log("Rewriting all imports related to vendored libs") log('Renaming specified libs...') for item in vendor_dir.iterdir(): if item.is_dir(): @@ -401,16 +397,17 @@ def find_and_extract_license(vendor_dir, tar, members): def license_fallback(vendor_dir, sdist_name): """Hardcoded license URLs. Check when updating if those are still needed""" - for libname, url in HARDCODED_LICENSE_URLS.items(): - if libname in sdist_name: - _, _, name = url.rpartition('/') - dest = license_destination(vendor_dir, libname, name) - r = requests.get(url, allow_redirects=True) - log('Downloading {}'.format(url)) - r.raise_for_status() - dest.write_bytes(r.content) - return - raise ValueError('No hardcoded URL for {} license'.format(sdist_name)) + libname = libname_from_dir(sdist_name) + if libname not in HARDCODED_LICENSE_URLS: + raise ValueError('No hardcoded URL for {} license'.format(libname)) + + url = HARDCODED_LICENSE_URLS[libname] + _, _, name = url.rpartition('/') + dest = license_destination(vendor_dir, libname, name) + r = requests.get(url, allow_redirects=True) + log('Downloading {}'.format(url)) + r.raise_for_status() + dest.write_bytes(r.content) def libname_from_dir(dirname): @@ -420,7 +417,7 @@ def libname_from_dir(dirname): if part[0].isdigit(): break parts.append(part) - return'-'.join(parts) + return '-'.join(parts) def license_destination(vendor_dir, libname, filename): @@ -432,16 +429,17 @@ def license_destination(vendor_dir, libname, filename): if lowercase.is_dir(): return lowercase / filename rename_dict = LIBRARY_RENAMES if vendor_dir.name != 'patched' else PATCHED_RENAMES + # Short circuit all logic if we are renaming the whole library if libname in rename_dict: return vendor_dir / rename_dict[libname] / filename - if libname in LIBRARY_OVERRIDES: - override = vendor_dir / LIBRARY_OVERRIDES[libname] + if libname in LIBRARY_DIRNAMES: + override = vendor_dir / LIBRARY_DIRNAMES[libname] if not override.exists() and override.parent.exists(): # for flattened subdeps, specifically backports/weakref.py - target_dir = vendor_dir / override.parent - target_file = '{0}.{1}'.format(override.name, filename) - return target_dir / target_file - return vendor_dir / LIBRARY_OVERRIDES[libname] / filename + return ( + vendor_dir / override.parent + ) / '{0}.{1}'.format(override.name, filename) + return vendor_dir / LIBRARY_DIRNAMES[libname] / filename # fallback to libname.LICENSE (used for nondirs) return vendor_dir / '{}.{}'.format(libname, filename) @@ -451,8 +449,6 @@ def extract_license_member(vendor_dir, tar, member, name): dirname = list(mpath.parents)[-2].name # -1 is . libname = libname_from_dir(dirname) dest = license_destination(vendor_dir, libname, mpath.name) - # dest_relative = dest.relative_to(Path.cwd()) - # log('Extracting {} into {}'.format(name, dest_relative)) log('Extracting {} into {}'.format(name, dest)) try: fileobj = tar.extractfile(member) @@ -461,36 +457,6 @@ def extract_license_member(vendor_dir, tar, member, name): dest.write_bytes(tar.read(member)) -@invoke.task -def update_stubs(ctx): - vendor_dir = _get_vendor_dir(ctx) - vendored_libs = detect_vendored_libs(vendor_dir) - - print("[vendoring.update_stubs] Add mypy stubs") - - extra_stubs_needed = { - # Some projects need stubs other than a simple .pyi - "six": ["six.__init__", "six.moves"], - # Some projects should not have stubs coz they're single file modules - "appdirs": [], - } - - for lib in vendored_libs: - if lib not in extra_stubs_needed: - (vendor_dir / (lib + ".pyi")).write_text("from %s import *" % lib) - continue - - for selector in extra_stubs_needed[lib]: - fname = selector.replace(".", os.sep) + ".pyi" - if selector.endswith(".__init__"): - selector = selector[:-9] - - f_path = vendor_dir / fname - if not f_path.parent.exists(): - f_path.parent.mkdir() - f_path.write_text("from %s import *" % selector) - - @invoke.task(name=TASK_NAME) def main(ctx): vendor_dir = _get_vendor_dir(ctx) @@ -502,5 +468,11 @@ def main(ctx): vendor(ctx, patched_dir, rewrite=False) download_licenses(ctx, vendor_dir) download_licenses(ctx, patched_dir, 'patched.txt') + for pip_dir in [vendor_dir / 'pip9', patched_dir / 'notpip']: + _vendor_dir = pip_dir / '_vendor' + vendor_src_file = vendor_dir / 'vendor_pip.txt' + vendor_file = _vendor_dir / 'vendor.txt' + vendor_file.write_bytes(vendor_src_file.read_bytes()) + download_licenses(ctx, _vendor_dir) # update_safety(ctx) log('Revendoring complete') diff --git a/tasks/vendoring/patches/patched/pipfile-update-pypi-uri.patch b/tasks/vendoring/patches/patched/pipfile-update-pypi-uri.patch deleted file mode 100644 index ecbe2aac..00000000 --- a/tasks/vendoring/patches/patched/pipfile-update-pypi-uri.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/pipenv/patched/pipfile/api.py b/pipenv/patched/pipfile/api.py -index 18a1ea2..e8fa027 100644 ---- a/pipenv/patched/pipfile/api.py -+++ b/pipenv/patched/pipfile/api.py -@@ -10,7 +10,7 @@ import os - - - DEFAULT_SOURCE = { -- u'url': u'https://pypi.python.org/simple', -+ u'url': u'https://pypi.org/simple', - u'verify_ssl': True, - u'name': u'pypi', - }