From 1d751a13f1cd61a15998740151df02be2565bb19 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sat, 18 Nov 2017 19:02:39 -0500 Subject: [PATCH 01/46] Properly split and parse requirements lines - Split out markers and local paths - Fixes #870 and #1083 and #858 - Also splits markers properly using pip's approach - The requirements parser we use really needs a lot of help --- pipenv/utils.py | 40 ++++++++++++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/pipenv/utils.py b/pipenv/utils.py index 8d4632a9..919f8241 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -277,15 +277,33 @@ def get_requirement(dep): remote URIs, and package names, and that we pass only valid requirement strings to the requirements parser. Performs necessary modifications to requirements object if the user input was a local relative path. + + :param str dep: A requirement line + :returns: :class:`requirements.Requirement` object """ path = None + # Split out markers if they are present - similar to how pip does it + # See pip.req.req_install.InstallRequirement.from_line + if not any(dep.startswith(uri_prefix) for uri_prefix in SCHEME_LIST): + marker_sep = ';' + else: + marker_sep = '; ' + if marker_sep in dep: + dep, markers = dep.split(marker_sep, 1) + markers = markers.strip() + if not markers: + markers = None + else: + markers = None + # Strip extras from the requirement so we can make a properly parseable req + dep, extras = pip.req.req_install._strip_extras(dep) # Only operate on local, existing, non-URI formatted paths if (is_file(dep) and isinstance(dep, six.string_types) and not any(dep.startswith(uri_prefix) for uri_prefix in SCHEME_LIST)): dep_path = Path(dep) # Only parse if it is a file or an installable dir if dep_path.is_file() or (dep_path.is_dir() and pip.utils.is_installable_dir(dep)): - if dep_path.is_absolute(): + if dep_path.is_absolute() or dep_path.as_posix() == '.': path = dep else: path = get_converted_relative_path(dep) @@ -296,6 +314,11 @@ def get_requirement(dep): if req.local_file and req.uri and not req.path and path: req.path = path req.uri = None + if markers: + req.markers = markers + if extras: + # Bizarrely this is also what pip does... + req.extras = [r for r in requirements.parse('fakepkg{0}'.format(extras))][0].extras return req @@ -611,28 +634,26 @@ def convert_deps_from_pip(dep): hashable_path = req.uri if req.uri else req.path req.name = hashlib.sha256(hashable_path.encode('utf-8')).hexdigest() req.name = req.name[len(req.name) - 7:] - # {path: uri} TOML (spec 4 I guess...) if req.uri: dependency[req.name] = {'file': hashable_path} else: dependency[req.name] = {'path': hashable_path} + if req.extras: + dependency[req.name].update(extras) + # Add --editable if applicable if req.editable: dependency[req.name].update({'editable': True}) # VCS Installs. Extra check for unparsed git over SSH - if req.vcs or is_vcs(req.path): + elif req.vcs or is_vcs(req.path): if req.name is None: raise ValueError('pipenv requires an #egg fragment for version controlled ' 'dependencies. Please install remote dependency ' 'in the form {0}#egg=.'.format(req.uri)) - # Extras: e.g. #egg=requests[security] - if req.extras: - dependency[req.name] = extras - # Set up this requirement as a proper VCS requirement if it was not if not req.vcs and req.path.startswith(VCS_LIST): req.vcs = [vcs for vcs in VCS_LIST if req.path.startswith(vcs)][0] @@ -654,6 +675,10 @@ def convert_deps_from_pip(dep): if req.revision: dependency[req.name].update({'ref': req.revision}) + # Extras: e.g. #egg=requests[security] + if req.extras: + dependency[req.name].update({'extras': req.extras}) + elif req.extras or req.specs: specs = None @@ -679,7 +704,6 @@ def convert_deps_from_pip(dep): for key in dependency.copy(): if not hasattr(dependency[key], 'keys'): del dependency[key] - return dependency From 0a5e5dab48400bb290bd5f9d8b6df071285ffad5 Mon Sep 17 00:00:00 2001 From: Erin O'Connell Date: Sat, 18 Nov 2017 23:18:48 -0700 Subject: [PATCH 02/46] removed unnecessary encoding --- pipenv/vendor/dotenv/main.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pipenv/vendor/dotenv/main.py b/pipenv/vendor/dotenv/main.py index ba74cb63..ba150270 100644 --- a/pipenv/vendor/dotenv/main.py +++ b/pipenv/vendor/dotenv/main.py @@ -102,13 +102,11 @@ def parse_dotenv(dotenv_path): if not line or line.startswith('#') or '=' not in line: continue k, v = line.split('=', 1) - # Remove any leading and trailing spaces in key, value - k, v = k.strip(), v.strip().encode('unicode-escape').decode('ascii') + k, v = k.strip(), v.strip() if len(v) > 0: quoted = v[0] == v[len(v) - 1] in ['"', "'"] - if quoted: v = decode_escaped(v[1:-1]) From aac0ef92a54519bea17f1c9c8f4c6f278c4d408a Mon Sep 17 00:00:00 2001 From: Erin O'Connell Date: Sun, 19 Nov 2017 15:17:51 -0700 Subject: [PATCH 03/46] moved dotenv from vendor to patched --- pipenv/{vendor => patched}/dotenv/__init__.py | 0 pipenv/{vendor => patched}/dotenv/cli.py | 0 pipenv/{vendor => patched}/dotenv/ipython.py | 0 pipenv/{vendor => patched}/dotenv/main.py | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename pipenv/{vendor => patched}/dotenv/__init__.py (100%) rename pipenv/{vendor => patched}/dotenv/cli.py (100%) rename pipenv/{vendor => patched}/dotenv/ipython.py (100%) rename pipenv/{vendor => patched}/dotenv/main.py (100%) diff --git a/pipenv/vendor/dotenv/__init__.py b/pipenv/patched/dotenv/__init__.py similarity index 100% rename from pipenv/vendor/dotenv/__init__.py rename to pipenv/patched/dotenv/__init__.py diff --git a/pipenv/vendor/dotenv/cli.py b/pipenv/patched/dotenv/cli.py similarity index 100% rename from pipenv/vendor/dotenv/cli.py rename to pipenv/patched/dotenv/cli.py diff --git a/pipenv/vendor/dotenv/ipython.py b/pipenv/patched/dotenv/ipython.py similarity index 100% rename from pipenv/vendor/dotenv/ipython.py rename to pipenv/patched/dotenv/ipython.py diff --git a/pipenv/vendor/dotenv/main.py b/pipenv/patched/dotenv/main.py similarity index 100% rename from pipenv/vendor/dotenv/main.py rename to pipenv/patched/dotenv/main.py From ddde2b227eaca079ef4ba0b0b667fd81972d0db9 Mon Sep 17 00:00:00 2001 From: Erin O'Connell Date: Sun, 19 Nov 2017 15:50:05 -0700 Subject: [PATCH 04/46] merging a PR from the official dotenv repo that addresses the newline issue --- pipenv/patched/dotenv/main.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/pipenv/patched/dotenv/main.py b/pipenv/patched/dotenv/main.py index ba150270..59f05174 100644 --- a/pipenv/patched/dotenv/main.py +++ b/pipenv/patched/dotenv/main.py @@ -97,16 +97,10 @@ def dotenv_values(dotenv_path): def parse_dotenv(dotenv_path): with open(dotenv_path) as f: - for line in f: - line = line.strip() - if not line or line.startswith('#') or '=' not in line: - continue - k, v = line.split('=', 1) - # Remove any leading and trailing spaces in key, value - k, v = k.strip(), v.strip() - + for k, v in re.findall('^\s*(\w*)\s*=\s*("[^"]*"|[^\s]*)\s*$', f.read(), flags=re.MULTILINE): if len(v) > 0: quoted = v[0] == v[len(v) - 1] in ['"', "'"] + if quoted: v = decode_escaped(v[1:-1]) From 35d5457823cca234c588fd3b92fcbbc75b12d79b Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 19 Nov 2017 20:49:49 -0500 Subject: [PATCH 05/46] Add tests for local package extras - Will be even more complete with #1098 --- tests/test_pipenv.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tests/test_pipenv.py b/tests/test_pipenv.py index 280b1df0..453dc66d 100644 --- a/tests/test_pipenv.py +++ b/tests/test_pipenv.py @@ -334,6 +334,39 @@ class TestPipenv: assert 'urllib3' in p.lockfile['default'] assert 'pysocks' in p.lockfile['default'] + @pytest.mark.extras + @pytest.mark.install + @pytest.mark.local + def test_local_extras_install(self): + with PipenvInstance() as p: + setup_py = os.path.join(p.path, 'setup.py') + with open(setup_py, 'w') as fh: + contents = """ +from setuptools import setup, find_packages + +setup( + name='test_pipenv', + version='0.1', + description='Pipenv Test Package', + author='Pipenv Test', + author_email='test@pipenv.package', + license='PIPENV', + packages=find_packages(), + install_requires=['tablib'], + extras_require={'dev': ['flake8', 'pylint']}, + zip_safe=False +) + """.strip() + fh.write(contents) + c = p.pipenv('install .[dev]') + assert c.return_code == 0 + key = [k for k in p.pipfile['packages'].keys()][0] + dep = p.pipfile['packages'][key] + assert dep['path'] == '.' + assert dep['extras'] == ['dev'] + assert key in p.lockfile['default'] + assert 'dev' in p.lockfile['default'][key]['extras'] + @pytest.mark.vcs @pytest.mark.install def test_basic_vcs_install(self): From 22df63ea711150fcaa042b393dfab3e115fb165d Mon Sep 17 00:00:00 2001 From: Erin O'Connell Date: Sun, 19 Nov 2017 21:53:07 -0700 Subject: [PATCH 06/46] Update HISTORY.txt --- HISTORY.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HISTORY.txt b/HISTORY.txt index b006d2e2..fad70831 100644 --- a/HISTORY.txt +++ b/HISTORY.txt @@ -1,5 +1,5 @@ -8.3.3 - Added enhancement for pip-tools to resolve dependecies with specific versions of python + - Fixed bug where newlines were not escaped in .env files when loaded 8.3.2: - Moved automated update check to once every 24 hours. - Better default for PYENV_ROOT. From 11e878d81930fd3adc26d6a49557bfa213d77e7a Mon Sep 17 00:00:00 2001 From: Erin O'Connell Date: Mon, 20 Nov 2017 19:30:58 -0700 Subject: [PATCH 07/46] hid the search bar while links are broken --- docs/_templates/hacks.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/_templates/hacks.html b/docs/_templates/hacks.html index 45ef5859..52c0e339 100644 --- a/docs/_templates/hacks.html +++ b/docs/_templates/hacks.html @@ -6,8 +6,8 @@ Alabaster (krTheme++) Hacks --> /* Rezzy requires precise alignment. */ img.logo {margin-left: -20px!important;} - /* "Quick Search" should be capitalized. */ - div#searchbox h3 {text-transform: capitalize;} + /* "Quick Search" should be not be shown for now. */ + div#searchbox h3 {display: none;} /* Make the document a little wider, less code is cut-off. */ div.document {width: 1008px;} @@ -69,4 +69,4 @@ Alabaster (krTheme++) Hacks --> - +