From 0e48c2ab55aa400fee5d971d54840762e1549630 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Fri, 6 Apr 2018 05:37:51 -0700 Subject: [PATCH 1/5] Use util.rmtree() in cleanup_virtualenv() Issue #1910 --- pipenv/core.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pipenv/core.py b/pipenv/core.py index 9967eed0..33450d49 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -47,6 +47,7 @@ from .utils import ( is_pinned, is_star, TemporaryDirectory, + rmtree, ) from .import pep508checker, progress from .environments import ( @@ -168,7 +169,7 @@ def cleanup_virtualenv(bare=True): click.echo(crayons.red('Environment creation aborted.')) try: # Delete the virtualenv. - shutil.rmtree(project.virtualenv_location, ignore_errors=True) + rmtree(project.virtualenv_location) except OSError as e: click.echo(e) From 1cca6f19a6a1d9d6401811c367006c855fc486ea Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Fri, 6 Apr 2018 06:57:27 -0700 Subject: [PATCH 2/5] Improve cleanup_virtualenv() error message Issue #1910 --- pipenv/core.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pipenv/core.py b/pipenv/core.py index 33450d49..c74aa598 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -171,7 +171,14 @@ def cleanup_virtualenv(bare=True): # Delete the virtualenv. rmtree(project.virtualenv_location) except OSError as e: - click.echo(e) + click.echo( + '{0} An error occurred while removing {1}!'.format( + crayons.red('Error: ', bold=True), + crayons.green(project.virtualenv_location), + ), + err=True, + ) + click.echo(crayons.blue(e), err=True) def import_requirements(r=None, dev=False): From b653017168bbaf257a4500709da4b0a8d1ee5c24 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Sat, 7 Apr 2018 01:13:36 +0800 Subject: [PATCH 3/5] Update python-dotenv to 0.8.2 --- pipenv/patched/dotenv/__init__.py | 42 +++++- pipenv/patched/dotenv/cli.py | 38 ++---- pipenv/patched/dotenv/compat.py | 4 + pipenv/patched/dotenv/ipython.py | 41 ++++++ pipenv/patched/dotenv/main.py | 205 +++++++++++++++++++---------- pipenv/patched/dotenv/test_main.py | 85 ------------ 6 files changed, 232 insertions(+), 183 deletions(-) create mode 100644 pipenv/patched/dotenv/compat.py create mode 100644 pipenv/patched/dotenv/ipython.py delete mode 100644 pipenv/patched/dotenv/test_main.py diff --git a/pipenv/patched/dotenv/__init__.py b/pipenv/patched/dotenv/__init__.py index cbe5930c..50f27cd4 100644 --- a/pipenv/patched/dotenv/__init__.py +++ b/pipenv/patched/dotenv/__init__.py @@ -1,4 +1,40 @@ -from .cli import get_cli_string -from .main import load_dotenv, get_key, set_key, unset_key, find_dotenv +from .main import load_dotenv, get_key, set_key, unset_key, find_dotenv, dotenv_values -__all__ = ['get_cli_string', 'load_dotenv', 'get_key', 'set_key', 'unset_key', 'find_dotenv'] + +def load_ipython_extension(ipython): + from .ipython import load_ipython_extension + load_ipython_extension(ipython) + + +def get_cli_string(path=None, action=None, key=None, value=None, quote=None): + """Returns a string suitable for running as a shell script. + + Useful for converting a arguments passed to a fabric task + to be passed to a `local` or `run` command. + """ + command = ['dotenv'] + if quote: + command.append('-q %s' % quote) + if path: + command.append('-f %s' % path) + if action: + command.append(action) + if key: + command.append(key) + if value: + if ' ' in value: + command.append('"%s"' % value) + else: + command.append(value) + + return ' '.join(command).strip() + + +__all__ = ['get_cli_string', + 'load_dotenv', + 'dotenv_values', + 'get_key', + 'set_key', + 'unset_key', + 'find_dotenv', + 'load_ipython_extension'] diff --git a/pipenv/patched/dotenv/cli.py b/pipenv/patched/dotenv/cli.py index bd9bd7a7..dd7c2418 100644 --- a/pipenv/patched/dotenv/cli.py +++ b/pipenv/patched/dotenv/cli.py @@ -1,8 +1,14 @@ import os +import sys -import click +try: + import click +except ImportError: + sys.stderr.write('It seems python-dotenv is not installed with cli option. \n' + 'Run pip install "python-dotenv[cli]" to fix this.') + sys.exit(1) -from .main import get_key, dotenv_values, set_key, unset_key +from .main import dotenv_values, get_key, set_key, unset_key @click.group() @@ -27,7 +33,7 @@ def list(ctx): file = ctx.obj['FILE'] dotenv_as_dict = dotenv_values(file) for k, v in dotenv_as_dict.items(): - click.echo('%s="%s"' % (k, v)) + click.echo('%s=%s' % (k, v)) @cli.command() @@ -40,7 +46,7 @@ def set(ctx, key, value): quote = ctx.obj['QUOTE'] success, key, value = set_key(file, key, value, quote) if success: - click.echo('%s="%s"' % (key, value)) + click.echo('%s=%s' % (key, value)) else: exit(1) @@ -53,7 +59,7 @@ def get(ctx, key): file = ctx.obj['FILE'] stored_value = get_key(file, key) if stored_value: - click.echo('%s="%s"' % (key, stored_value)) + click.echo('%s=%s' % (key, stored_value)) else: exit(1) @@ -72,27 +78,5 @@ def unset(ctx, key): exit(1) -def get_cli_string(path=None, action=None, key=None, value=None): - """Returns a string suitable for running as a shell script. - - Useful for converting a arguments passed to a fabric task - to be passed to a `local` or `run` command. - """ - command = ['dotenv'] - if path: - command.append('-f %s' % path) - if action: - command.append(action) - if key: - command.append(key) - if value: - if ' ' in value: - command.append('"%s"' % value) - else: - command.append(value) - - return ' '.join(command).strip() - - if __name__ == "__main__": cli() diff --git a/pipenv/patched/dotenv/compat.py b/pipenv/patched/dotenv/compat.py new file mode 100644 index 00000000..c4a481e6 --- /dev/null +++ b/pipenv/patched/dotenv/compat.py @@ -0,0 +1,4 @@ +try: + from StringIO import StringIO # noqa +except ImportError: + from io import StringIO # noqa diff --git a/pipenv/patched/dotenv/ipython.py b/pipenv/patched/dotenv/ipython.py new file mode 100644 index 00000000..06252f1e --- /dev/null +++ b/pipenv/patched/dotenv/ipython.py @@ -0,0 +1,41 @@ +from __future__ import print_function + +from IPython.core.magic import Magics, line_magic, magics_class +from IPython.core.magic_arguments import (argument, magic_arguments, + parse_argstring) + +from .main import find_dotenv, load_dotenv + + +@magics_class +class IPythonDotEnv(Magics): + + @magic_arguments() + @argument( + '-o', '--override', action='store_true', + help="Indicate to override existing variables" + ) + @argument( + '-v', '--verbose', action='store_true', + help="Indicate function calls to be verbose" + ) + @argument('dotenv_path', nargs='?', type=str, default='.env', + help='Search in increasingly higher folders for the `dotenv_path`') + @line_magic + def dotenv(self, line): + args = parse_argstring(self.dotenv, line) + # Locate the .env file + dotenv_path = args.dotenv_path + try: + dotenv_path = find_dotenv(dotenv_path, True, True) + except IOError: + print("cannot find .env file") + return + + # Load the .env file + load_dotenv(dotenv_path, verbose=args.verbose, override=args.override) + + +def load_ipython_extension(ipython): + """Register the %dotenv magic.""" + ipython.register_magics(IPythonDotEnv) diff --git a/pipenv/patched/dotenv/main.py b/pipenv/patched/dotenv/main.py index 65842d00..3d1bd72f 100644 --- a/pipenv/patched/dotenv/main.py +++ b/pipenv/patched/dotenv/main.py @@ -1,37 +1,113 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import codecs +import fileinput +import io import os +import re import sys import warnings -import re from collections import OrderedDict +from .compat import StringIO + __escape_decoder = codecs.getdecoder('unicode_escape') __posix_variable = re.compile('\$\{[^\}]*\}') -__variable_declaration = re.compile('^\s*(\w*)\s*=\s*("[^"]*"|\'[^\']*\'|[^\s]*)\s*$', - flags=re.MULTILINE) def decode_escaped(escaped): return __escape_decoder(escaped)[0] -def load_dotenv(dotenv_path, verbose=False, override=False): - """ - Read a .env file and load into os.environ. - """ - if not os.path.exists(dotenv_path): - if verbose: - warnings.warn("Not loading %s - it doesn't exist." % dotenv_path) - return None - for k, v in dotenv_values(dotenv_path).items(): - if override: +def parse_line(line): + line = line.strip() + + # Ignore lines with `#` or which doesn't have `=` in it. + if not line or line.startswith('#') or '=' not in line: + return None, None + + k, v = line.split('=', 1) + + if k.startswith('export '): + k = k.lstrip('export ') + + # Remove any leading and trailing spaces in key, value + k, v = k.strip(), v.strip() + + if v: + v = v.encode('unicode-escape').decode('ascii') + quoted = v[0] == v[-1] in ['"', "'"] + if quoted: + v = decode_escaped(v[1:-1]) + + return k, v + + +class DotEnv(): + + def __init__(self, dotenv_path, verbose=False): + self.dotenv_path = dotenv_path + self._dict = None + self.verbose = verbose + + def _get_stream(self): + self._is_file = False + if isinstance(self.dotenv_path, StringIO): + return self.dotenv_path + + if os.path.exists(self.dotenv_path): + self._is_file = True + return io.open(self.dotenv_path) + + if self.verbose: + warnings.warn("File doesn't exist {}".format(self.dotenv_path)) + + return StringIO('') + + def dict(self): + """Return dotenv as dict""" + if self._dict: + return self._dict + + values = OrderedDict(self.parse()) + self._dict = resolve_nested_variables(values) + return self._dict + + def parse(self): + f = self._get_stream() + + for line in f: + key, value = parse_line(line) + if not key: + continue + + yield key, value + + if self._is_file: + f.close() + + def set_as_environment_variables(self, override=False): + """ + Load the current dotenv as system environemt variable. + """ + for k, v in self.dict().items(): + if k in os.environ and not override: + continue os.environ[k] = v - else: - os.environ.setdefault(k, v) - return True + + return True + + def get(self, key): + """ + """ + data = self.dict() + + if key in data: + return data[key] + + if self.verbose: + warnings.warn("key %s not found in %s." % (key, self.dotenv_path)) def get_key(dotenv_path, key_to_get): @@ -40,16 +116,7 @@ def get_key(dotenv_path, key_to_get): If the .env path given doesn't exist, fails """ - key_to_get = str(key_to_get) - if not os.path.exists(dotenv_path): - warnings.warn("can't read %s - it doesn't exist." % dotenv_path) - return None - dotenv_as_dict = dotenv_values(dotenv_path) - if key_to_get in dotenv_as_dict: - return dotenv_as_dict[key_to_get] - else: - warnings.warn("key %s not found in %s." % (key_to_get, dotenv_path)) - return None + return DotEnv(dotenv_path, verbose=True).get(key_to_get) def set_key(dotenv_path, key_to_set, value_to_set, quote_mode="always"): @@ -59,15 +126,30 @@ def set_key(dotenv_path, key_to_set, value_to_set, quote_mode="always"): If the .env path given doesn't exist, fails instead of risking creating an orphan .env somewhere in the filesystem """ - key_to_set = str(key_to_set) - value_to_set = str(value_to_set).strip("'").strip('"') + value_to_set = value_to_set.strip("'").strip('"') if not os.path.exists(dotenv_path): warnings.warn("can't write to %s - it doesn't exist." % dotenv_path) return None, key_to_set, value_to_set - dotenv_as_dict = OrderedDict(parse_dotenv(dotenv_path)) - dotenv_as_dict[key_to_set] = value_to_set - success = flatten_and_write(dotenv_path, dotenv_as_dict, quote_mode) - return success, key_to_set, value_to_set + + if " " in value_to_set: + quote_mode = "always" + + line_template = '{}="{}"' if quote_mode == "always" else '{}={}' + line_out = line_template.format(key_to_set, value_to_set) + + replaced = False + for line in fileinput.input(dotenv_path, inplace=True): + k, v = parse_line(line) + if k == key_to_set: + replaced = True + line = line_out + print(line, end='') + + if not replaced: + with io.open(dotenv_path, "a") as f: + f.write("{}\n".format(line_out)) + + return True, key_to_set, value_to_set def unset_key(dotenv_path, key_to_unset, quote_mode="always"): @@ -77,36 +159,24 @@ def unset_key(dotenv_path, key_to_unset, quote_mode="always"): If the .env path given doesn't exist, fails If the given key doesn't exist in the .env, fails """ - key_to_unset = str(key_to_unset) + removed = False + if not os.path.exists(dotenv_path): warnings.warn("can't delete from %s - it doesn't exist." % dotenv_path) return None, key_to_unset - dotenv_as_dict = dotenv_values(dotenv_path) - if key_to_unset in dotenv_as_dict: - dotenv_as_dict.pop(key_to_unset, None) - else: + + for line in fileinput.input(dotenv_path, inplace=True): + k, v = parse_line(line) + if k == key_to_unset: + removed = True + line = '' + print(line, end='') + + if not removed: warnings.warn("key %s not removed from %s - key doesn't exist." % (key_to_unset, dotenv_path)) return None, key_to_unset - success = flatten_and_write(dotenv_path, dotenv_as_dict, quote_mode) - return success, key_to_unset - -def dotenv_values(dotenv_path): - values = OrderedDict(parse_dotenv(dotenv_path)) - values = resolve_nested_variables(values) - return values - - -def parse_dotenv(dotenv_path): - with open(dotenv_path) as f: - for k, v in __variable_declaration.findall(f.read()): - if len(v) > 0: - quoted = v[0] == v[len(v) - 1] in ['"', "'"] - - if quoted: - v = decode_escaped(v[1:-1]) - - yield k, v + return removed, key_to_unset def resolve_nested_variables(values): @@ -132,17 +202,6 @@ def resolve_nested_variables(values): return values -def flatten_and_write(dotenv_path, dotenv_as_dict, quote_mode="always"): - with open(dotenv_path, "w") as f: - for k, v in dotenv_as_dict.items(): - _mode = quote_mode - if _mode == "auto" and " " in v: - _mode = "always" - str_format = '%s="%s"\n' if _mode == "always" else '%s=%s\n' - f.write(str_format % (k, v)) - return True - - def _walk_to_root(path): """ Yield directories starting from the given directory up to the root @@ -184,3 +243,13 @@ def find_dotenv(filename='.env', raise_error_if_not_found=False, usecwd=False): raise IOError('File not found') return '' + + +def load_dotenv(dotenv_path=None, stream=None, verbose=False, override=False): + f = dotenv_path or stream or find_dotenv() + return DotEnv(f, verbose=verbose).set_as_environment_variables(override=override) + + +def dotenv_values(dotenv_path=None, stream=None, verbose=False): + f = dotenv_path or stream or find_dotenv() + return DotEnv(f, verbose=verbose).dict() diff --git a/pipenv/patched/dotenv/test_main.py b/pipenv/patched/dotenv/test_main.py deleted file mode 100644 index cd7638b5..00000000 --- a/pipenv/patched/dotenv/test_main.py +++ /dev/null @@ -1,85 +0,0 @@ -import os -from textwrap import dedent -import unittest - -from main import parse_dotenv - - -class TestParseDotenv(unittest.TestCase): - filename = 'testfile.conf' - - def tearDown(self): - if os.path.exists(self.filename): - os.remove(self.filename) - - def write_file(self, contents): - with open(self.filename, 'w') as f: - f.write(contents) - - def assert_parsed(self, *expected): - parsed = parse_dotenv(self.filename) - for expected_key, expected_val in expected: - actual_key, actual_val = next(parsed) - self.assertEqual(actual_key, expected_key) - self.assertEqual(actual_val, expected_val) - - with self.assertRaises(StopIteration): - next(parsed) - - def test_value_unquoted(self): - self.write_file(dedent(""" - var1 = value1 - var2=value2 - """)) - self.assert_parsed(('var1', 'value1'), ('var2', 'value2')) - - def test_value_double_quoted(self): - self.write_file(dedent(""" - var1 = "value1" - var2="value2" - """)) - self.assert_parsed(('var1', 'value1'), ('var2', 'value2')) - - def test_value_single_quoted(self): - self.write_file(dedent(""" - var1 = 'value1' - var2='value2' - """)) - self.assert_parsed(('var1', 'value1'), ('var2', 'value2')) - - def test_value_with_space_double_quoted(self): - self.write_file(dedent(""" - var1 = "value1 with spaces" - var2 = "othervalue" - """)) - self.assert_parsed(('var1', 'value1 with spaces'), - ('var2', 'othervalue')) - - def test_value_with_space_single_quoted(self): - self.write_file(dedent(""" - var1 = 'value with spaces' - var2 = 'othervalue' - """)) - self.assert_parsed(('var1', 'value with spaces'), - ('var2', 'othervalue')) - - def test_values_with_mixed_quotes_and_spaces(self): - self.write_file(dedent(""" - var1 = 'value with spaces' - var2= othervalue - var3="double-quoted value with spaces" - var4 = "double-quoted value - with - newlines" - var5='single quote - and newline' - """)) - self.assert_parsed(('var1', 'value with spaces'), - ('var2', 'othervalue'), - ('var3', 'double-quoted value with spaces'), - ('var4', 'double-quoted value\nwith\nnewlines'), - ('var5', 'single quote\nand newline')) - - -if __name__ == '__main__': - unittest.main() From ce584f0713f7b8fd7f7acf1794ac715c5b9d69bb Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Sat, 7 Apr 2018 01:17:48 +0800 Subject: [PATCH 4/5] Move dotenv back to vendor and add README --- pipenv/vendor/README.md | 9 +++++++++ pipenv/{patched => vendor}/dotenv/__init__.py | 0 pipenv/{patched => vendor}/dotenv/cli.py | 0 pipenv/{patched => vendor}/dotenv/compat.py | 0 pipenv/{patched => vendor}/dotenv/ipython.py | 0 pipenv/{patched => vendor}/dotenv/main.py | 0 6 files changed, 9 insertions(+) create mode 100644 pipenv/vendor/README.md rename pipenv/{patched => vendor}/dotenv/__init__.py (100%) rename pipenv/{patched => vendor}/dotenv/cli.py (100%) rename pipenv/{patched => vendor}/dotenv/compat.py (100%) rename pipenv/{patched => vendor}/dotenv/ipython.py (100%) rename pipenv/{patched => vendor}/dotenv/main.py (100%) diff --git a/pipenv/vendor/README.md b/pipenv/vendor/README.md new file mode 100644 index 00000000..431693d8 --- /dev/null +++ b/pipenv/vendor/README.md @@ -0,0 +1,9 @@ +# Vendored packages + +These packages are copied as-is from upstream to reduce Pipenv dependencies. +They should always be kept synced with upstream. DO NOT MODIFY DIRECTLY! If +you need to patch anything, move the package to `patched`. + +Known vendored versions: + +- python-dotenv: 0.8.2 diff --git a/pipenv/patched/dotenv/__init__.py b/pipenv/vendor/dotenv/__init__.py similarity index 100% rename from pipenv/patched/dotenv/__init__.py rename to pipenv/vendor/dotenv/__init__.py diff --git a/pipenv/patched/dotenv/cli.py b/pipenv/vendor/dotenv/cli.py similarity index 100% rename from pipenv/patched/dotenv/cli.py rename to pipenv/vendor/dotenv/cli.py diff --git a/pipenv/patched/dotenv/compat.py b/pipenv/vendor/dotenv/compat.py similarity index 100% rename from pipenv/patched/dotenv/compat.py rename to pipenv/vendor/dotenv/compat.py diff --git a/pipenv/patched/dotenv/ipython.py b/pipenv/vendor/dotenv/ipython.py similarity index 100% rename from pipenv/patched/dotenv/ipython.py rename to pipenv/vendor/dotenv/ipython.py diff --git a/pipenv/patched/dotenv/main.py b/pipenv/vendor/dotenv/main.py similarity index 100% rename from pipenv/patched/dotenv/main.py rename to pipenv/vendor/dotenv/main.py From e46cabcb2c555d53b62eb56b029f52bf2c3eee91 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Thu, 5 Apr 2018 17:29:02 +0800 Subject: [PATCH 5/5] Fix Python version parser for 2.7.0 Fix #1893. --- pipenv/utils.py | 48 ++++++++++++++++++++++++++++++--------------- tests/test_utils.py | 20 +++++++++++++++++++ 2 files changed, 52 insertions(+), 16 deletions(-) diff --git a/pipenv/utils.py b/pipenv/utils.py index de2d4fe3..80db89d4 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import errno import os +import re import hashlib import tempfile import sys @@ -175,29 +176,44 @@ def cleanup_toml(tml): return toml +def parse_python_version(output): + """Parse a Python version output returned by `python --version`. + + Return a dict with three keys: major, minor, and micro. Each value is a + string containing a version part. + + Note: The micro part would be `'0'` if it's missing from the input string. + """ + version_pattern = re.compile(r''' + ^ # Beginning of line. + Python # Literally "Python". + \s # Space. + (?P\d+) # Major = one or more digits. + \. # Dot. + (?P\d+) # Minor = one or more digits. + (?: # Unnamed group for dot-micro. + \. # Dot. + (?P\d+) # Micro = one or more digit. + )? # Micro is optional because pypa/pipenv#1893. + .* # Trailing garbage. + $ # End of line. + ''', re.VERBOSE) + + match = version_pattern.match(output) + if not match: + return None + return match.groupdict(default='0') + + def python_version(path_to_python): if not path_to_python: return None - try: c = delegator.run([path_to_python, '--version'], block=False) except Exception: return None - - output = c.out.strip() or c.err.strip() - - @parse.with_pattern(r'.*') - def allow_empty(text): - return text - - TEMPLATE = 'Python {}.{}.{:d}{:AllowEmpty}' - parsed = parse.parse(TEMPLATE, output, dict(AllowEmpty=allow_empty)) - if parsed: - parsed = parsed.fixed - else: - return None - - return u"{v[0]}.{v[1]}.{v[2]}".format(v=parsed) + version = parse_python_version(c.out.strip() or c.err.strip()) + return u'{major}.{minor}.{micro}'.format(**version) def escape_grouped_arguments(s): diff --git a/tests/test_utils.py b/tests/test_utils.py index 26db37fb..4622e185 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -401,3 +401,23 @@ twine = "*" ) def test_prepare_pip_source_args(self, sources, expected_args): assert pipenv.utils.prepare_pip_source_args(sources, pip_args=None) == expected_args + + @pytest.mark.utils + def test_parse_python_version(self): + ver = pipenv.utils.parse_python_version('Python 3.6.5\n') + assert ver == {'major': '3', 'minor': '6', 'micro': '5'} + + @pytest.mark.utils + def test_parse_python_version_suffix(self): + ver = pipenv.utils.parse_python_version('Python 3.6.5rc1\n') + assert ver == {'major': '3', 'minor': '6', 'micro': '5'} + + @pytest.mark.utils + def test_parse_python_version_270(self): + ver = pipenv.utils.parse_python_version('Python 2.7\n') + assert ver == {'major': '2', 'minor': '7', 'micro': '0'} + + @pytest.mark.utils + def test_parse_python_version_270_garbage(self): + ver = pipenv.utils.parse_python_version('Python 2.7+\n') + assert ver == {'major': '2', 'minor': '7', 'micro': '0'}