Merge branch 'master' into 1910-try_harder_to_cleanup_virtualenv

This commit is contained in:
Kyle Altendorf
2018-04-08 07:59:49 -07:00
committed by GitHub
9 changed files with 360 additions and 302 deletions
-4
View File
@@ -1,4 +0,0 @@
from .cli import get_cli_string
from .main import load_dotenv, get_key, set_key, unset_key, find_dotenv
__all__ = ['get_cli_string', 'load_dotenv', 'get_key', 'set_key', 'unset_key', 'find_dotenv']
-186
View File
@@ -1,186 +0,0 @@
# -*- coding: utf-8 -*-
from __future__ import absolute_import
import codecs
import os
import sys
import warnings
import re
from collections import OrderedDict
__escape_decoder = codecs.getdecoder('unicode_escape')
__posix_variable = re.compile('\$\{[^\}]*\}')
__variable_declaration = re.compile('^\s*(\w*)\s*=\s*("[^"]*"|\'[^\']*\'|[^\s]*)\s*$',
flags=re.MULTILINE)
def decode_escaped(escaped):
return __escape_decoder(escaped)[0]
def load_dotenv(dotenv_path, verbose=False, override=False):
"""
Read a .env file and load into os.environ.
"""
if not os.path.exists(dotenv_path):
if verbose:
warnings.warn("Not loading %s - it doesn't exist." % dotenv_path)
return None
for k, v in dotenv_values(dotenv_path).items():
if override:
os.environ[k] = v
else:
os.environ.setdefault(k, v)
return True
def get_key(dotenv_path, key_to_get):
"""
Gets the value of a given key from the given .env
If the .env path given doesn't exist, fails
"""
key_to_get = str(key_to_get)
if not os.path.exists(dotenv_path):
warnings.warn("can't read %s - it doesn't exist." % dotenv_path)
return None
dotenv_as_dict = dotenv_values(dotenv_path)
if key_to_get in dotenv_as_dict:
return dotenv_as_dict[key_to_get]
else:
warnings.warn("key %s not found in %s." % (key_to_get, dotenv_path))
return None
def set_key(dotenv_path, key_to_set, value_to_set, quote_mode="always"):
"""
Adds or Updates a key/value to the given .env
If the .env path given doesn't exist, fails instead of risking creating
an orphan .env somewhere in the filesystem
"""
key_to_set = str(key_to_set)
value_to_set = str(value_to_set).strip("'").strip('"')
if not os.path.exists(dotenv_path):
warnings.warn("can't write to %s - it doesn't exist." % dotenv_path)
return None, key_to_set, value_to_set
dotenv_as_dict = OrderedDict(parse_dotenv(dotenv_path))
dotenv_as_dict[key_to_set] = value_to_set
success = flatten_and_write(dotenv_path, dotenv_as_dict, quote_mode)
return success, key_to_set, value_to_set
def unset_key(dotenv_path, key_to_unset, quote_mode="always"):
"""
Removes a given key from the given .env
If the .env path given doesn't exist, fails
If the given key doesn't exist in the .env, fails
"""
key_to_unset = str(key_to_unset)
if not os.path.exists(dotenv_path):
warnings.warn("can't delete from %s - it doesn't exist." % dotenv_path)
return None, key_to_unset
dotenv_as_dict = dotenv_values(dotenv_path)
if key_to_unset in dotenv_as_dict:
dotenv_as_dict.pop(key_to_unset, None)
else:
warnings.warn("key %s not removed from %s - key doesn't exist." % (key_to_unset, dotenv_path))
return None, key_to_unset
success = flatten_and_write(dotenv_path, dotenv_as_dict, quote_mode)
return success, key_to_unset
def dotenv_values(dotenv_path):
values = OrderedDict(parse_dotenv(dotenv_path))
values = resolve_nested_variables(values)
return values
def parse_dotenv(dotenv_path):
with open(dotenv_path) as f:
for k, v in __variable_declaration.findall(f.read()):
if len(v) > 0:
quoted = v[0] == v[len(v) - 1] in ['"', "'"]
if quoted:
v = decode_escaped(v[1:-1])
yield k, v
def resolve_nested_variables(values):
def _replacement(name):
"""
get appropriate value for a variable name.
first search in environ, if not found,
then look into the dotenv variables
"""
ret = os.getenv(name, values.get(name, ""))
return ret
def _re_sub_callback(match_object):
"""
From a match object gets the variable name and returns
the correct replacement
"""
return _replacement(match_object.group()[2:-1])
for k, v in values.items():
values[k] = __posix_variable.sub(_re_sub_callback, v)
return values
def flatten_and_write(dotenv_path, dotenv_as_dict, quote_mode="always"):
with open(dotenv_path, "w") as f:
for k, v in dotenv_as_dict.items():
_mode = quote_mode
if _mode == "auto" and " " in v:
_mode = "always"
str_format = '%s="%s"\n' if _mode == "always" else '%s=%s\n'
f.write(str_format % (k, v))
return True
def _walk_to_root(path):
"""
Yield directories starting from the given directory up to the root
"""
if not os.path.exists(path):
raise IOError('Starting path not found')
if os.path.isfile(path):
path = os.path.dirname(path)
last_dir = None
current_dir = os.path.abspath(path)
while last_dir != current_dir:
yield current_dir
parent_dir = os.path.abspath(os.path.join(current_dir, os.path.pardir))
last_dir, current_dir = current_dir, parent_dir
def find_dotenv(filename='.env', raise_error_if_not_found=False, usecwd=False):
"""
Search in increasingly higher folders for the given file
Returns path to the file if found, or an empty string otherwise
"""
if usecwd or '__file__' not in globals():
# should work without __file__, e.g. in REPL or IPython notebook
path = os.getcwd()
else:
# will work for .py files
frame_filename = sys._getframe().f_back.f_code.co_filename
path = os.path.dirname(os.path.abspath(frame_filename))
for dirname in _walk_to_root(path):
check_path = os.path.join(dirname, filename)
if os.path.exists(check_path):
return check_path
if raise_error_if_not_found:
raise IOError('File not found')
return ''
-85
View File
@@ -1,85 +0,0 @@
import os
from textwrap import dedent
import unittest
from main import parse_dotenv
class TestParseDotenv(unittest.TestCase):
filename = 'testfile.conf'
def tearDown(self):
if os.path.exists(self.filename):
os.remove(self.filename)
def write_file(self, contents):
with open(self.filename, 'w') as f:
f.write(contents)
def assert_parsed(self, *expected):
parsed = parse_dotenv(self.filename)
for expected_key, expected_val in expected:
actual_key, actual_val = next(parsed)
self.assertEqual(actual_key, expected_key)
self.assertEqual(actual_val, expected_val)
with self.assertRaises(StopIteration):
next(parsed)
def test_value_unquoted(self):
self.write_file(dedent("""
var1 = value1
var2=value2
"""))
self.assert_parsed(('var1', 'value1'), ('var2', 'value2'))
def test_value_double_quoted(self):
self.write_file(dedent("""
var1 = "value1"
var2="value2"
"""))
self.assert_parsed(('var1', 'value1'), ('var2', 'value2'))
def test_value_single_quoted(self):
self.write_file(dedent("""
var1 = 'value1'
var2='value2'
"""))
self.assert_parsed(('var1', 'value1'), ('var2', 'value2'))
def test_value_with_space_double_quoted(self):
self.write_file(dedent("""
var1 = "value1 with spaces"
var2 = "othervalue"
"""))
self.assert_parsed(('var1', 'value1 with spaces'),
('var2', 'othervalue'))
def test_value_with_space_single_quoted(self):
self.write_file(dedent("""
var1 = 'value with spaces'
var2 = 'othervalue'
"""))
self.assert_parsed(('var1', 'value with spaces'),
('var2', 'othervalue'))
def test_values_with_mixed_quotes_and_spaces(self):
self.write_file(dedent("""
var1 = 'value with spaces'
var2= othervalue
var3="double-quoted value with spaces"
var4 = "double-quoted value
with
newlines"
var5='single quote
and newline'
"""))
self.assert_parsed(('var1', 'value with spaces'),
('var2', 'othervalue'),
('var3', 'double-quoted value with spaces'),
('var4', 'double-quoted value\nwith\nnewlines'),
('var5', 'single quote\nand newline'))
if __name__ == '__main__':
unittest.main()
+9
View File
@@ -0,0 +1,9 @@
# Vendored packages
These packages are copied as-is from upstream to reduce Pipenv dependencies.
They should always be kept synced with upstream. DO NOT MODIFY DIRECTLY! If
you need to patch anything, move the package to `patched`.
Known vendored versions:
- python-dotenv: 0.8.2
+40
View File
@@ -0,0 +1,40 @@
from .main import load_dotenv, get_key, set_key, unset_key, find_dotenv, dotenv_values
def load_ipython_extension(ipython):
from .ipython import load_ipython_extension
load_ipython_extension(ipython)
def get_cli_string(path=None, action=None, key=None, value=None, quote=None):
"""Returns a string suitable for running as a shell script.
Useful for converting a arguments passed to a fabric task
to be passed to a `local` or `run` command.
"""
command = ['dotenv']
if quote:
command.append('-q %s' % quote)
if path:
command.append('-f %s' % path)
if action:
command.append(action)
if key:
command.append(key)
if value:
if ' ' in value:
command.append('"%s"' % value)
else:
command.append(value)
return ' '.join(command).strip()
__all__ = ['get_cli_string',
'load_dotenv',
'dotenv_values',
'get_key',
'set_key',
'unset_key',
'find_dotenv',
'load_ipython_extension']
+11 -27
View File
@@ -1,8 +1,14 @@
import os
import sys
import click
try:
import click
except ImportError:
sys.stderr.write('It seems python-dotenv is not installed with cli option. \n'
'Run pip install "python-dotenv[cli]" to fix this.')
sys.exit(1)
from .main import get_key, dotenv_values, set_key, unset_key
from .main import dotenv_values, get_key, set_key, unset_key
@click.group()
@@ -27,7 +33,7 @@ def list(ctx):
file = ctx.obj['FILE']
dotenv_as_dict = dotenv_values(file)
for k, v in dotenv_as_dict.items():
click.echo('%s="%s"' % (k, v))
click.echo('%s=%s' % (k, v))
@cli.command()
@@ -40,7 +46,7 @@ def set(ctx, key, value):
quote = ctx.obj['QUOTE']
success, key, value = set_key(file, key, value, quote)
if success:
click.echo('%s="%s"' % (key, value))
click.echo('%s=%s' % (key, value))
else:
exit(1)
@@ -53,7 +59,7 @@ def get(ctx, key):
file = ctx.obj['FILE']
stored_value = get_key(file, key)
if stored_value:
click.echo('%s="%s"' % (key, stored_value))
click.echo('%s=%s' % (key, stored_value))
else:
exit(1)
@@ -72,27 +78,5 @@ def unset(ctx, key):
exit(1)
def get_cli_string(path=None, action=None, key=None, value=None):
"""Returns a string suitable for running as a shell script.
Useful for converting a arguments passed to a fabric task
to be passed to a `local` or `run` command.
"""
command = ['dotenv']
if path:
command.append('-f %s' % path)
if action:
command.append(action)
if key:
command.append(key)
if value:
if ' ' in value:
command.append('"%s"' % value)
else:
command.append(value)
return ' '.join(command).strip()
if __name__ == "__main__":
cli()
+4
View File
@@ -0,0 +1,4 @@
try:
from StringIO import StringIO # noqa
except ImportError:
from io import StringIO # noqa
+41
View File
@@ -0,0 +1,41 @@
from __future__ import print_function
from IPython.core.magic import Magics, line_magic, magics_class
from IPython.core.magic_arguments import (argument, magic_arguments,
parse_argstring)
from .main import find_dotenv, load_dotenv
@magics_class
class IPythonDotEnv(Magics):
@magic_arguments()
@argument(
'-o', '--override', action='store_true',
help="Indicate to override existing variables"
)
@argument(
'-v', '--verbose', action='store_true',
help="Indicate function calls to be verbose"
)
@argument('dotenv_path', nargs='?', type=str, default='.env',
help='Search in increasingly higher folders for the `dotenv_path`')
@line_magic
def dotenv(self, line):
args = parse_argstring(self.dotenv, line)
# Locate the .env file
dotenv_path = args.dotenv_path
try:
dotenv_path = find_dotenv(dotenv_path, True, True)
except IOError:
print("cannot find .env file")
return
# Load the .env file
load_dotenv(dotenv_path, verbose=args.verbose, override=args.override)
def load_ipython_extension(ipython):
"""Register the %dotenv magic."""
ipython.register_magics(IPythonDotEnv)
+255
View File
@@ -0,0 +1,255 @@
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import codecs
import fileinput
import io
import os
import re
import sys
import warnings
from collections import OrderedDict
from .compat import StringIO
__escape_decoder = codecs.getdecoder('unicode_escape')
__posix_variable = re.compile('\$\{[^\}]*\}')
def decode_escaped(escaped):
return __escape_decoder(escaped)[0]
def parse_line(line):
line = line.strip()
# Ignore lines with `#` or which doesn't have `=` in it.
if not line or line.startswith('#') or '=' not in line:
return None, None
k, v = line.split('=', 1)
if k.startswith('export '):
k = k.lstrip('export ')
# Remove any leading and trailing spaces in key, value
k, v = k.strip(), v.strip()
if v:
v = v.encode('unicode-escape').decode('ascii')
quoted = v[0] == v[-1] in ['"', "'"]
if quoted:
v = decode_escaped(v[1:-1])
return k, v
class DotEnv():
def __init__(self, dotenv_path, verbose=False):
self.dotenv_path = dotenv_path
self._dict = None
self.verbose = verbose
def _get_stream(self):
self._is_file = False
if isinstance(self.dotenv_path, StringIO):
return self.dotenv_path
if os.path.exists(self.dotenv_path):
self._is_file = True
return io.open(self.dotenv_path)
if self.verbose:
warnings.warn("File doesn't exist {}".format(self.dotenv_path))
return StringIO('')
def dict(self):
"""Return dotenv as dict"""
if self._dict:
return self._dict
values = OrderedDict(self.parse())
self._dict = resolve_nested_variables(values)
return self._dict
def parse(self):
f = self._get_stream()
for line in f:
key, value = parse_line(line)
if not key:
continue
yield key, value
if self._is_file:
f.close()
def set_as_environment_variables(self, override=False):
"""
Load the current dotenv as system environemt variable.
"""
for k, v in self.dict().items():
if k in os.environ and not override:
continue
os.environ[k] = v
return True
def get(self, key):
"""
"""
data = self.dict()
if key in data:
return data[key]
if self.verbose:
warnings.warn("key %s not found in %s." % (key, self.dotenv_path))
def get_key(dotenv_path, key_to_get):
"""
Gets the value of a given key from the given .env
If the .env path given doesn't exist, fails
"""
return DotEnv(dotenv_path, verbose=True).get(key_to_get)
def set_key(dotenv_path, key_to_set, value_to_set, quote_mode="always"):
"""
Adds or Updates a key/value to the given .env
If the .env path given doesn't exist, fails instead of risking creating
an orphan .env somewhere in the filesystem
"""
value_to_set = value_to_set.strip("'").strip('"')
if not os.path.exists(dotenv_path):
warnings.warn("can't write to %s - it doesn't exist." % dotenv_path)
return None, key_to_set, value_to_set
if " " in value_to_set:
quote_mode = "always"
line_template = '{}="{}"' if quote_mode == "always" else '{}={}'
line_out = line_template.format(key_to_set, value_to_set)
replaced = False
for line in fileinput.input(dotenv_path, inplace=True):
k, v = parse_line(line)
if k == key_to_set:
replaced = True
line = line_out
print(line, end='')
if not replaced:
with io.open(dotenv_path, "a") as f:
f.write("{}\n".format(line_out))
return True, key_to_set, value_to_set
def unset_key(dotenv_path, key_to_unset, quote_mode="always"):
"""
Removes a given key from the given .env
If the .env path given doesn't exist, fails
If the given key doesn't exist in the .env, fails
"""
removed = False
if not os.path.exists(dotenv_path):
warnings.warn("can't delete from %s - it doesn't exist." % dotenv_path)
return None, key_to_unset
for line in fileinput.input(dotenv_path, inplace=True):
k, v = parse_line(line)
if k == key_to_unset:
removed = True
line = ''
print(line, end='')
if not removed:
warnings.warn("key %s not removed from %s - key doesn't exist." % (key_to_unset, dotenv_path))
return None, key_to_unset
return removed, key_to_unset
def resolve_nested_variables(values):
def _replacement(name):
"""
get appropriate value for a variable name.
first search in environ, if not found,
then look into the dotenv variables
"""
ret = os.getenv(name, values.get(name, ""))
return ret
def _re_sub_callback(match_object):
"""
From a match object gets the variable name and returns
the correct replacement
"""
return _replacement(match_object.group()[2:-1])
for k, v in values.items():
values[k] = __posix_variable.sub(_re_sub_callback, v)
return values
def _walk_to_root(path):
"""
Yield directories starting from the given directory up to the root
"""
if not os.path.exists(path):
raise IOError('Starting path not found')
if os.path.isfile(path):
path = os.path.dirname(path)
last_dir = None
current_dir = os.path.abspath(path)
while last_dir != current_dir:
yield current_dir
parent_dir = os.path.abspath(os.path.join(current_dir, os.path.pardir))
last_dir, current_dir = current_dir, parent_dir
def find_dotenv(filename='.env', raise_error_if_not_found=False, usecwd=False):
"""
Search in increasingly higher folders for the given file
Returns path to the file if found, or an empty string otherwise
"""
if usecwd or '__file__' not in globals():
# should work without __file__, e.g. in REPL or IPython notebook
path = os.getcwd()
else:
# will work for .py files
frame_filename = sys._getframe().f_back.f_code.co_filename
path = os.path.dirname(os.path.abspath(frame_filename))
for dirname in _walk_to_root(path):
check_path = os.path.join(dirname, filename)
if os.path.exists(check_path):
return check_path
if raise_error_if_not_found:
raise IOError('File not found')
return ''
def load_dotenv(dotenv_path=None, stream=None, verbose=False, override=False):
f = dotenv_path or stream or find_dotenv()
return DotEnv(f, verbose=verbose).set_as_environment_variables(override=override)
def dotenv_values(dotenv_path=None, stream=None, verbose=False):
f = dotenv_path or stream or find_dotenv()
return DotEnv(f, verbose=verbose).dict()