From 6074ec623a4acff0049ac8d9928d11a0b74d0365 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 24 Jan 2017 15:18:21 -0500 Subject: [PATCH 01/11] preliminary support for hashes --- pipenv/cli.py | 94 +++++++++++++++++++++++++++++++++++++++++------ pipenv/project.py | 32 +++++++++++++++- 2 files changed, 113 insertions(+), 13 deletions(-) diff --git a/pipenv/cli.py b/pipenv/cli.py index 43aa179f..aa1b95a9 100644 --- a/pipenv/cli.py +++ b/pipenv/cli.py @@ -148,6 +148,46 @@ def do_install_dependencies(dev=False, only=False, bare=False, allow_global=Fals click.echo(crayons.blue(c.out)) +def do_download_dependencies(dev=False, only=False, bare=False): + """"Executes the install functionality.""" + + # Load the Pipfile. + p = pipfile.load(project.pipfile_location) + + # Load the lockfile if it exists, else use the Pipfile as a seed. + if not project.lockfile_exists: + if not bare: + click.echo(crayons.yellow('Installing dependencies from Pipfile...')) + lockfile = json.loads(p.lock()) + else: + if not bare: + click.echo(crayons.yellow('Installing dependencies from Pipfile.lock...')) + with open(project.lockfile_location, 'r') as f: + lockfile = json.load(f) + + # Install default dependencies, always. + deps = lockfile['default'] if not only else {} + + # Add development deps if --dev was passed. + if dev: + deps.update(lockfile['develop']) + + # Convert the deps to pip-compatible arguments. + deps = convert_deps_to_pip(deps) + + # Actually install each dependency into the virtualenv. + for package_name in deps: + + if not bare: + click.echo('Downloading {0}...'.format(crayons.green(package_name))) + + # pip install: + c = pip_download(package_name) + + if not bare: + click.echo(crayons.blue(c.out)) + + def do_create_virtualenv(three=None): """Creates a virtualenv.""" click.echo(crayons.yellow('Creating a virtualenv for this project...')) @@ -169,47 +209,68 @@ def do_create_virtualenv(three=None): do_where(virtualenv=True, bare=False) +def get_downloads_info(): + + info = [] + + for fname in os.listdir(project.download_location): + name = fname.split('-')[0] + + # Remove file extensions from name. + version = fname.split('-')[1] + version = version.replace('.tar.gz', '').replace('.zip', '').replace('.egg', '') + + # Get the hash of each file. + c = delegator.run('pip hash {}'.format(os.sep.join([project.download_location, fname]))) + hash = c.out.split('--hash=')[1].strip() + + info.append(dict(name=name, version=version, hash=hash)) + + return info + def do_lock(dev=False): """Executes the freeze functionality.""" click.echo(crayons.yellow('Assuring all dependencies from Pipfile are installed...')) # Purge the virtualenv, for development dependencies. - do_purge(bare=True) + do_purge(downloads=True) click.echo(crayons.yellow('Locking {0} dependencies...'.format(crayons.red('[dev-packages]')))) # Install only development dependencies. - do_install_dependencies(dev=True, only=True, bare=True) + do_download_dependencies(dev=True, only=True, bare=True) # Load the Pipfile and generate a lockfile. p = pipfile.load(project.pipfile_location) lockfile = json.loads(p.lock()) # Pip freeze development dependencies. - c = delegator.run('{0} freeze'.format(which_pip())) + # c = delegator.run('{0} freeze'.format(which_pip())) + results = get_downloads_info() # Add Development dependencies to lockfile. - for dep in c.out.split('\n'): + for dep in results: if dep: - lockfile['develop'].update(convert_deps_from_pip(dep)) + lockfile['develop'].update({dep['name']: {'hash': dep['hash'], 'version': '=={0}'.format(dep['version'])}}) # Purge the virtualenv. - do_purge(bare=True) + do_purge(downloads=True) click.echo(crayons.yellow('Locking {0} dependencies...'.format(crayons.red('[packages]')))) # Install only development dependencies. - do_install_dependencies(bare=True) + do_download_dependencies(bare=True) # Pip freeze default dependencies. - c = delegator.run('{0} freeze'.format(which_pip())) + # c = delegator.run('{0} freeze'.format(which_pip())) + results = get_downloads_info() # Add default dependencies to lockfile. - for dep in c.out.split('\n'): + for dep in results: if dep: - lockfile['default'].update(convert_deps_from_pip(dep)) + lockfile['default'].update({dep['name']: {'hash': dep['hash'], 'version': '=={0}'.format(dep['version'])}}) # Write out lockfile. with open(project.lockfile_location, 'w') as f: @@ -254,8 +315,13 @@ def do_activate_virtualenv(bare=False): click.echo(activate_virtualenv()) -def do_purge(bare=False, allow_global=False): +def do_purge(bare=False, downloads=False, allow_global=False): """Executes the purge functionality.""" + + if downloads: + shutil.rmtree(project.download_location) + return + freeze = delegator.run('{0} freeze'.format(which_pip(allow_global=allow_global))).out installed = freeze.split() @@ -311,6 +377,10 @@ def pip_install(package_name, allow_global=False): c = delegator.run('{0} install "{1}" -i {2}'.format(which_pip(allow_global=allow_global), package_name, project.source['url'])) return c +def pip_download(package_name): + c = delegator.run('{0} download "{1}" -d {2}'.format(which_pip(), package_name, project.download_location)) + return c + def which(command): return os.sep.join([project.virtualenv_location] + ['bin/{0}'.format(command)]) @@ -408,7 +478,7 @@ def install(package_name=False, more_packages=False, r=False, dev=False, three=F for package_name in package_names: # Lower-case incoming package name. - package_name = package_name.lower() + package_name = package_name click.echo('Installing {0}...'.format(crayons.green(package_name))) diff --git a/pipenv/project.py b/pipenv/project.py index 8204518e..9dd4f776 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -6,6 +6,26 @@ from . import _pipfile as pipfile from .utils import format_toml, multi_split from .utils import convert_deps_from_pip, convert_deps_to_pip +def mkdir_p(newdir): + """works the way a good mkdir should :) + - already exists, silently complete + - regular file in the way, raise an exception + - parent directory(ies) does not exist, make them as well + From: http://code.activestate.com/recipes/82465-a-friendly-mkdir/ + """ + if os.path.isdir(newdir): + pass + elif os.path.isfile(newdir): + raise OSError("a file with the same name as the desired " \ + "dir, '%s', already exists." % newdir) + else: + head, tail = os.path.split(newdir) + if head and not os.path.isdir(head): + _mkdir(head) + #print "_mkdir %s" % repr(newdir) + if tail: + os.mkdir(newdir) + class Project(object): """docstring for Project""" @@ -28,6 +48,16 @@ class Project(object): def virtualenv_location(self): return os.sep.join(self.pipfile_location.split(os.sep)[:-1] + ['.venv']) + @property + def download_location(self): + d_dir = os.sep.join(self.pipfile_location.split(os.sep)[:-1] + ['.venv', 'downloads']) + + # Create the directory, if it doesn't exist. + mkdir_p(d_dir) + + return d_dir + + @property def pipfile_location(self): try: @@ -80,7 +110,7 @@ class Project(object): def add_package_to_pipfile(self, package_name, dev=False): # Lower-case package name. - package_name = package_name.lower() + package_name = package_name # Find the Pipfile. pipfile_path = pipfile.Pipfile.find() From db6a3bbc30cc03b79af99dbc805206370636d668 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 24 Jan 2017 15:24:14 -0500 Subject: [PATCH 02/11] fix bug --- pipenv/cli.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/pipenv/cli.py b/pipenv/cli.py index aa1b95a9..afd11dcd 100644 --- a/pipenv/cli.py +++ b/pipenv/cli.py @@ -154,16 +154,10 @@ def do_download_dependencies(dev=False, only=False, bare=False): # Load the Pipfile. p = pipfile.load(project.pipfile_location) - # Load the lockfile if it exists, else use the Pipfile as a seed. - if not project.lockfile_exists: - if not bare: - click.echo(crayons.yellow('Installing dependencies from Pipfile...')) - lockfile = json.loads(p.lock()) - else: - if not bare: - click.echo(crayons.yellow('Installing dependencies from Pipfile.lock...')) - with open(project.lockfile_location, 'r') as f: - lockfile = json.load(f) + # Load the Pipfile. + if not bare: + click.echo(crayons.yellow('Installing dependencies from Pipfile...')) + lockfile = json.loads(p.lock()) # Install default dependencies, always. deps = lockfile['default'] if not only else {} From c6be7be245c1b5a96a1b7afbad5f5b5eeb43fab2 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 24 Jan 2017 15:42:29 -0500 Subject: [PATCH 03/11] more work on pip hashes --- pipenv/cli.py | 1 + pipenv/utils.py | 13 ++++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/pipenv/cli.py b/pipenv/cli.py index afd11dcd..34d646df 100644 --- a/pipenv/cli.py +++ b/pipenv/cli.py @@ -143,6 +143,7 @@ def do_install_dependencies(dev=False, only=False, bare=False, allow_global=Fals # pip install: c = pip_install(package_name, allow_global=allow_global) + print c.err if not bare: click.echo(crayons.blue(c.out)) diff --git a/pipenv/utils.py b/pipenv/utils.py index b5a9ca11..14216b42 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -64,17 +64,27 @@ def convert_deps_to_pip(deps): dependencies = [] for dep in deps.keys(): + # Default (e.g. '>1.10'). extra = deps[dep] + version = '' # Get rid of '*'. if deps[dep] == '*' or str(extra) == '{}': extra = '' + if 'hash' in deps[dep]: + # TODO: figure out why this doesn't work. + # extra = ' --hash={0}'.format(deps[dep]['hash']) + extra = '' + # Support for extras (e.g. requests[socks]) if 'extras' in deps[dep]: extra = '[{0}]'.format(deps[dep]['extras'][0]) + if 'version' in deps[dep]: + version = deps[dep]['version'] + # Support for git. # TODO: support SVN and others. if 'git' in deps[dep]: @@ -92,6 +102,7 @@ def convert_deps_to_pip(deps): dep = '-e ' else: dep = '' - dependencies.append('{0}{1}'.format(dep, extra)) + + dependencies.append('{0}{1}{2}'.format(dep, version, extra)) return dependencies From 0944ab01a39eb017fb6c32ded193716493e6cd05 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 24 Jan 2017 15:50:35 -0500 Subject: [PATCH 04/11] remove err printing --- pipenv/cli.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pipenv/cli.py b/pipenv/cli.py index 34d646df..afd11dcd 100644 --- a/pipenv/cli.py +++ b/pipenv/cli.py @@ -143,7 +143,6 @@ def do_install_dependencies(dev=False, only=False, bare=False, allow_global=Fals # pip install: c = pip_install(package_name, allow_global=allow_global) - print c.err if not bare: click.echo(crayons.blue(c.out)) From 6a8b1afee098840a3c79d55eeb7904944a381eef Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 24 Jan 2017 16:04:02 -0500 Subject: [PATCH 05/11] history notes --- HISTORY.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/HISTORY.txt b/HISTORY.txt index 78894125..11587d64 100644 --- a/HISTORY.txt +++ b/HISTORY.txt @@ -1,6 +1,8 @@ 0.3.0: - Lock fix. - Removed $ pipenv install -r functionality. + - Lock now uses downloads instead of + - Preliminary (non-enforced) hash functionality. 0.2.9: - Enhanced–enhanced PEP 508 checking capabilities! 0.2.8: From 8a8355162e873688bcdb2aca308bbbce5564c0aa Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 24 Jan 2017 16:04:32 -0500 Subject: [PATCH 06/11] no need for --dev for lock --- pipenv/cli.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/pipenv/cli.py b/pipenv/cli.py index c26bcd80..350a9b49 100644 --- a/pipenv/cli.py +++ b/pipenv/cli.py @@ -258,14 +258,6 @@ def do_lock(dev=False): with open(project.lockfile_location, 'w') as f: f.write(json.dumps(lockfile, indent=4, separators=(',', ': '))) - # Provide instructions for dev dependencies. - if not dev: - click.echo(crayons.yellow('Note: ') + 'your project now has only default {0} installed.'.format(crayons.red('[packages]'))) - click.echo('To keep {0} next time, run: $ {1}'.format(crayons.red('[dev-packages]'), crayons.green('pipenv lock --dev'))) - else: - # Install only development dependencies. - do_install_dependencies(dev=True, only=True, bare=True) - def activate_virtualenv(source=True): """Returns the string to activate a virtualenv.""" From f11f3c7dcb6ee07c1a07c43257f768b5c86aefe6 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 24 Jan 2017 16:04:45 -0500 Subject: [PATCH 07/11] hashes in lockfile --- Pipfile | 2 +- Pipfile.lock | 82 ++++++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 67 insertions(+), 17 deletions(-) diff --git a/Pipfile b/Pipfile index 0c411b8e..45b9b5f8 100644 --- a/Pipfile +++ b/Pipfile @@ -5,6 +5,6 @@ pytest = "*" crayons = "*" requirements-parser = "*" "delegator.py" = ">=0.0.6" -"backports.shutil-get-terminal-size" = "*" +"backports.shutil_get_terminal_size" = "*" toml = "*" click = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 47913f21..dd0ecbc2 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,22 +1,72 @@ { "default": { - "crayons": "==0.1.1", - "requirements-parser": "==0.1.0", - "pexpect": "==4.2.1", - "delegator.py": "==0.0.6", - "backports.shutil-get-terminal-size": "==1.0.0", - "ptyprocess": "==0.5.1", - "toml": "==0.9.2", - "colorama": "==0.3.7", - "click": "==6.7" + "crayons": { + "version": "==0.1.2", + "hash": "sha256:6f51241d0c4faec1c04c1c0ac6a68f1d66a4655476ce1570b3f37e5166a599cc" + }, + "requirements-parser": "*", + "pexpect": { + "version": "==4.2.1", + "hash": "sha256:f853b52afaf3b064d29854771e2db509ef80392509bde2dd7a6ecf2dfc3f0018" + }, + "delegator.py": { + "version": "==0.0.7", + "hash": "sha256:9908c3789bf50c4c1b55c9fb5dc3a83d1609d7f4a11450d436c6880a63656f12" + }, + "requirements": { + "version": "==parser", + "hash": "sha256:fee2380a469ffe4067bc7f0096a6fcfb27539da7496fae12b74b8d5d0f33a4ee" + }, + "backports.shutil_get_terminal_size": { + "version": "==1.0.0", + "hash": "sha256:0975ba55054c15e346944b38956a4c9cbee9009391e41b86c68990effb8c1f64" + }, + "ptyprocess": { + "version": "==0.5.1", + "hash": "sha256:464cb76f7a7122743dd25507650db89cd447c51f38e4671602b3eaa2e38e05ae" + }, + "toml": { + "version": "==0.9.2", + "hash": "sha256:b3953bffe848ad9a6d554114d82f2dcb3e23945e90b4d9addc9956f37f336594" + }, + "colorama": { + "version": "==0.3.7", + "hash": "sha256:a4c0f5bc358a62849653471e309dcc991223cf86abafbec17cd8f41327279e89" + }, + "click": { + "version": "==6.7", + "hash": "sha256:29f99fc6125fbc931b758dc053b3114e55c77a6e4c6c3a2674a2dc986016381d" + } }, "develop": { - "packaging": "==16.8", - "pytest": "==3.0.6", - "pyparsing": "==2.1.10", - "py": "==1.4.32", - "six": "==1.10.0", - "appdirs": "==1.4.0" + "packaging": { + "version": "==16.8", + "hash": "sha256:99276dc6e3a7851f32027a68f1095cd3f77c148091b092ea867a351811cfe388" + }, + "pytest": { + "version": "==3.0.6", + "hash": "sha256:da0ab50c7eec0683bc24f1c1137db1f4111752054ecdad63125e7ec71316b813" + }, + "setuptools": { + "version": "==34.0.2", + "hash": "sha256:f9ac44de8b36a13ce35d2989052a295a548c9fe5a161fb066dff53592200aa12" + }, + "pyparsing": { + "version": "==2.1.10", + "hash": "sha256:67101d7acee692962f33dd30b5dce079ff532dd9aa99ff48d52a3dad51d2fe84" + }, + "py": { + "version": "==1.4.32", + "hash": "sha256:2d4bba2e25fff58140e6bdce1e485e89bb59776adbe01d490baa6b1f37a3dd6b" + }, + "six": { + "version": "==1.10.0", + "hash": "sha256:0ff78c403d9bccf5a425a6d31a12aa6b47f1c21ca4dc2573a7e2f32a97335eb1" + }, + "appdirs": { + "version": "==1.4.0", + "hash": "sha256:85e58578db8f29538f3109c11250c2a5514a2fcdc9890d9b2fe777eb55517736" + } }, "_meta": { "sources": [ @@ -26,6 +76,6 @@ } ], "requires": {}, - "Pipfile-sha256": "9dce3ee2973403fc73a25f360f56c3b998cbba56bff36761cf58ba966181390f" + "Pipfile-sha256": "599210e2db7aaa8a4b39292cd2e5f58d7d608a83428e4a8c2b9eacde2211b4f1" } } \ No newline at end of file From 684df5b9b029b78d83d42fc6cd00d01bff1f2fa5 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 24 Jan 2017 16:07:40 -0500 Subject: [PATCH 08/11] code cleanup --- pipenv/cli.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pipenv/cli.py b/pipenv/cli.py index 350a9b49..66f54635 100644 --- a/pipenv/cli.py +++ b/pipenv/cli.py @@ -39,7 +39,7 @@ def ensure_latest_pip(): c = delegator.run('{0} install pip --upgrade'.format(which_pip()), block=False) click.echo(crayons.blue(c.out)) -def ensure_pipfile(dev=False): +def ensure_pipfile(): """Creates a Pipfile for the project, if it doesn't exist.""" # Assert Pipfile exists. @@ -68,9 +68,9 @@ def ensure_virtualenv(three=None): ensure_virtualenv(three=three) -def ensure_project(dev=False, three=None): +def ensure_project(three=None): """Ensures both Pipfile and virtualenv exist for the project.""" - ensure_pipfile(dev=dev) + ensure_pipfile() ensure_virtualenv(three=three) @@ -433,7 +433,7 @@ def cli(ctx, where=False, bare=False, three=False, help=False): def install(package_name=False, more_packages=False, r=False, dev=False, three=False, system=False): # Ensure that virtualenv is available. - ensure_project(dev=dev, three=three) + ensure_project(three=three) # Allow more than one package to be provided. package_names = (package_name,) + more_packages @@ -504,7 +504,7 @@ def uninstall(package_name=False, more_packages=False, three=None, system=False) @click.option('--three/--two', is_flag=True, default=None, help="Use Python 3/2 when creating virtualenv.") def lock(dev=False, three=None): # Ensure that virtualenv is available. - ensure_project(dev=dev, three=three) + ensure_project(three=three) do_lock(dev=dev) From 41ef0273312da92478c1df356720a076d91a88cb Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 24 Jan 2017 16:14:04 -0500 Subject: [PATCH 09/11] --lock --- pipenv/cli.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pipenv/cli.py b/pipenv/cli.py index 66f54635..6c1ad8a5 100644 --- a/pipenv/cli.py +++ b/pipenv/cli.py @@ -430,7 +430,8 @@ def cli(ctx, where=False, bare=False, three=False, help=False): @click.option('--dev','-d', is_flag=True, default=False, help="Install package(s) in [dev-packages].") @click.option('--three/--two', is_flag=True, default=None, help="Use Python 3/2 when creating virtualenv.") @click.option('--system', is_flag=True, default=False, help="System pip management.") -def install(package_name=False, more_packages=False, r=False, dev=False, three=False, system=False): +@click.option('--lock', is_flag=True, default=False, help="Lock afterwards.") +def install(package_name=False, more_packages=False, r=False, dev=False, three=False, system=False, lock=False): # Ensure that virtualenv is available. ensure_project(three=three) @@ -470,12 +471,16 @@ def install(package_name=False, more_packages=False, r=False, dev=False, three=F # Add the package to the Pipfile. project.add_package_to_pipfile(package_name, dev) + if lock: + do_lock() + @click.command(help="Un-installs a provided package and removes it from Pipfile, or (if none is given), un-installs all packages.") @click.argument('package_name', default=False) @click.argument('more_packages', nargs=-1) @click.option('--three/--two', is_flag=True, default=None, help="Use Python 3/2 when creating virtualenv.") @click.option('--system', is_flag=True, default=False, help="System pip management.") +@click.option('--lock', is_flag=True, default=False, help="Lock afterwards.") def uninstall(package_name=False, more_packages=False, three=None, system=False): # Ensure that virtualenv is available. ensure_project(three=three) @@ -498,6 +503,9 @@ def uninstall(package_name=False, more_packages=False, three=None, system=False) click.echo('Removing {0} from Pipfile...'.format(crayons.green(package_name))) project.remove_package_from_pipfile(package_name) + if lock: + do_lock() + @click.command(help="Generates Pipfile.lock.") @click.option('--dev','-d', is_flag=True, default=False, help="Keeps dev-packages installed.") From a30f8211d8771b44b30272a00606aa91157c63fc Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 24 Jan 2017 16:14:11 -0500 Subject: [PATCH 10/11] speed of locking improved --- HISTORY.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/HISTORY.txt b/HISTORY.txt index 11587d64..10d4e98b 100644 --- a/HISTORY.txt +++ b/HISTORY.txt @@ -1,7 +1,10 @@ 0.3.0: + - Speed of locking improved. + - Lock now uses downloads instead of installation functionality. - Lock fix. - Removed $ pipenv install -r functionality. - - Lock now uses downloads instead of + - Removal of $ pipenv lock --dev. + - Addition of $ pipenv install/uninstall --lock. - Preliminary (non-enforced) hash functionality. 0.2.9: - Enhanced–enhanced PEP 508 checking capabilities! From 098f5f53a86a140727da9087e75a6f6b1fa58300 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 24 Jan 2017 16:14:21 -0500 Subject: [PATCH 11/11] updated pipfile.lock --- Pipfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Pipfile.lock b/Pipfile.lock index dd0ecbc2..936f08fd 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -76,6 +76,6 @@ } ], "requires": {}, - "Pipfile-sha256": "599210e2db7aaa8a4b39292cd2e5f58d7d608a83428e4a8c2b9eacde2211b4f1" + "Pipfile-sha256": "1efce5cb2014fdc6b3dfed893e3e166b337c7f08521d1c53c96587148f049354" } } \ No newline at end of file