From 6d2e208aaeed24c71958e93d4e94ff8381c4b5b0 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <kyle.altendorf@epcpower.com>
Date: Thu, 26 Apr 2018 15:50:36 -0400
Subject: [PATCH 01/26] Default to \n, retain consistent \r\n

When writing the Pipfile and Pipfile.lock make an effort to retain
their existing newlines if consistent.  Default to \n (LF) for new
files and files with inconsistent line endings.
---
 pipenv/core.py                    |  8 +---
 pipenv/project.py                 | 76 +++++++++++++++++++++++--------
 tests/integration/conftest.py     |  6 ++-
 tests/integration/test_project.py | 34 ++++++++++++++
 4 files changed, 96 insertions(+), 28 deletions(-)

diff --git a/pipenv/core.py b/pipenv/core.py
index 8965668d..087b0d46 100644
--- a/pipenv/core.py
+++ b/pipenv/core.py
@@ -1166,13 +1166,7 @@ def do_lock(
                 default_package
             ]
     if write:
-        # Write out the lockfile.
-        with open(project.lockfile_location, 'w') as f:
-            simplejson.dump(
-                lockfile, f, indent=4, separators=(',', ': '), sort_keys=True
-            )
-            # Write newline at end of document. GH Issue #319.
-            f.write('\n')
+        project.write_lockfile(lockfile)
         click.echo(
             '{0}'.format(
                 crayons.normal(
diff --git a/pipenv/project.py b/pipenv/project.py
index a914c990..89d79db7 100644
--- a/pipenv/project.py
+++ b/pipenv/project.py
@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 import codecs
+import io
 import json
 import os
 import re
@@ -11,6 +12,7 @@ from first import first
 import pipfile
 import pipfile.api
 import toml
+import json as simplejson
 
 try:
     import pathlib
@@ -49,6 +51,16 @@ def _normalized(p):
     return normalize_drive(str(pathlib.Path(p).resolve()))
 
 
+DEFAULT_NEWLINES = u'\n'
+
+
+def preferred_newlines(f):
+    if isinstance(f.newlines, type(u'')):
+        return f.newlines
+
+    return DEFAULT_NEWLINES
+
+
 if PIPENV_PIPFILE:
     if not os.path.isfile(PIPENV_PIPFILE):
         raise RuntimeError('Given PIPENV_PIPFILE is not found!')
@@ -90,6 +102,8 @@ class Project(object):
         self._download_location = None
         self._proper_names_location = None
         self._pipfile_location = None
+        self._pipfile_newlines = DEFAULT_NEWLINES
+        self._lockfile_newlines = DEFAULT_NEWLINES
         self._requirements_location = None
         self._original_dir = os.path.abspath(os.curdir)
         self.which = which
@@ -369,9 +383,7 @@ class Project(object):
         """Parse Pipfile into a TOMLFile and cache it
 
         (call clear_pipfile_cache() afterwards if mutating)"""
-        # Open the pipfile, read it into memory.
-        with open(self.pipfile_location) as f:
-            contents = f.read()
+        contents = self.read_pipfile()
         # use full contents to get around str/bytes 2/3 issues
         cache_key = (self.pipfile_location, contents)
         if cache_key not in _pipfile_cache:
@@ -379,10 +391,17 @@ class Project(object):
             _pipfile_cache[cache_key] = parsed
         return _pipfile_cache[cache_key]
 
+    def read_pipfile(self):
+        # Open the pipfile, read it into memory.
+        with io.open(self.pipfile_location) as f:
+            contents = f.read()
+            self._pipfile_newlines = preferred_newlines(f)
+
+        return contents
+
     @property
     def pased_pure_pipfile(self):
-        with open(self.pipfile_location) as f:
-            contents = f.read()
+        contents = self.read_pipfile()
 
         return self._parse_pipfile(contents)
 
@@ -474,14 +493,7 @@ class Project(object):
 
     @property
     def lockfile_content(self):
-        with open(self.lockfile_location) as lock:
-            j = json.load(lock)
-
-        # Expand environment variables in Pipfile.lock at runtime.
-        for i, source in enumerate(j['_meta']['sources'][:]):
-            j['_meta']['sources'][i]['url'] = os.path.expandvars(j['_meta']['sources'][i]['url'])
-
-        return j
+        return self.load_lockfile()
 
     @property
     def editable_packages(self):
@@ -546,9 +558,8 @@ class Project(object):
         if not self.pipfile_exists:
             return True
 
-        with open(self.pipfile_location, 'r') as f:
-            if not f.read():
-                return True
+        if not len(self.read_pipfile()):
+            return True
 
         return False
 
@@ -609,12 +620,26 @@ class Project(object):
                         )
                         data[section][package].update(_data)
             formatted_data = toml.dumps(data).rstrip()
+        if os.path.abspath(path) == os.path.abspath(self.pipfile_location):
+            newlines = self._pipfile_newlines
+        else:
+            newlines = preferred_newlines()
         formatted_data = cleanup_toml(formatted_data)
-        with open(path, 'w') as f:
+        with io.open(path, 'w', newline=newlines) as f:
             f.write(formatted_data)
         # pipfile is mutated!
         self.clear_pipfile_cache()
 
+    def write_lockfile(self, content):
+        # Write out the lockfile.
+        newlines = self._lockfile_newlines
+        with open(self.lockfile_location, 'w', newline=newlines) as f:
+            simplejson.dump(
+                content, f, indent=4, separators=(',', ': '), sort_keys=True
+            )
+            # Write newline at end of document. GH Issue #319.
+            f.write('\n')
+
     @property
     def pipfile_sources(self):
         if 'source' in self.parsed_pipfile:
@@ -742,12 +767,23 @@ class Project(object):
         if self.ensure_proper_casing():
             self.write_toml(self.parsed_pipfile)
 
+    def load_lockfile(self, expand_env_vars=True):
+        with io.open(self.lockfile_location) as lock:
+            j = json.load(lock)
+            self._lockfile_newlines = preferred_newlines(lock)
+
+        if expand_env_vars:
+            # Expand environment variables in Pipfile.lock at runtime.
+            for i, source in enumerate(j['_meta']['sources'][:]):
+                j['_meta']['sources'][i]['url'] = os.path.expandvars(j['_meta']['sources'][i]['url'])
+
+        return j
+
     def get_lockfile_hash(self):
         if not os.path.exists(self.lockfile_location):
             return
-        # Open the lockfile.
-        with codecs.open(self.lockfile_location, 'r') as f:
-            lockfile = json.load(f)
+
+        lockfile = self.load_lockfile(expand_env_vars=False)
         return lockfile['_meta'].get('hash', {}).get('sha256')
 
     def calculate_pipfile_hash(self):
diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py
index 955e20be..40eaccea 100644
--- a/tests/integration/conftest.py
+++ b/tests/integration/conftest.py
@@ -113,10 +113,14 @@ class _PipenvInstance(object):
 
     @property
     def lockfile(self):
-        p_path = os.sep.join([self.path, 'Pipfile.lock'])
+        p_path = self.lockfile_path
         with open(p_path, 'r') as f:
             return json.loads(f.read())
 
+    @property
+    def lockfile_path(self):
+        return os.sep.join([self.path, 'Pipfile.lock'])
+
 
 @pytest.fixture()
 def PipenvInstance():
diff --git a/tests/integration/test_project.py b/tests/integration/test_project.py
index 62781c11..c9c95e7c 100644
--- a/tests/integration/test_project.py
+++ b/tests/integration/test_project.py
@@ -1,4 +1,5 @@
 # -*- coding=utf-8 -*-
+import io
 import pytest
 import os
 from pipenv.project import Project
@@ -73,3 +74,36 @@ six = {{version = "*", index = "pypi"}}
             assert sorted(source.items()) == sorted(project.get_source(url=url).items())
             assert sorted(source.items()) == sorted(project.find_source(name).items())
             assert sorted(source.items()) == sorted(project.find_source(url).items())
+
+
+@pytest.mark.install
+@pytest.mark.project
+@pytest.mark.parametrize('newlines', [u'\n', u'\r\n'])
+@pytest.mark.parametrize('target', ['pipfile_path', 'lockfile_path'])
+def test_maintain_file_line_endings(PipenvInstance, pypi, newlines, target):
+    with PipenvInstance(pypi=pypi, chdir=True) as p:
+        path = getattr(p, target)
+
+        c = p.pipenv('install')
+        assert c.return_code == 0
+
+        with io.open(path) as f:
+            contents = f.read()
+
+        assert f.newlines == u'\n'
+
+        with io.open(path, 'w', newline=newlines) as f:
+            f.write(contents)
+
+        before = os.path.getmtime(path)
+
+        c = p.pipenv('install chardet')
+        assert c.return_code == 0
+
+        assert os.path.getmtime(path) != before
+
+        with io.open(path) as f:
+            f.read()
+            actual_newlines = f.newlines
+
+        assert actual_newlines == newlines

From dd11e2c196bebf5d47729a6bfeafe3072974d954 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <kyle.altendorf@epcpower.com>
Date: Thu, 26 Apr 2018 16:35:15 -0400
Subject: [PATCH 02/26] Fix writing of json/lockfile for py2

---
 pipenv/project.py | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/pipenv/project.py b/pipenv/project.py
index 89d79db7..1e6bfb95 100644
--- a/pipenv/project.py
+++ b/pipenv/project.py
@@ -623,7 +623,7 @@ class Project(object):
         if os.path.abspath(path) == os.path.abspath(self.pipfile_location):
             newlines = self._pipfile_newlines
         else:
-            newlines = preferred_newlines()
+            newlines = DEFAULT_NEWLINES
         formatted_data = cleanup_toml(formatted_data)
         with io.open(path, 'w', newline=newlines) as f:
             f.write(formatted_data)
@@ -633,12 +633,17 @@ class Project(object):
     def write_lockfile(self, content):
         # Write out the lockfile.
         newlines = self._lockfile_newlines
-        with open(self.lockfile_location, 'w', newline=newlines) as f:
-            simplejson.dump(
-                content, f, indent=4, separators=(',', ': '), sort_keys=True
-            )
+        s = simplejson.dumps(
+            content, indent=4, separators=(',', ': '), sort_keys=True
+        )
+
+        if sys.version_info[0] < 3:
+            s = s.decode('ascii')
+
+        with io.open(self.lockfile_location, 'w', newline=newlines) as f:
+            f.write(s)
             # Write newline at end of document. GH Issue #319.
-            f.write('\n')
+            f.write(u'\n')
 
     @property
     def pipfile_sources(self):

From 9b9721d6e5b83dd25ccec9d55e7761ab97b51482 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <kyle.altendorf@epcpower.com>
Date: Thu, 26 Apr 2018 18:24:16 -0400
Subject: [PATCH 03/26] Use six.string_types in preferred_newlines()

---
 pipenv/project.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/pipenv/project.py b/pipenv/project.py
index 1e6bfb95..e0b936de 100644
--- a/pipenv/project.py
+++ b/pipenv/project.py
@@ -11,6 +11,7 @@ import contoml
 from first import first
 import pipfile
 import pipfile.api
+import six
 import toml
 import json as simplejson
 
@@ -55,7 +56,7 @@ DEFAULT_NEWLINES = u'\n'
 
 
 def preferred_newlines(f):
-    if isinstance(f.newlines, type(u'')):
+    if isinstance(f.newlines, six.string_types):
         return f.newlines
 
     return DEFAULT_NEWLINES

From 749e1430688d6361a6fd19aa19bfeb74a6eb1123 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Fri, 27 Apr 2018 07:16:28 -0400
Subject: [PATCH 04/26] Change to six.text_type for newline detection

---
 pipenv/project.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pipenv/project.py b/pipenv/project.py
index e0b936de..7d9cf10b 100644
--- a/pipenv/project.py
+++ b/pipenv/project.py
@@ -56,7 +56,7 @@ DEFAULT_NEWLINES = u'\n'
 
 
 def preferred_newlines(f):
-    if isinstance(f.newlines, six.string_types):
+    if isinstance(f.newlines, six.text_type):
         return f.newlines
 
     return DEFAULT_NEWLINES

From 18704a5e21720b6fbc7dc62e6f345dc284020dc5 Mon Sep 17 00:00:00 2001
From: Dan Ryan <dan@danryan.co>
Date: Wed, 2 May 2018 20:49:52 -0400
Subject: [PATCH 05/26] Update newline tests

---
 tests/integration/test_project.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/tests/integration/test_project.py b/tests/integration/test_project.py
index c9c95e7c..eb8f195a 100644
--- a/tests/integration/test_project.py
+++ b/tests/integration/test_project.py
@@ -83,25 +83,25 @@ six = {{version = "*", index = "pypi"}}
 def test_maintain_file_line_endings(PipenvInstance, pypi, newlines, target):
     with PipenvInstance(pypi=pypi, chdir=True) as p:
         path = getattr(p, target)
-
         c = p.pipenv('install')
         assert c.return_code == 0
 
         with io.open(path) as f:
             contents = f.read()
+            written_newlines = f.newlines
 
         assert f.newlines == u'\n'
-
-        with io.open(path, 'w', newline=newlines) as f:
-            f.write(contents)
-
-        before = os.path.getmtime(path)
+        if target == 'lockfile_path':
+            project = Project()
+            project._lockfile_newlines = newlines
+            project.write_lockfile(contents)
+        else:
+            with io.open(path, 'w', newline=newlines) as f:
+                f.write(contents)
 
         c = p.pipenv('install chardet')
         assert c.return_code == 0
 
-        assert os.path.getmtime(path) != before
-
         with io.open(path) as f:
             f.read()
             actual_newlines = f.newlines

From 76827408ad187c42ff1876b0b6b5e3a8acedcc23 Mon Sep 17 00:00:00 2001
From: Dan Ryan <dan@danryan.co>
Date: Wed, 2 May 2018 21:36:46 -0400
Subject: [PATCH 06/26] Fix edge case of existing but empty lockfiles

Signed-off-by: Dan Ryan <dan@danryan.co>
---
 pipenv/project.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/pipenv/project.py b/pipenv/project.py
index 2ea46735..73356e61 100644
--- a/pipenv/project.py
+++ b/pipenv/project.py
@@ -664,7 +664,7 @@ class Project(object):
 
     @property
     def sources(self):
-        if self.lockfile_exists:
+        if self.lockfile_exists and hasattr(self.lockfile_content, 'keys'):
             meta_ = self.lockfile_content['_meta']
             sources_ = meta_.get('sources')
             if sources_:
@@ -771,6 +771,9 @@ class Project(object):
         with io.open(self.lockfile_location) as lock:
             j = json.load(lock)
             self._lockfile_newlines = preferred_newlines(lock)
+        # lockfile is just a string
+        if not j or not hasattr(j, 'keys'):
+            return j
 
         if expand_env_vars:
             # Expand environment variables in Pipfile.lock at runtime.
@@ -784,7 +787,10 @@ class Project(object):
             return
 
         lockfile = self.load_lockfile(expand_env_vars=False)
-        return lockfile['_meta'].get('hash', {}).get('sha256')
+        if '_meta' in lockfile and hasattr(lockfile, 'keys'):
+            return lockfile['_meta'].get('hash', {}).get('sha256')
+        # Lockfile exists but has no hash at all
+        return ''
 
     def calculate_pipfile_hash(self):
         # Update the lockfile if it is out-of-date.

From 415566f5d6d1e47be4ab327d977129e3bca3f771 Mon Sep 17 00:00:00 2001
From: Dan Ryan <dan@danryan.co>
Date: Wed, 2 May 2018 22:25:22 -0400
Subject: [PATCH 07/26] Update newline check

Signed-off-by: Dan Ryan <dan@danryan.co>
---
 tests/integration/test_project.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/integration/test_project.py b/tests/integration/test_project.py
index eb8f195a..464a40a9 100644
--- a/tests/integration/test_project.py
+++ b/tests/integration/test_project.py
@@ -83,14 +83,14 @@ six = {{version = "*", index = "pypi"}}
 def test_maintain_file_line_endings(PipenvInstance, pypi, newlines, target):
     with PipenvInstance(pypi=pypi, chdir=True) as p:
         path = getattr(p, target)
-        c = p.pipenv('install')
+        c = p.pipenv('install six')
         assert c.return_code == 0
 
         with io.open(path) as f:
             contents = f.read()
             written_newlines = f.newlines
 
-        assert f.newlines == u'\n'
+        assert written_newlines == u'\n'
         if target == 'lockfile_path':
             project = Project()
             project._lockfile_newlines = newlines

From 5718ae8f429ee61e92c43003cc12bd1f7062953e Mon Sep 17 00:00:00 2001
From: Dan Ryan <dan@danryan.co>
Date: Thu, 3 May 2018 03:27:45 -0400
Subject: [PATCH 08/26] Fix newline tests for windows

- Adjust for proper parameterization

Signed-off-by: Dan Ryan <dan@danryan.co>
---
 tests/integration/test_project.py | 37 ++++++++++++++-----------------
 1 file changed, 17 insertions(+), 20 deletions(-)

diff --git a/tests/integration/test_project.py b/tests/integration/test_project.py
index 464a40a9..9ffa6d8d 100644
--- a/tests/integration/test_project.py
+++ b/tests/integration/test_project.py
@@ -79,31 +79,28 @@ six = {{version = "*", index = "pypi"}}
 @pytest.mark.install
 @pytest.mark.project
 @pytest.mark.parametrize('newlines', [u'\n', u'\r\n'])
-@pytest.mark.parametrize('target', ['pipfile_path', 'lockfile_path'])
-def test_maintain_file_line_endings(PipenvInstance, pypi, newlines, target):
+def test_maintain_file_line_endings(PipenvInstance, pypi, newlines):
     with PipenvInstance(pypi=pypi, chdir=True) as p:
-        path = getattr(p, target)
-        c = p.pipenv('install six')
+        # Initial pipfile + lockfile generation
+        c = p.pipenv('install')
         assert c.return_code == 0
 
-        with io.open(path) as f:
-            contents = f.read()
-            written_newlines = f.newlines
-
-        assert written_newlines == u'\n'
-        if target == 'lockfile_path':
-            project = Project()
-            project._lockfile_newlines = newlines
-            project.write_lockfile(contents)
-        else:
-            with io.open(path, 'w', newline=newlines) as f:
+        # Rewrite each file with parameterized newlines
+        for fn in [p.pipfile_path, p.lockfile_path]:
+            with io.open(fn) as f:
+                contents = f.read()
+                written_newlines = f.newlines
+            assert written_newlines == u'\n'
+            with io.open(fn, 'w', newline=newlines) as f:
                 f.write(contents)
 
+        # Run pipenv install to programatically rewrite
         c = p.pipenv('install chardet')
         assert c.return_code == 0
 
-        with io.open(path) as f:
-            f.read()
-            actual_newlines = f.newlines
-
-        assert actual_newlines == newlines
+        # Make sure we kept the right newlines
+        for fn in [p.pipfile_path, p.lockfile_path]:
+            with io.open(fn) as f:
+                contents = f.read()
+                actual_newlines = f.newlines
+            assert actual_newlines == newlines

From 8a25696528a62d60d64f663446bd97ed6b9e3744 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <kyle.altendorf@epcpower.com>
Date: Thu, 3 May 2018 08:38:25 -0700
Subject: [PATCH 09/26] Revert "Update newline check"

This reverts commit 415566f5d6d1e47be4ab327d977129e3bca3f771.
---
 tests/integration/test_project.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/integration/test_project.py b/tests/integration/test_project.py
index 464a40a9..eb8f195a 100644
--- a/tests/integration/test_project.py
+++ b/tests/integration/test_project.py
@@ -83,14 +83,14 @@ six = {{version = "*", index = "pypi"}}
 def test_maintain_file_line_endings(PipenvInstance, pypi, newlines, target):
     with PipenvInstance(pypi=pypi, chdir=True) as p:
         path = getattr(p, target)
-        c = p.pipenv('install six')
+        c = p.pipenv('install')
         assert c.return_code == 0
 
         with io.open(path) as f:
             contents = f.read()
             written_newlines = f.newlines
 
-        assert written_newlines == u'\n'
+        assert f.newlines == u'\n'
         if target == 'lockfile_path':
             project = Project()
             project._lockfile_newlines = newlines

From 536d6febc3d4143d522652d64470abc1cfe18dd7 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <kyle.altendorf@epcpower.com>
Date: Thu, 3 May 2018 08:38:33 -0700
Subject: [PATCH 10/26] Revert "Update newline tests"

This reverts commit 18704a5e21720b6fbc7dc62e6f345dc284020dc5.
---
 tests/integration/test_project.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/tests/integration/test_project.py b/tests/integration/test_project.py
index eb8f195a..c9c95e7c 100644
--- a/tests/integration/test_project.py
+++ b/tests/integration/test_project.py
@@ -83,25 +83,25 @@ six = {{version = "*", index = "pypi"}}
 def test_maintain_file_line_endings(PipenvInstance, pypi, newlines, target):
     with PipenvInstance(pypi=pypi, chdir=True) as p:
         path = getattr(p, target)
+
         c = p.pipenv('install')
         assert c.return_code == 0
 
         with io.open(path) as f:
             contents = f.read()
-            written_newlines = f.newlines
 
         assert f.newlines == u'\n'
-        if target == 'lockfile_path':
-            project = Project()
-            project._lockfile_newlines = newlines
-            project.write_lockfile(contents)
-        else:
-            with io.open(path, 'w', newline=newlines) as f:
-                f.write(contents)
+
+        with io.open(path, 'w', newline=newlines) as f:
+            f.write(contents)
+
+        before = os.path.getmtime(path)
 
         c = p.pipenv('install chardet')
         assert c.return_code == 0
 
+        assert os.path.getmtime(path) != before
+
         with io.open(path) as f:
             f.read()
             actual_newlines = f.newlines

From ab63176851def334508a162437b3670eee4c9355 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <kyle.altendorf@epcpower.com>
Date: Thu, 3 May 2018 08:45:44 -0700
Subject: [PATCH 11/26] Add message to newline assertions

Compensating for https://github.com/pytest-dev/pytest/issues/3443
---
 tests/integration/test_project.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/tests/integration/test_project.py b/tests/integration/test_project.py
index c9c95e7c..c3022ca1 100644
--- a/tests/integration/test_project.py
+++ b/tests/integration/test_project.py
@@ -90,7 +90,11 @@ def test_maintain_file_line_endings(PipenvInstance, pypi, newlines, target):
         with io.open(path) as f:
             contents = f.read()
 
-        assert f.newlines == u'\n'
+        assert f.newlines == u'\n', "expected {}, got {}".format(
+            repr(u'\n'),
+            repr(f.newlines),
+        )
+        # message because of  https://github.com/pytest-dev/pytest/issues/3443
 
         with io.open(path, 'w', newline=newlines) as f:
             f.write(contents)
@@ -106,4 +110,8 @@ def test_maintain_file_line_endings(PipenvInstance, pypi, newlines, target):
             f.read()
             actual_newlines = f.newlines
 
-        assert actual_newlines == newlines
+        assert actual_newlines == newlines, "expected {}, got {}".format(
+            repr(newlines),
+            repr(actual_newlines),
+        )
+        # message because of  https://github.com/pytest-dev/pytest/issues/3443

From 2ca94fb010c11aa99d1477c9bc2f79411903792d Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <kyle.altendorf@epcpower.com>
Date: Thu, 3 May 2018 08:50:08 -0700
Subject: [PATCH 12/26] normpath(abspath()) for path comparison

---
 pipenv/project.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/pipenv/project.py b/pipenv/project.py
index 73356e61..22b230b8 100644
--- a/pipenv/project.py
+++ b/pipenv/project.py
@@ -623,7 +623,11 @@ class Project(object):
                         )
                         data[section][package].update(_data)
             formatted_data = toml.dumps(data).rstrip()
-        if os.path.abspath(path) == os.path.abspath(self.pipfile_location):
+
+        def clean_path(path):
+            return os.path.normpath(os.path.abspath(path))
+
+        if clean_path(path) == clean_path(self.pipfile_location):
             newlines = self._pipfile_newlines
         else:
             newlines = DEFAULT_NEWLINES

From a0754453672b3921ec4f8e311cd2c1d2873f922b Mon Sep 17 00:00:00 2001
From: Dan Ryan <dan@danryan.co>
Date: Thu, 3 May 2018 11:59:55 -0400
Subject: [PATCH 13/26] Updated versions

Signed-off-by: Dan Ryan <dan@danryan.co>
---
 pipenv/_compat.py                 | 7 ++++++-
 pipenv/project.py                 | 7 ++++---
 pipenv/utils.py                   | 5 +++++
 tests/integration/test_project.py | 2 +-
 4 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/pipenv/_compat.py b/pipenv/_compat.py
index d6ae5644..3dede246 100644
--- a/pipenv/_compat.py
+++ b/pipenv/_compat.py
@@ -258,6 +258,7 @@ def NamedTemporaryFile(
     if os.name == "nt" and delete:
         flags |= os.O_TEMPORARY
     if six.PY2:
+        # if newline or 'b' not in mode:
         flags = _text_openflags if 'b' not in mode else flags
         (fd, name) = _mkstemp_inner(dir, prefix, suffix, flags)
     else:
@@ -269,6 +270,10 @@ def NamedTemporaryFile(
         return _TemporaryFileWrapper(file, name, delete)
 
     except BaseException:
-        os.unlink(name)
+        try:
+            os.unlink(name)
+        except OSError:
+            os.close(fd)
+            os.unlink(name)
         os.close(fd)
         raise
diff --git a/pipenv/project.py b/pipenv/project.py
index 73356e61..06f8af01 100644
--- a/pipenv/project.py
+++ b/pipenv/project.py
@@ -585,7 +585,7 @@ class Project(object):
                     u'name': source_name,
                 }
             )
-            
+
         data = {
             u'source': sources,
             # Default packages.
@@ -644,9 +644,10 @@ class Project(object):
             s = s.decode('ascii')
 
         with atomic_open_for_write(self.lockfile_location, newline=newlines) as f:
-            f.write(s)
             # Write newline at end of document. GH Issue #319.
-            f.write(u'\n')
+            if not s.endswith(newlines):
+                s = '{0}{1}'.format(s, newlines)
+            f.write(u'{0}'.format(s))
 
     @property
     def pipfile_sources(self):
diff --git a/pipenv/utils.py b/pipenv/utils.py
index e8c94215..f6448586 100644
--- a/pipenv/utils.py
+++ b/pipenv/utils.py
@@ -1305,6 +1305,9 @@ def atomic_open_for_write(target, binary=False, newline=None, encoding=None):
       target with this new file.
     """
     from ._compat import NamedTemporaryFile
+    if six.PY2 and not binary and newline == u'\n':
+        binary = True
+
     mode = 'w+b' if binary else 'w'
     f = NamedTemporaryFile(
         dir=os.path.dirname(target),
@@ -1332,3 +1335,5 @@ def atomic_open_for_write(target, binary=False, newline=None, encoding=None):
         except OSError:
             pass
         os.rename(f.name, target)  # No os.replace() on Python 2.
+    finally:
+        f.close()
diff --git a/tests/integration/test_project.py b/tests/integration/test_project.py
index 9ffa6d8d..4a3145ea 100644
--- a/tests/integration/test_project.py
+++ b/tests/integration/test_project.py
@@ -82,7 +82,7 @@ six = {{version = "*", index = "pypi"}}
 def test_maintain_file_line_endings(PipenvInstance, pypi, newlines):
     with PipenvInstance(pypi=pypi, chdir=True) as p:
         # Initial pipfile + lockfile generation
-        c = p.pipenv('install')
+        c = p.pipenv('install pytz')
         assert c.return_code == 0
 
         # Rewrite each file with parameterized newlines

From d26e16a539b2fd3bc10b91d47852c50f523c8749 Mon Sep 17 00:00:00 2001
From: Dan Ryan <dan@danryan.co>
Date: Thu, 3 May 2018 12:07:02 -0400
Subject: [PATCH 14/26] Read in universal newline mode

Signed-off-by: Dan Ryan <dan@danryan.co>
---
 pipenv/_compat.py                 | 7 +------
 pipenv/project.py                 | 4 ++--
 pipenv/utils.py                   | 4 ----
 tests/integration/test_project.py | 4 ++--
 4 files changed, 5 insertions(+), 14 deletions(-)

diff --git a/pipenv/_compat.py b/pipenv/_compat.py
index 3dede246..d6ae5644 100644
--- a/pipenv/_compat.py
+++ b/pipenv/_compat.py
@@ -258,7 +258,6 @@ def NamedTemporaryFile(
     if os.name == "nt" and delete:
         flags |= os.O_TEMPORARY
     if six.PY2:
-        # if newline or 'b' not in mode:
         flags = _text_openflags if 'b' not in mode else flags
         (fd, name) = _mkstemp_inner(dir, prefix, suffix, flags)
     else:
@@ -270,10 +269,6 @@ def NamedTemporaryFile(
         return _TemporaryFileWrapper(file, name, delete)
 
     except BaseException:
-        try:
-            os.unlink(name)
-        except OSError:
-            os.close(fd)
-            os.unlink(name)
+        os.unlink(name)
         os.close(fd)
         raise
diff --git a/pipenv/project.py b/pipenv/project.py
index 06f8af01..58cbb1ba 100644
--- a/pipenv/project.py
+++ b/pipenv/project.py
@@ -395,7 +395,7 @@ class Project(object):
 
     def read_pipfile(self):
         # Open the pipfile, read it into memory.
-        with io.open(self.pipfile_location) as f:
+        with io.open(self.pipfile_location, newline='') as f:
             contents = f.read()
             self._pipfile_newlines = preferred_newlines(f)
 
@@ -769,7 +769,7 @@ class Project(object):
             self.write_toml(self.parsed_pipfile)
 
     def load_lockfile(self, expand_env_vars=True):
-        with io.open(self.lockfile_location) as lock:
+        with io.open(self.lockfile_location, newline='') as lock:
             j = json.load(lock)
             self._lockfile_newlines = preferred_newlines(lock)
         # lockfile is just a string
diff --git a/pipenv/utils.py b/pipenv/utils.py
index f6448586..5f1f26e1 100644
--- a/pipenv/utils.py
+++ b/pipenv/utils.py
@@ -1305,8 +1305,6 @@ def atomic_open_for_write(target, binary=False, newline=None, encoding=None):
       target with this new file.
     """
     from ._compat import NamedTemporaryFile
-    if six.PY2 and not binary and newline == u'\n':
-        binary = True
 
     mode = 'w+b' if binary else 'w'
     f = NamedTemporaryFile(
@@ -1335,5 +1333,3 @@ def atomic_open_for_write(target, binary=False, newline=None, encoding=None):
         except OSError:
             pass
         os.rename(f.name, target)  # No os.replace() on Python 2.
-    finally:
-        f.close()
diff --git a/tests/integration/test_project.py b/tests/integration/test_project.py
index 4a3145ea..68caad84 100644
--- a/tests/integration/test_project.py
+++ b/tests/integration/test_project.py
@@ -87,7 +87,7 @@ def test_maintain_file_line_endings(PipenvInstance, pypi, newlines):
 
         # Rewrite each file with parameterized newlines
         for fn in [p.pipfile_path, p.lockfile_path]:
-            with io.open(fn) as f:
+            with io.open(fn, newline='') as f:
                 contents = f.read()
                 written_newlines = f.newlines
             assert written_newlines == u'\n'
@@ -100,7 +100,7 @@ def test_maintain_file_line_endings(PipenvInstance, pypi, newlines):
 
         # Make sure we kept the right newlines
         for fn in [p.pipfile_path, p.lockfile_path]:
-            with io.open(fn) as f:
+            with io.open(fn, newline='') as f:
                 contents = f.read()
                 actual_newlines = f.newlines
             assert actual_newlines == newlines

From fb9fb2e57542580f7feb2ec2ddca58d3129fb8ef Mon Sep 17 00:00:00 2001
From: Tzu-ping Chung <uranusjr@gmail.com>
Date: Fri, 4 May 2018 00:58:32 +0800
Subject: [PATCH 15/26] Simplify JSON dump compatibility code

---
 pipenv/project.py | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/pipenv/project.py b/pipenv/project.py
index 58cbb1ba..d9bbc385 100644
--- a/pipenv/project.py
+++ b/pipenv/project.py
@@ -634,20 +634,17 @@ class Project(object):
         self.clear_pipfile_cache()
 
     def write_lockfile(self, content):
-        # Write out the lockfile.
+        """Write out the lockfile.
+        """
         newlines = self._lockfile_newlines
-        s = simplejson.dumps(
-            content, indent=4, separators=(',', ': '), sort_keys=True
+        s = simplejson.dumps(   # Send Unicode in to guarentee Unicode out.
+            content, indent=4, separators=(u',', u': '), sort_keys=True,
         )
-
-        if sys.version_info[0] < 3:
-            s = s.decode('ascii')
-
         with atomic_open_for_write(self.lockfile_location, newline=newlines) as f:
-            # Write newline at end of document. GH Issue #319.
+            f.write(s)
+            # Write newline at end of document. GH#319.
             if not s.endswith(newlines):
-                s = '{0}{1}'.format(s, newlines)
-            f.write(u'{0}'.format(s))
+                f.write(newlines)
 
     @property
     def pipfile_sources(self):

From 0c35920406cd89d77142fe2d8b9bce58823a0b99 Mon Sep 17 00:00:00 2001
From: Tzu-ping Chung <uranusjr@gmail.com>
Date: Fri, 4 May 2018 02:34:48 +0800
Subject: [PATCH 16/26] Always use binary mode in NamedTempoeraryFile

Text mode would enable auto line ending translation, but we want to handle
them ourselves (in io.open) instead. This would not affect the ability to
write text files (io.open does the right thing as long as we give it the
correct mode).
---
 pipenv/_compat.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/pipenv/_compat.py b/pipenv/_compat.py
index d6ae5644..6769d8d4 100644
--- a/pipenv/_compat.py
+++ b/pipenv/_compat.py
@@ -9,7 +9,7 @@ import io
 import os
 import six
 import warnings
-from tempfile import _bin_openflags, gettempdir, _mkstemp_inner, mkdtemp, _text_openflags
+from tempfile import _bin_openflags, gettempdir, _mkstemp_inner, mkdtemp
 from .utils import (logging, rmtree)
 
 try:
@@ -258,13 +258,12 @@ def NamedTemporaryFile(
     if os.name == "nt" and delete:
         flags |= os.O_TEMPORARY
     if six.PY2:
-        flags = _text_openflags if 'b' not in mode else flags
         (fd, name) = _mkstemp_inner(dir, prefix, suffix, flags)
     else:
         (fd, name) = _mkstemp_inner(dir, prefix, suffix, flags, output_type)
     try:
         file = io.open(
-            fd, mode, buffering=buffering, newline=newline, encoding=encoding
+            fd, mode, buffering=buffering, newline=newline, encoding=encoding,
         )
         return _TemporaryFileWrapper(file, name, delete)
 

From acdd2ff1261578b2342b5f05c722de21afd49a95 Mon Sep 17 00:00:00 2001
From: Tzu-ping Chung <uranusjr@gmail.com>
Date: Fri, 4 May 2018 02:37:28 +0800
Subject: [PATCH 17/26] Test for '\n', not platform newlines in user code

Python always generates '\n' in string (unless we explicitly specify
otherwise), so we should test that. Leave line ending handling to library
code.
---
 pipenv/project.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/pipenv/project.py b/pipenv/project.py
index d9bbc385..6baff6c9 100644
--- a/pipenv/project.py
+++ b/pipenv/project.py
@@ -642,9 +642,8 @@ class Project(object):
         )
         with atomic_open_for_write(self.lockfile_location, newline=newlines) as f:
             f.write(s)
-            # Write newline at end of document. GH#319.
-            if not s.endswith(newlines):
-                f.write(newlines)
+            if not s.endswith(u'\n'):
+                f.write(u'\n')  # Write newline at end of document. GH #319.
 
     @property
     def pipfile_sources(self):

From 116ce7191d4a6b73511f068e5e3d83e00e7d1f66 Mon Sep 17 00:00:00 2001
From: Tzu-ping Chung <uranusjr@gmail.com>
Date: Fri, 4 May 2018 02:39:53 +0800
Subject: [PATCH 18/26] Better test errors and remove useless newline=''

---
 tests/integration/test_project.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/tests/integration/test_project.py b/tests/integration/test_project.py
index 68caad84..4808dd2d 100644
--- a/tests/integration/test_project.py
+++ b/tests/integration/test_project.py
@@ -87,10 +87,12 @@ def test_maintain_file_line_endings(PipenvInstance, pypi, newlines):
 
         # Rewrite each file with parameterized newlines
         for fn in [p.pipfile_path, p.lockfile_path]:
-            with io.open(fn, newline='') as f:
+            with io.open(fn) as f:
                 contents = f.read()
                 written_newlines = f.newlines
-            assert written_newlines == u'\n'
+            assert written_newlines == u'\n', '{0!r} != {1!r} for {2}'.format(
+                written_newlines, u'\n', fn,
+            )
             with io.open(fn, 'w', newline=newlines) as f:
                 f.write(contents)
 
@@ -100,7 +102,9 @@ def test_maintain_file_line_endings(PipenvInstance, pypi, newlines):
 
         # Make sure we kept the right newlines
         for fn in [p.pipfile_path, p.lockfile_path]:
-            with io.open(fn, newline='') as f:
-                contents = f.read()
+            with io.open(fn) as f:
+                f.read()    # Consumes the content to detect newlines.
                 actual_newlines = f.newlines
-            assert actual_newlines == newlines
+            assert actual_newlines == newlines, '{0!r} != {1!r} for {2}'.format(
+                actual_newlines, newlines, fn,
+            )

From 6bac97c50b210c3e815fa00ee7a3794224edabca Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <kyle.altendorf@epcpower.com>
Date: Thu, 3 May 2018 11:59:14 -0700
Subject: [PATCH 19/26] Always os.open() with os.O_BINARY() for
 NamedTemporaryFile

---
 pipenv/_compat.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/pipenv/_compat.py b/pipenv/_compat.py
index d6ae5644..692efa90 100644
--- a/pipenv/_compat.py
+++ b/pipenv/_compat.py
@@ -9,7 +9,7 @@ import io
 import os
 import six
 import warnings
-from tempfile import _bin_openflags, gettempdir, _mkstemp_inner, mkdtemp, _text_openflags
+from tempfile import _bin_openflags, gettempdir, _mkstemp_inner, mkdtemp
 from .utils import (logging, rmtree)
 
 try:
@@ -258,7 +258,6 @@ def NamedTemporaryFile(
     if os.name == "nt" and delete:
         flags |= os.O_TEMPORARY
     if six.PY2:
-        flags = _text_openflags if 'b' not in mode else flags
         (fd, name) = _mkstemp_inner(dir, prefix, suffix, flags)
     else:
         (fd, name) = _mkstemp_inner(dir, prefix, suffix, flags, output_type)

From 7e32d26e62001199d39d6b21eb5440775d73eec7 Mon Sep 17 00:00:00 2001
From: Dan Ryan <dan@danryan.co>
Date: Thu, 3 May 2018 18:49:17 -0400
Subject: [PATCH 20/26] Update pathlib import, drop extra io.open arg

Signed-off-by: Dan Ryan <dan@danryan.co>
---
 pipenv/project.py | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/pipenv/project.py b/pipenv/project.py
index c406f1d6..0990afa1 100644
--- a/pipenv/project.py
+++ b/pipenv/project.py
@@ -1,5 +1,4 @@
 # -*- coding: utf-8 -*-
-import codecs
 import io
 import json
 import os
@@ -16,9 +15,9 @@ import toml
 import json as simplejson
 
 try:
-    import pathlib
+    from pathlib import Path
 except ImportError:
-    import pathlib2 as pathlib
+    from pathlib2 import Path
 
 from .cmdparse import Script
 from .utils import (
@@ -50,7 +49,7 @@ from .environments import (
 def _normalized(p):
     if p is None:
         return None
-    return normalize_drive(str(pathlib.Path(p).resolve()))
+    return normalize_drive(str(Path(p).resolve()))
 
 
 DEFAULT_NEWLINES = u'\n'
@@ -395,7 +394,7 @@ class Project(object):
 
     def read_pipfile(self):
         # Open the pipfile, read it into memory.
-        with io.open(self.pipfile_location, newline='') as f:
+        with io.open(self.pipfile_location) as f:
             contents = f.read()
             self._pipfile_newlines = preferred_newlines(f)
 
@@ -624,10 +623,7 @@ class Project(object):
                         data[section][package].update(_data)
             formatted_data = toml.dumps(data).rstrip()
 
-        def clean_path(path):
-            return os.path.normpath(os.path.abspath(path))
-
-        if clean_path(path) == clean_path(self.pipfile_location):
+        if Path(path).absolute() == Path(self.pipfile_location).absolute():
             newlines = self._pipfile_newlines
         else:
             newlines = DEFAULT_NEWLINES
@@ -769,7 +765,7 @@ class Project(object):
             self.write_toml(self.parsed_pipfile)
 
     def load_lockfile(self, expand_env_vars=True):
-        with io.open(self.lockfile_location, newline='') as lock:
+        with io.open(self.lockfile_location) as lock:
             j = json.load(lock)
             self._lockfile_newlines = preferred_newlines(lock)
         # lockfile is just a string

From d21efa08c79427bb05189269ae05e892b9196e7a Mon Sep 17 00:00:00 2001
From: Tzu-ping Chung <uranusjr@gmail.com>
Date: Thu, 3 May 2018 23:19:47 +0800
Subject: [PATCH 21/26] Check lockfile exists on sync, and don't update it

Matching the behaviour mentioned in:
https://github.com/pypa/pipenv/issues/1463#issuecomment-386297896
---
 pipenv/core.py                 | 23 ++++++++++++++-----
 tests/integration/test_sync.py | 42 ++++++++++++++++++++++++++++++++++
 2 files changed, 59 insertions(+), 6 deletions(-)
 create mode 100644 tests/integration/test_sync.py

diff --git a/pipenv/core.py b/pipenv/core.py
index 711ec28d..7ca65872 100644
--- a/pipenv/core.py
+++ b/pipenv/core.py
@@ -2485,19 +2485,30 @@ def do_sync(
     unused=False,
     sequential=False,
 ):
+    # The lock file needs to exist because sync won't write to it.
+    if not project.lockfile_exists:
+        click.echo(
+            '{0}: Pipfile.lock is missing! You need to run {1} first.'.format(
+                crayons.red('Error', bold=True),
+                crayons.red('$ pipenv lock', bold=True),
+            ),
+            err=True,
+        )
+        sys.exit(1)
+
+    # Ensure that virtualenv is available.
+    ensure_project(three=three, python=python, validate=False)
+
+    # Install everything.
     requirements_dir = TemporaryDirectory(
         suffix='-requirements', prefix='pipenv-'
     )
-    # Ensure that virtualenv is available.
-    ensure_project(three=three, python=python, validate=False)
-    concurrent = (not sequential)
-    ensure_lockfile()
-    # Install everything.
     do_init(
         dev=dev,
         verbose=verbose,
-        concurrent=concurrent,
+        concurrent=(not sequential),
         requirements_dir=requirements_dir,
+        ignore_pipfile=True,    # Don't check if Pipfile and lock match.
     )
     requirements_dir.cleanup()
     click.echo(crayons.green('All dependencies are now up-to-date!'))
diff --git a/tests/integration/test_sync.py b/tests/integration/test_sync.py
new file mode 100644
index 00000000..1116ed83
--- /dev/null
+++ b/tests/integration/test_sync.py
@@ -0,0 +1,42 @@
+import pytest
+
+
+@pytest.mark.sync
+def test_sync_error_without_lockfile(PipenvInstance, pypi):
+    with PipenvInstance(pypi=pypi) as p:
+        with open(p.pipfile_path, 'w') as f:
+            f.write("""
+[packages]
+            """.strip())
+
+        c = p.pipenv('sync')
+        assert c.return_code != 0
+        assert 'Pipfile.lock is missing!' in c.err
+
+
+@pytest.mark.sync
+@pytest.mark.lock
+def test_sync_should_not_lock(PipenvInstance, pypi):
+    """Sync should not touch the lock file, even if Pipfile is changed.
+    """
+    with PipenvInstance(pypi=pypi) as p:
+        with open(p.pipfile_path, 'w') as f:
+            f.write("""
+[packages]
+            """.strip())
+
+        # Perform initial lock.
+        c = p.pipenv('lock')
+        assert c.return_code == 0
+        lockfile_content = p.lockfile
+        assert lockfile_content
+
+        # Make sure sync does not trigger lockfile update.
+        with open(p.pipfile_path, 'w') as f:
+            f.write("""
+[packages]
+six = "*"
+            """.strip())
+        c = p.pipenv('sync')
+        assert c.return_code == 0
+        assert lockfile_content == p.lockfile

From af0150b07e3bfc3c0eea14714df0aed9fe518861 Mon Sep 17 00:00:00 2001
From: Dan Ryan <dan@danryan.co>
Date: Wed, 2 May 2018 00:53:49 -0400
Subject: [PATCH 22/26] Update vendoring patches

Signed-off-by: Dan Ryan <dan@danryan.co>
---
 pipenv/__version__.py                                 |  2 +-
 .../patches/patched/_post-pip-update-pypi-uri.patch   | 11 +++++++++++
 tasks/vendoring/patches/patched/pipfile.patch         |  2 +-
 tasks/vendoring/patches/patched/piptools.patch        |  3 ++-
 4 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/pipenv/__version__.py b/pipenv/__version__.py
index d58e2474..9714b0cd 100644
--- a/pipenv/__version__.py
+++ b/pipenv/__version__.py
@@ -2,4 +2,4 @@
 #   //   ) ) / / //   ) ) //___) ) //   ) ) ||  / /
 #  //___/ / / / //___/ / //       //   / /  || / /
 # //       / / //       ((____   //   / /   ||/ /
-__version__ = '11.10.1'
+__version__ = '11.10.2.dev1'
diff --git a/tasks/vendoring/patches/patched/_post-pip-update-pypi-uri.patch b/tasks/vendoring/patches/patched/_post-pip-update-pypi-uri.patch
index 58adff3e..d6d1ed93 100644
--- a/tasks/vendoring/patches/patched/_post-pip-update-pypi-uri.patch
+++ b/tasks/vendoring/patches/patched/_post-pip-update-pypi-uri.patch
@@ -55,3 +55,14 @@ index 48aaa35..bf1cba9 100644
  
          url:
              url of the resource pointed to (href of the link)
+ 
+diff --git a/pipenv/patched/notpip/models/index.py b/pipenv/patched/notpip/models/index.py
+index 25fd488d..db324287 100644
+--- a/pipenv/patched/notpip/models/index.py
++++ b/pipenv/patched/notpip/models/index.py
+@@ -13,4 +13,4 @@ class Index(object):
+         return urllib_parse.urljoin(self.url, path)
+ 
+ 
+-PyPI = Index('https://pypi.python.org/')
++PyPI = Index('https://pypi.org/')
diff --git a/tasks/vendoring/patches/patched/pipfile.patch b/tasks/vendoring/patches/patched/pipfile.patch
index 90a3d05b..4344e4bd 100644
--- a/tasks/vendoring/patches/patched/pipfile.patch
+++ b/tasks/vendoring/patches/patched/pipfile.patch
@@ -12,7 +12,7 @@ index 8a2a6a3..18a1ea2 100644
  
  
 +DEFAULT_SOURCE = {
-+    u'url': u'https://pypi.python.org/simple',
++    u'url': u'https://pypi.org/simple',
 +    u'verify_ssl': True,
 +    u'name': u'pypi',
 +}
diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch
index 4946c491..951ce87d 100644
--- a/tasks/vendoring/patches/patched/piptools.patch
+++ b/tasks/vendoring/patches/patched/piptools.patch
@@ -85,7 +85,8 @@ index d3b7fe7..e1f63d2 100644
 +
 +
  class PyPIRepository(BaseRepository):
-     DEFAULT_INDEX_URL = 'https://pypi.python.org/simple'
+-    DEFAULT_INDEX_URL = 'https://pypi.python.org/simple'
++    DEFAULT_INDEX_URL = 'https://pypi.org/simple'
  
 @@ -30,8 +69,9 @@ class PyPIRepository(BaseRepository):
      config), but any other PyPI mirror can be used if index_urls is

From 17888fb1319e3ed9b3e2e720d89e6e750f0ef1df Mon Sep 17 00:00:00 2001
From: Dan Ryan <dan@danryan.co>
Date: Wed, 2 May 2018 02:00:28 -0400
Subject: [PATCH 23/26] Auto-download pip licenses

Signed-off-by: Dan Ryan <dan@danryan.co>
---
 pipenv/vendor/vendor_pip.txt                  |  21 ++++
 tasks/vendoring/__init__.py                   | 100 +++++++-----------
 .../patched/pipfile-update-pypi-uri.patch     |  13 ---
 3 files changed, 57 insertions(+), 77 deletions(-)
 create mode 100644 pipenv/vendor/vendor_pip.txt
 delete mode 100644 tasks/vendoring/patches/patched/pipfile-update-pypi-uri.patch

diff --git a/pipenv/vendor/vendor_pip.txt b/pipenv/vendor/vendor_pip.txt
new file mode 100644
index 00000000..53181c39
--- /dev/null
+++ b/pipenv/vendor/vendor_pip.txt
@@ -0,0 +1,21 @@
+setuptools==39.1.0
+appdirs==1.4.0
+distlib==0.2.4
+distro==1.2.0
+html5lib==1.0b10
+six==1.10.0
+colorama==0.3.7
+requests==2.18.4
+chardet==3.0.4
+idna==2.6
+urllib3==1.22
+certifi==2018.1.18
+CacheControl==0.11.7
+lockfile==0.12.2
+ordereddict==1.1
+progress==1.2
+ipaddress==1.0.17
+packaging==16.8
+pyparsing==2.1.10
+retrying==1.3.3
+webencodings==0.5
diff --git a/tasks/vendoring/__init__.py b/tasks/vendoring/__init__.py
index 37db03fc..5280b799 100644
--- a/tasks/vendoring/__init__.py
+++ b/tasks/vendoring/__init__.py
@@ -17,13 +17,14 @@ import requests
 
 TASK_NAME = 'update'
 
-LIBRARY_OVERRIDES = {
+LIBRARY_DIRNAMES = {
     'requirements-parser': 'requirements',
     'backports.shutil_get_terminal_size': 'backports/shutil_get_terminal_size',
     'backports.weakref': 'backports/weakref',
     'shutil_backports': 'backports/shutil_get_terminal_size',
     'python-dotenv': 'dotenv',
-    'pip-tools': 'piptools'
+    'pip-tools': 'piptools',
+    'setuptools': 'pkg_resources',
 }
 
 # from time to time, remove the no longer needed ones
@@ -38,7 +39,10 @@ HARDCODED_LICENSE_URLS = {
     'semver': 'https://raw.githubusercontent.com/k-bx/python-semver/master/LICENSE.txt',
     'crayons': 'https://raw.githubusercontent.com/kennethreitz/crayons/master/LICENSE',
     'pip-tools': 'https://raw.githubusercontent.com/jazzband/pip-tools/master/LICENSE',
-    'pew': 'https://raw.githubusercontent.com/berdario/pew/master/LICENSE'
+    'pew': 'https://raw.githubusercontent.com/berdario/pew/master/LICENSE',
+    'pytoml': 'https://github.com/avakar/pytoml/raw/master/LICENSE',
+    'webencodings': 'https://github.com/SimonSapin/python-webencodings/raw/'
+                    'master/LICENSE',
 }
 
 FILE_WHITE_LIST = (
@@ -50,7 +54,8 @@ FILE_WHITE_LIST = (
     'README.md',
     'appdirs.py',
     'safety.zip',
-    'cacert.pem'
+    'cacert.pem',
+    'vendor_pip.txt',
 )
 
 LIBRARY_RENAMES = {
@@ -212,14 +217,6 @@ cli(prog_name="safety")
             else:
                 lib = yaml_build_dir / 'lib3' / 'yaml'
             shutil.copytree(str(lib.absolute()), str(safety_dir / 'yaml{0}'.format(version_choices[0])))
-#             yaml_init = yaml_dir / '__init__.py'
-#             yaml_init.write_text("""
-# import sys
-# if sys.version_info[0] == 3:
-#     from .yaml3 import *
-# else:
-#     from .yaml2 import *
-#             """.strip())
         requests_dir = safety_dir / 'requests'
         cacert = vendor_dir / 'requests' / 'cacert.pem'
         if not cacert.exists():
@@ -247,8 +244,8 @@ cli(prog_name="safety")
 def rename_if_needed(ctx, vendor_dir, item):
     rename_dict = LIBRARY_RENAMES if vendor_dir.name != 'patched' else PATCHED_RENAMES
     new_path = None
-    if item.name in rename_dict or item.name in LIBRARY_OVERRIDES:
-        new_name = rename_dict.get(item.name, LIBRARY_OVERRIDES.get(item.name))
+    if item.name in rename_dict or item.name in LIBRARY_DIRNAMES:
+        new_name = rename_dict.get(item.name, LIBRARY_DIRNAMES.get(item.name))
         new_path = item.parent / new_name
         log('Renaming %s => %s' % (item.name, new_path))
         # handle existing directories
@@ -307,7 +304,6 @@ def vendor(ctx, vendor_dir, rewrite=True):
                 apply_patch(ctx, patch)
 
     # Global import rewrites
-    # log("Rewriting all imports related to vendored libs")
     log('Renaming specified libs...')
     for item in vendor_dir.iterdir():
         if item.is_dir():
@@ -401,16 +397,17 @@ def find_and_extract_license(vendor_dir, tar, members):
 
 def license_fallback(vendor_dir, sdist_name):
     """Hardcoded license URLs. Check when updating if those are still needed"""
-    for libname, url in HARDCODED_LICENSE_URLS.items():
-        if libname in sdist_name:
-            _, _, name = url.rpartition('/')
-            dest = license_destination(vendor_dir, libname, name)
-            r = requests.get(url, allow_redirects=True)
-            log('Downloading {}'.format(url))
-            r.raise_for_status()
-            dest.write_bytes(r.content)
-            return
-    raise ValueError('No hardcoded URL for {} license'.format(sdist_name))
+    libname = libname_from_dir(sdist_name)
+    if libname not in HARDCODED_LICENSE_URLS:
+        raise ValueError('No hardcoded URL for {} license'.format(libname))
+
+    url = HARDCODED_LICENSE_URLS[libname]
+    _, _, name = url.rpartition('/')
+    dest = license_destination(vendor_dir, libname, name)
+    r = requests.get(url, allow_redirects=True)
+    log('Downloading {}'.format(url))
+    r.raise_for_status()
+    dest.write_bytes(r.content)
 
 
 def libname_from_dir(dirname):
@@ -420,7 +417,7 @@ def libname_from_dir(dirname):
         if part[0].isdigit():
             break
         parts.append(part)
-    return'-'.join(parts)
+    return '-'.join(parts)
 
 
 def license_destination(vendor_dir, libname, filename):
@@ -432,16 +429,17 @@ def license_destination(vendor_dir, libname, filename):
     if lowercase.is_dir():
         return lowercase / filename
     rename_dict = LIBRARY_RENAMES if vendor_dir.name != 'patched' else PATCHED_RENAMES
+    # Short circuit all logic if we are renaming the whole library
     if libname in rename_dict:
         return vendor_dir / rename_dict[libname] / filename
-    if libname in LIBRARY_OVERRIDES:
-        override = vendor_dir / LIBRARY_OVERRIDES[libname]
+    if libname in LIBRARY_DIRNAMES:
+        override = vendor_dir / LIBRARY_DIRNAMES[libname]
         if not override.exists() and override.parent.exists():
             # for flattened subdeps, specifically backports/weakref.py
-            target_dir = vendor_dir / override.parent
-            target_file = '{0}.{1}'.format(override.name, filename)
-            return target_dir / target_file
-        return vendor_dir / LIBRARY_OVERRIDES[libname] / filename
+            return (
+                vendor_dir / override.parent
+            ) / '{0}.{1}'.format(override.name, filename)
+        return vendor_dir / LIBRARY_DIRNAMES[libname] / filename
     # fallback to libname.LICENSE (used for nondirs)
     return vendor_dir / '{}.{}'.format(libname, filename)
 
@@ -451,8 +449,6 @@ def extract_license_member(vendor_dir, tar, member, name):
     dirname = list(mpath.parents)[-2].name  # -1 is .
     libname = libname_from_dir(dirname)
     dest = license_destination(vendor_dir, libname, mpath.name)
-    # dest_relative = dest.relative_to(Path.cwd())
-    # log('Extracting {} into {}'.format(name, dest_relative))
     log('Extracting {} into {}'.format(name, dest))
     try:
         fileobj = tar.extractfile(member)
@@ -461,36 +457,6 @@ def extract_license_member(vendor_dir, tar, member, name):
         dest.write_bytes(tar.read(member))
 
 
-@invoke.task
-def update_stubs(ctx):
-    vendor_dir = _get_vendor_dir(ctx)
-    vendored_libs = detect_vendored_libs(vendor_dir)
-
-    print("[vendoring.update_stubs] Add mypy stubs")
-
-    extra_stubs_needed = {
-        # Some projects need stubs other than a simple <name>.pyi
-        "six": ["six.__init__", "six.moves"],
-        # Some projects should not have stubs coz they're single file modules
-        "appdirs": [],
-    }
-
-    for lib in vendored_libs:
-        if lib not in extra_stubs_needed:
-            (vendor_dir / (lib + ".pyi")).write_text("from %s import *" % lib)
-            continue
-
-        for selector in extra_stubs_needed[lib]:
-            fname = selector.replace(".", os.sep) + ".pyi"
-            if selector.endswith(".__init__"):
-                selector = selector[:-9]
-
-            f_path = vendor_dir / fname
-            if not f_path.parent.exists():
-                f_path.parent.mkdir()
-        f_path.write_text("from %s import *" % selector)
-
-
 @invoke.task(name=TASK_NAME)
 def main(ctx):
     vendor_dir = _get_vendor_dir(ctx)
@@ -502,5 +468,11 @@ def main(ctx):
     vendor(ctx, patched_dir, rewrite=False)
     download_licenses(ctx, vendor_dir)
     download_licenses(ctx, patched_dir, 'patched.txt')
+    for pip_dir in [vendor_dir / 'pip9', patched_dir / 'notpip']:
+        _vendor_dir = pip_dir / '_vendor'
+        vendor_src_file = vendor_dir / 'vendor_pip.txt'
+        vendor_file = _vendor_dir / 'vendor.txt'
+        vendor_file.write_bytes(vendor_src_file.read_bytes())
+        download_licenses(ctx, _vendor_dir)
     # update_safety(ctx)
     log('Revendoring complete')
diff --git a/tasks/vendoring/patches/patched/pipfile-update-pypi-uri.patch b/tasks/vendoring/patches/patched/pipfile-update-pypi-uri.patch
deleted file mode 100644
index ecbe2aac..00000000
--- a/tasks/vendoring/patches/patched/pipfile-update-pypi-uri.patch
+++ /dev/null
@@ -1,13 +0,0 @@
-diff --git a/pipenv/patched/pipfile/api.py b/pipenv/patched/pipfile/api.py
-index 18a1ea2..e8fa027 100644
---- a/pipenv/patched/pipfile/api.py
-+++ b/pipenv/patched/pipfile/api.py
-@@ -10,7 +10,7 @@ import os
- 
- 
- DEFAULT_SOURCE = {
--    u'url': u'https://pypi.python.org/simple',
-+    u'url': u'https://pypi.org/simple',
-     u'verify_ssl': True,
-     u'name': u'pypi',
- }

From 992645ead0e29fef640234b665ccdb69d7e10db9 Mon Sep 17 00:00:00 2001
From: Dan Ryan <dan@danryan.co>
Date: Wed, 2 May 2018 16:37:23 -0400
Subject: [PATCH 24/26] Update licenses

Signed-off-by: Dan Ryan <dan@danryan.co>
---
 .../patched/notpip/_vendor/requests/LICENSE   |  2 +-
 pipenv/patched/notpip/_vendor/vendor.txt      | 21 +++++++++++++++++++
 pipenv/vendor/pip9/_vendor/requests/LICENSE   |  2 +-
 pipenv/vendor/pip9/_vendor/vendor.txt         | 21 +++++++++++++++++++
 4 files changed, 44 insertions(+), 2 deletions(-)
 create mode 100644 pipenv/patched/notpip/_vendor/vendor.txt
 create mode 100644 pipenv/vendor/pip9/_vendor/vendor.txt

diff --git a/pipenv/patched/notpip/_vendor/requests/LICENSE b/pipenv/patched/notpip/_vendor/requests/LICENSE
index 8c1dd448..db78ea69 100644
--- a/pipenv/patched/notpip/_vendor/requests/LICENSE
+++ b/pipenv/patched/notpip/_vendor/requests/LICENSE
@@ -1,4 +1,4 @@
-Copyright 2016 Kenneth Reitz
+Copyright 2017 Kenneth Reitz
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
diff --git a/pipenv/patched/notpip/_vendor/vendor.txt b/pipenv/patched/notpip/_vendor/vendor.txt
new file mode 100644
index 00000000..53181c39
--- /dev/null
+++ b/pipenv/patched/notpip/_vendor/vendor.txt
@@ -0,0 +1,21 @@
+setuptools==39.1.0
+appdirs==1.4.0
+distlib==0.2.4
+distro==1.2.0
+html5lib==1.0b10
+six==1.10.0
+colorama==0.3.7
+requests==2.18.4
+chardet==3.0.4
+idna==2.6
+urllib3==1.22
+certifi==2018.1.18
+CacheControl==0.11.7
+lockfile==0.12.2
+ordereddict==1.1
+progress==1.2
+ipaddress==1.0.17
+packaging==16.8
+pyparsing==2.1.10
+retrying==1.3.3
+webencodings==0.5
diff --git a/pipenv/vendor/pip9/_vendor/requests/LICENSE b/pipenv/vendor/pip9/_vendor/requests/LICENSE
index 8c1dd448..db78ea69 100644
--- a/pipenv/vendor/pip9/_vendor/requests/LICENSE
+++ b/pipenv/vendor/pip9/_vendor/requests/LICENSE
@@ -1,4 +1,4 @@
-Copyright 2016 Kenneth Reitz
+Copyright 2017 Kenneth Reitz
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
diff --git a/pipenv/vendor/pip9/_vendor/vendor.txt b/pipenv/vendor/pip9/_vendor/vendor.txt
new file mode 100644
index 00000000..53181c39
--- /dev/null
+++ b/pipenv/vendor/pip9/_vendor/vendor.txt
@@ -0,0 +1,21 @@
+setuptools==39.1.0
+appdirs==1.4.0
+distlib==0.2.4
+distro==1.2.0
+html5lib==1.0b10
+six==1.10.0
+colorama==0.3.7
+requests==2.18.4
+chardet==3.0.4
+idna==2.6
+urllib3==1.22
+certifi==2018.1.18
+CacheControl==0.11.7
+lockfile==0.12.2
+ordereddict==1.1
+progress==1.2
+ipaddress==1.0.17
+packaging==16.8
+pyparsing==2.1.10
+retrying==1.3.3
+webencodings==0.5

From 5f047b5f5f3c38d6bf75d9a5662ee2e7f2bb5137 Mon Sep 17 00:00:00 2001
From: Dan Ryan <dan@danryan.co>
Date: Thu, 3 May 2018 20:06:47 -0400
Subject: [PATCH 25/26] Patches need to have 1 whitespace on empty lines

Signed-off-by: Dan Ryan <dan@danryan.co>
---
 .../patches/patched/prettytoml-table-iter.patch          | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/tasks/vendoring/patches/patched/prettytoml-table-iter.patch b/tasks/vendoring/patches/patched/prettytoml-table-iter.patch
index 5758e122..9ec52633 100644
--- a/tasks/vendoring/patches/patched/prettytoml-table-iter.patch
+++ b/tasks/vendoring/patches/patched/prettytoml-table-iter.patch
@@ -10,19 +10,20 @@ index 59fd5748..48663aed 100644
 +
  from prettytoml.elements.common import ContainerElement
  from prettytoml.elements import traversal
-
-
+ 
+ 
 -class AbstractTable(ContainerElement, traversal.TraversalMixin):
 +class AbstractTable(ContainerElement, traversal.TraversalMixin, Mapping):
      """
      Common code for handling tables as key-value pairs with metadata elements sprinkled all over.
-
+ 
 @@ -37,6 +42,9 @@ class AbstractTable(ContainerElement, traversal.TraversalMixin):
      def __len__(self):
          return len(tuple(self._enumerate_items()))
-
+ 
 +    def __iter__(self):
 +        return (key for key, _ in self.items())
 +
      def __contains__(self, item):
          return item in self.keys()
+ 

From f0c33367fa2dd9686c03db3494dbe272d78fb781 Mon Sep 17 00:00:00 2001
From: Dan Ryan <dan@danryan.co>
Date: Thu, 3 May 2018 20:53:48 -0400
Subject: [PATCH 26/26] Parse and include extras from vcs and non-vcs urls

- Include extras when rebuilding urls from pipfiles
- Fixes #1997, #2128

Signed-off-by: Dan Ryan <dan@danryan.co>
---
 pipenv/utils.py          | 15 +++++++++++----
 tests/unit/test_utils.py | 30 +++++++++++++++---------------
 2 files changed, 26 insertions(+), 19 deletions(-)

diff --git a/pipenv/utils.py b/pipenv/utils.py
index 5f1f26e1..46add117 100644
--- a/pipenv/utils.py
+++ b/pipenv/utils.py
@@ -657,6 +657,7 @@ def convert_deps_to_pip(deps, project=None, r=True, include_index=False):
     for dep in deps.keys():
         # Default (e.g. '>1.10').
         extra = deps[dep] if isinstance(deps[dep], six.string_types) else ''
+        extras = ''
         version = ''
         index = ''
         # Get rid of '*'.
@@ -675,7 +676,7 @@ def convert_deps_to_pip(deps, project=None, r=True, include_index=False):
             )
         # Support for extras (e.g. requests[socks])
         if 'extras' in deps[dep]:
-            extra = '[{0}]'.format(','.join(deps[dep]['extras']))
+            extras = '[{0}]'.format(','.join(deps[dep]['extras']))
         if 'version' in deps[dep]:
             if not is_star(deps[dep]['version']):
                 version = deps[dep]['version']
@@ -709,9 +710,14 @@ def convert_deps_to_pip(deps, project=None, r=True, include_index=False):
         # Support for version control
         maybe_vcs = [vcs for vcs in VCS_LIST if vcs in deps[dep]]
         vcs = maybe_vcs[0] if maybe_vcs else None
+        if not any(key in deps[dep] for key in ['path', 'vcs', 'file']):
+            extra += extras
         # Support for files.
         if 'file' in deps[dep]:
-            extra = '{1}{0}'.format(extra, deps[dep]['file']).strip()
+            dep_file = deps[dep]['file']
+            if is_valid_url(dep_file) and dep_file.startswith('http'):
+                dep_file += '#egg={0}'.format(dep)
+            extra = '{0}{1}'.format(dep_file, extras).strip()
             # Flag the file as editable if it is a local relative path
             if 'editable' in deps[dep]:
                 dep = '-e '
@@ -719,7 +725,7 @@ def convert_deps_to_pip(deps, project=None, r=True, include_index=False):
                 dep = ''
         # Support for paths.
         elif 'path' in deps[dep]:
-            extra = '{1}{0}'.format(extra, deps[dep]['path']).strip()
+            extra = '{1}{0}'.format(extras, deps[dep]['path']).strip()
             # Flag the file as editable if it is a local relative path
             if 'editable' in deps[dep]:
                 dep = '-e '
@@ -730,7 +736,7 @@ def convert_deps_to_pip(deps, project=None, r=True, include_index=False):
             # Support for @refs.
             if 'ref' in deps[dep]:
                 extra += '@{0}'.format(deps[dep]['ref'])
-            extra += '#egg={0}'.format(dep)
+            extra += '#egg={0}{1}'.format(dep, extras)
             # Support for subdirectory
             if 'subdirectory' in deps[dep]:
                 extra += '&subdirectory={0}'.format(deps[dep]['subdirectory'])
@@ -740,6 +746,7 @@ def convert_deps_to_pip(deps, project=None, r=True, include_index=False):
                 dep = '-e '
             else:
                 dep = ''
+
         s = '{0}{1}{2}{3}{4} {5}'.format(
             dep, extra, version, specs, hash, index
         ).strip()
diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py
index b10e273f..1eedcccc 100644
--- a/tests/unit/test_utils.py
+++ b/tests/unit/test_utils.py
@@ -44,7 +44,21 @@ DEP_PIP_PAIRS = [
         }},
         '-e svn+svn://svn.myproject.org/svn/MyProject#egg=MyProject',
     ),
-
+    (
+        # Extras in url
+        {'discord.py': {
+                'file': 'https://github.com/Rapptz/discord.py/archive/rewrite.zip',
+                'extras': ['voice']
+        }},
+        'https://github.com/Rapptz/discord.py/archive/rewrite.zip#egg=discord.py[voice]',
+    ),
+    (
+        {'requests': {
+            'git': 'https://github.com/requests/requests.git',
+            'ref': 'master', 'extras': ['security'],
+        }},
+        'git+https://github.com/requests/requests.git@master#egg=requests[security]',
+    ),
 ]
 
 
@@ -97,20 +111,6 @@ def test_convert_from_pip(expected, requirement):
     assert pipenv.utils.convert_deps_from_pip(requirement) == expected
 
 
-@pytest.mark.utils
-@pytest.mark.parametrize('expected, requirement', [
-    (   # XXX: This should work the other way around as well, but does not atm.
-        {'requests': {
-            'git': 'https://github.com/requests/requests.git',
-            'ref': 'master', 'extras': ['security'],
-        }},
-        'git+https://github.com/requests/requests.git@master#egg=requests[security]',
-    ),
-])
-def test_convert_from_pip_vcs_with_extra(expected, requirement):
-    assert pipenv.utils.convert_deps_from_pip(requirement) == expected
-
-
 @pytest.mark.utils
 def test_convert_from_pip_fail_if_no_egg():
     """Parsing should fail without `#egg=`.