diff --git a/.vsts-ci/docs.yml b/.azure-pipelines/docs.yml similarity index 73% rename from .vsts-ci/docs.yml rename to .azure-pipelines/docs.yml index 1b9d3701..b65fdfb4 100644 --- a/.vsts-ci/docs.yml +++ b/.azure-pipelines/docs.yml @@ -1,7 +1,8 @@ -phases: -- phase: +jobs: +- job: displayName: Docs - queue: Hosted Linux Preview + pool: + vmImage: ubuntu-16.04 steps: - task: UsePythonVersion@0 @@ -16,5 +17,5 @@ phases: - task: PublishBuildArtifacts@1 displayName: 'Publish Artifact: docs' inputs: - PathtoPublish: docs/build - ArtifactName: docs + pathToPublish: docs/build + artifactName: docs diff --git a/.vsts-ci/phases/run-manifest-check.yml b/.azure-pipelines/jobs/run-manifest-check.yml similarity index 63% rename from .vsts-ci/phases/run-manifest-check.yml rename to .azure-pipelines/jobs/run-manifest-check.yml index 484641dc..6d0b97eb 100644 --- a/.vsts-ci/phases/run-manifest-check.yml +++ b/.azure-pipelines/jobs/run-manifest-check.yml @@ -8,7 +8,7 @@ steps: - template: ../steps/install-dependencies.yml - bash: | - export GIT_SSL_CAINFO=$(python -m certifi) - export LANG=C.UTF-8 - python -m pip install check-manifest - check-manifest + export GIT_SSL_CAINFO=$(python -m certifi) + export LANG=C.UTF-8 + python -m pip install check-manifest + check-manifest diff --git a/.vsts-ci/phases/run-tests-windows.yml b/.azure-pipelines/jobs/run-tests-windows.yml similarity index 100% rename from .vsts-ci/phases/run-tests-windows.yml rename to .azure-pipelines/jobs/run-tests-windows.yml diff --git a/.vsts-ci/phases/run-tests.yml b/.azure-pipelines/jobs/run-tests.yml similarity index 55% rename from .vsts-ci/phases/run-tests.yml rename to .azure-pipelines/jobs/run-tests.yml index 81f02875..fe98e8ec 100644 --- a/.vsts-ci/phases/run-tests.yml +++ b/.azure-pipelines/jobs/run-tests.yml @@ -8,23 +8,23 @@ steps: - template: ../steps/install-dependencies.yml - bash: | - mkdir -p "$AGENT_HOMEDIRECTORY/.virtualenvs" - mkdir -p "$WORKON_HOME" - pip install certifi - export GIT_SSL_CAINFO=$(python -m certifi) - export LANG=C.UTF-8 - export PIP_PROCESS_DEPENDENCY_LINKS="1" - echo "Path: $PATH" - echo "Installing Pipenv…" - pip install -e "$(pwd)" --upgrade - pipenv install --deploy --dev - echo pipenv --venv && echo pipenv --py && echo pipenv run python --version + mkdir -p "$AGENT_HOMEDIRECTORY/.virtualenvs" + mkdir -p "$WORKON_HOME" + pip install certifi + export GIT_SSL_CAINFO="$(python -m certifi)" + export LANG="C.UTF-8" + export PIP_PROCESS_DEPENDENCY_LINKS="1" + echo "Path $PATH" + echo "Installing Pipenv…" + pip install -e "$(pwd)" --upgrade + pipenv install --deploy --dev + echo pipenv --venv && echo pipenv --py && echo pipenv run python --version displayName: Make Virtualenv - script: | # Fix Git SSL errors - export GIT_SSL_CAINFO=$(python -m certifi) - export LANG=C.UTF-8 + export GIT_SSL_CAINFO="$(python -m certifi)" + export LANG="C.UTF-8" export PIP_PROCESS_DEPENDENCY_LINKS="1" pipenv run pytest --junitxml=test-results.xml displayName: Run integration tests diff --git a/.azure-pipelines/jobs/run-vendor-scripts.yml b/.azure-pipelines/jobs/run-vendor-scripts.yml new file mode 100644 index 00000000..7fe75731 --- /dev/null +++ b/.azure-pipelines/jobs/run-vendor-scripts.yml @@ -0,0 +1,39 @@ +parameters: + vmImage: + +jobs: +- job: Vendor_Scripts + displayName: Test Vendor Scripts + pool: + vmImage: ${{ parameters.vmImage }} + strategy: + maxParallel: 4 + matrix: + ${{ if eq(parameters.vmImage, 'vs2017-win2016') }}: + # TODO remove once vs2017-win2016 has Python 3.7 + Python37: + python.version: '>= 3.7.0-b2' + python.architecture: x64 + ${{ if ne(parameters.vmImage, 'vs2017-win2016' )}}: + Python37: + python.version: '>= 3.7' + python.architecture: x64 + steps: + - task: UsePythonVersion@0 + displayName: Use Python $(python.version) + inputs: + versionSpec: '$(python.version)' + architecture: '$(python.architecture)' + + - template: ../steps/install-dependencies.yml + + - bash: | + mkdir -p "$AGENT_HOMEDIRECTORY/.virtualenvs" + mkdir -p "$WORKON_HOME" + pip install certifi + export GIT_SSL_CAINFO=$(python -m certifi) + export LANG=C.UTF-8 + python -m pip install --upgrade invoke requests parver + python -m invoke vendoring.update + + - template: ./run-manifest-check.yml diff --git a/.vsts-ci/phases/test.yml b/.azure-pipelines/jobs/test.yml similarity index 51% rename from .vsts-ci/phases/test.yml rename to .azure-pipelines/jobs/test.yml index b0de1e23..150937ef 100644 --- a/.vsts-ci/phases/test.yml +++ b/.azure-pipelines/jobs/test.yml @@ -1,49 +1,48 @@ parameters: - queue: + vmImage: -phases: -- phase: Test_Primary +jobs: +- job: Test_Primary displayName: Test Primary - - queue: - name: ${{ parameters.queue }} - parallel: 4 + pool: + vmImage: ${{ parameters.vmImage }} + strategy: + maxParallel: 4 matrix: Python27: python.version: '2.7' python.architecture: x64 - ${{ if eq(parameters.queue, 'Hosted VS2017') }}: - # TODO remove once Hosted VS2017 has Python 3.7 + ${{ if eq(parameters.vmImage, 'vs2017-win2016') }}: + # TODO remove once vs2017-win2016 has Python 3.7 Python37: python.version: '>= 3.7.0-b2' python.architecture: x64 - ${{ if ne(parameters.queue, 'Hosted VS2017' )}}: + ${{ if ne(parameters.vmImage, 'vs2017-win2016' )}}: Python37: python.version: '>= 3.7' python.architecture: x64 steps: - - ${{ if eq(parameters.queue, 'Hosted VS2017') }}: + - ${{ if eq(parameters.vmImage, 'vs2017-win2016') }}: - template: ./run-tests-windows.yml - - ${{ if ne(parameters.queue, 'Hosted VS2017') }}: + - ${{ if ne(parameters.vmImage, 'vs2017-win2016') }}: - template: ./run-tests.yml -- phase: Test_Secondary +- job: Test_Secondary displayName: Test python3.6 # Run after Test_Primary so we don't devour time and jobs if tests are going to fail -# dependsOn: Test_Primary - - queue: - name: ${{ parameters.queue }} - parallel: 4 + # dependsOn: Test_Primary + pool: + vmImage: ${{ parameters.vmImage }} + strategy: + maxParallel: 4 matrix: Python36: python.version: '3.6' python.architecture: x64 steps: - - ${{ if eq(parameters.queue, 'Hosted VS2017') }}: + - ${{ if eq(parameters.vmImage, 'vs2017-win2016') }}: - template: ./run-tests-windows.yml - - ${{ if ne(parameters.queue, 'Hosted VS2017') }}: + - ${{ if ne(parameters.vmImage, 'vs2017-win2016') }}: - template: ./run-tests.yml - diff --git a/.azure-pipelines/linux.yml b/.azure-pipelines/linux.yml new file mode 100644 index 00000000..a2f34d4c --- /dev/null +++ b/.azure-pipelines/linux.yml @@ -0,0 +1,27 @@ +name: Pipenv Build Rules +trigger: + batch: true + branches: + include: + - master + paths: + exclude: + - docs/* + - news/* + - README.md + - pipenv/*.txt + - CHANGELOG.rst + - CONTRIBUTING.md + - CODE_OF_CONDUCT.md + - .gitignore + - .gitattributes + - .editorconfig + +jobs: +- template: jobs/test.yml + parameters: + vmImage: ubuntu-16.04 + +- template: jobs/run-vendor-scripts.yml + parameters: + vmImage: ubuntu-16.04 diff --git a/.azure-pipelines/steps/create-virtualenv.yml b/.azure-pipelines/steps/create-virtualenv.yml new file mode 100644 index 00000000..9d1a6903 --- /dev/null +++ b/.azure-pipelines/steps/create-virtualenv.yml @@ -0,0 +1,6 @@ +steps: +- script: | + virtualenv D:\.venv + D:\.venv\Scripts\pip.exe install -e . && D:\.venv\Scripts\pipenv install --dev + echo D:\.venv\Scripts\pipenv --venv && echo D:\.venv\Scripts\pipenv --py && echo D:\.venv\Scripts\pipenv run python --version + displayName: Make Virtualenv diff --git a/.vsts-ci/steps/install-dependencies.yml b/.azure-pipelines/steps/install-dependencies.yml similarity index 100% rename from .vsts-ci/steps/install-dependencies.yml rename to .azure-pipelines/steps/install-dependencies.yml diff --git a/.vsts-ci/steps/run-tests.yml b/.azure-pipelines/steps/run-tests.yml similarity index 50% rename from .vsts-ci/steps/run-tests.yml rename to .azure-pipelines/steps/run-tests.yml index cb24b092..7496c76f 100644 --- a/.vsts-ci/steps/run-tests.yml +++ b/.azure-pipelines/steps/run-tests.yml @@ -3,13 +3,16 @@ steps: # Fix Git SSL errors pip install certifi python -m certifi > cacert.txt - $env:GIT_SSL_CAINFO = $(Get-Content cacert.txt) + Write-Host "##vso[task.setvariable variable=GIT_SSL_CAINFO]$(Get-Content cacert.txt)" + $env:GIT_SSL_CAINFO="$(Get-Content cacert.txt)" # Shorten paths to get under MAX_PATH or else integration tests will fail # https://bugs.python.org/issue18199 - subst T: $env:TEMP - $env:TEMP = "T:\" - $env:TMP = "T:\" - D:\.venv\Scripts\pipenv run pytest -n 4 --ignore=pipenv\patched --ignore=pipenv\vendor --junitxml=test-results.xml tests + subst T: "$env:TEMP" + Write-Host "##vso[task.setvariable variable=TEMP]T:\" + $env:TEMP='T:\' + Write-Host "##vso[task.setvariable variable=TMP]T:\" + $env:TMP='T:\' + D:\.venv\Scripts\pipenv run pytest -ra --ignore=pipenv\patched --ignore=pipenv\vendor --junitxml=test-results.xml tests displayName: Run integration tests - task: PublishTestResults@2 diff --git a/.azure-pipelines/windows.yml b/.azure-pipelines/windows.yml new file mode 100644 index 00000000..35d5c16e --- /dev/null +++ b/.azure-pipelines/windows.yml @@ -0,0 +1,23 @@ +name: Pipenv Build Rules +trigger: + batch: true + branches: + include: + - master + paths: + exclude: + - docs/* + - news/* + - README.md + - pipenv/*.txt + - CHANGELOG.rst + - CONTRIBUTING.md + - CODE_OF_CONDUCT.md + - .gitignore + - .gitattributes + - .editorconfig + +jobs: +- template: jobs/test.yml + parameters: + vmImage: vs2017-win2016 diff --git a/.gitignore b/.gitignore index 2cee5455..766ffe3a 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,9 @@ .AppleDouble .LSOverride +# PyCharm +.idea/ + # VSCode .vscode @@ -130,6 +133,9 @@ venv.bak/ .spyderproject .spyproject +# PyCharm project settings +.idea + # Rope project settings .ropeproject diff --git a/.vsts-ci/linux.yml b/.vsts-ci/linux.yml deleted file mode 100644 index 7968801c..00000000 --- a/.vsts-ci/linux.yml +++ /dev/null @@ -1,8 +0,0 @@ -phases: -- template: phases/test.yml - parameters: - queue: Hosted Linux Preview - -- template: phases/run-vendor-scripts.yml - parameters: - queue: Hosted Linux Preview diff --git a/.vsts-ci/phases/run-vendor-scripts.yml b/.vsts-ci/phases/run-vendor-scripts.yml deleted file mode 100644 index b96f64f5..00000000 --- a/.vsts-ci/phases/run-vendor-scripts.yml +++ /dev/null @@ -1,39 +0,0 @@ -parameters: - queue: - -phases: -- phase: Vendor_Scripts - displayName: Test Vendor Scripts - - queue: - name: ${{ parameters.queue }} - parallel: 4 - matrix: - ${{ if eq(parameters.queue, 'Hosted VS2017') }}: - # TODO remove once Hosted VS2017 has Python 3.7 - Python37: - python.version: '>= 3.7.0-b2' - python.architecture: x64 - ${{ if ne(parameters.queue, 'Hosted VS2017' )}}: - Python37: - python.version: '>= 3.7' - python.architecture: x64 - steps: - - task: UsePythonVersion@0 - displayName: Use Python $(python.version) - inputs: - versionSpec: '$(python.version)' - architecture: '$(python.architecture)' - - - template: ../steps/install-dependencies.yml - - - bash: | - mkdir -p "$AGENT_HOMEDIRECTORY/.virtualenvs" - mkdir -p "$WORKON_HOME" - pip install certifi - export GIT_SSL_CAINFO=$(python -m certifi) - export LANG=C.UTF-8 - python -m pip install --upgrade invoke requests parver - python -m invoke vendoring.update - - - template: ./run-manifest-check.yml diff --git a/.vsts-ci/steps/create-virtualenv.yml b/.vsts-ci/steps/create-virtualenv.yml deleted file mode 100644 index a72e9964..00000000 --- a/.vsts-ci/steps/create-virtualenv.yml +++ /dev/null @@ -1,6 +0,0 @@ -steps: -- script: | - virtualenv D:\.venv - D:\.venv\Scripts\pip.exe install -e . && D:\.venv\Scripts\pipenv install --dev - echo D:\.venv\Scripts\pipenv --venv && echo D:\.venv\Scripts\pipenv --py && echo D:\.venv\Scripts\pipenv run python --version - displayName: Make Virtualenv diff --git a/.vsts-ci/windows.yml b/.vsts-ci/windows.yml deleted file mode 100644 index a397a23c..00000000 --- a/.vsts-ci/windows.yml +++ /dev/null @@ -1,4 +0,0 @@ -phases: -- template: phases/test.yml - parameters: - queue: Hosted VS2017 diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ec8eaef1..45494be7 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,3 +1,89 @@ +2018.11.14 (2018-11-14) +======================= + +Features & Improvements +----------------------- + +- Improved exceptions and error handling on failures. `#1977 `_ +- Added persistent settings for all CLI flags via ``PIPENV_{FLAG_NAME}`` environment variables by enabling ``auto_envvar_prefix=PIPENV`` in click (implements PEEP-0002). `#2200 `_ +- Added improved messaging about available but skipped updates due to dependency conflicts when running ``pipenv update --outdated``. `#2411 `_ +- Added environment variable `PIPENV_PYUP_API_KEY` to add ability + to override the bundled pyup.io API key. `#2825 `_ +- Added additional output to ``pipenv update --outdated`` to indicate that the operation succeded and all packages were already up to date. `#2828 `_ +- Updated ``crayons`` patch to enable colors on native powershell but swap native blue for magenta. `#3020 `_ +- Added support for ``--bare`` to ``pipenv clean``, and fixed ``pipenv sync --bare`` to actually reduce output. `#3041 `_ +- Added windows-compatible spinner via upgraded ``vistir`` dependency. `#3089 `_ +- - Added support for python installations managed by ``asdf``. `#3096 `_ +- Improved runtime performance of no-op commands such as ``pipenv --venv`` by around 2/3. `#3158 `_ +- Do not show error but success for running ``pipenv uninstall --all`` in a fresh virtual environment. `#3170 `_ +- Improved asynchronous installation and error handling via queued subprocess paralleization. `#3217 `_ + +Bug Fixes +--------- + +- Remote non-PyPI artifacts and local wheels and artifacts will now include their own hashes rather than including hashes from ``PyPI``. `#2394 `_ +- Non-ascii characters will now be handled correctly when parsed by pipenv's ``ToML`` parsers. `#2737 `_ +- Updated ``pipenv uninstall`` to respect the ``--skip-lock`` argument. `#2848 `_ +- Fixed a bug which caused uninstallation to sometimes fail to successfullly remove packages from ``Pipfiles`` with comments on preceding or following lines. `#2885 `_, + `#3099 `_ +- Pipenv will no longer fail when encountering python versions on Windows that have been uninstalled. `#2983 `_ +- Fixed unnecessary extras are added when translating markers `#3026 `_ +- Fixed a virtualenv creation issue which could cause new virtualenvs to inadvertently attempt to read and write to global site packages. `#3047 `_ +- Fixed an issue with virtualenv path derivation which could cause errors, particularly for users on WSL bash. `#3055 `_ +- Fixed a bug which caused ``Unexpected EOF`` errors to be thrown when ``pip`` was waiting for input from users who had put login credentials in environment variables. `#3088 `_ +- Fixed a bug in ``requirementslib`` which prevented successful installation from mercurial repositories. `#3090 `_ +- Fixed random resource warnings when using pyenv or any other subprocess calls. `#3094 `_ +- - Fixed a bug which sometimes prevented cloning and parsing ``mercurial`` requirements. `#3096 `_ +- Fixed an issue in ``delegator.py`` related to subprocess calls when using ``PopenSpawn`` to stream output, which sometimes threw unexpected ``EOF`` errors. `#3102 `_, + `#3114 `_, + `#3117 `_ +- Fix the path casing issue that makes `pipenv clean` fail on Windows `#3104 `_ +- Pipenv will avoid leaving build artifacts in the current working directory. `#3106 `_ +- Fixed issues with broken subprocess calls leaking resource handles and causing random and sporadic failures. `#3109 `_ +- Fixed an issue which caused ``pipenv clean`` to sometimes clean packages from the base ``site-packages`` folder or fail entirely. `#3113 `_ +- Updated ``pythonfinder`` to correct an issue with unnesting of nested paths when searching for python versions. `#3121 `_ +- Added additional logic for ignoring and replacing non-ascii characters when formatting console output on non-UTF-8 systems. `#3131 `_ +- Fix virtual environment discovery when `PIPENV_VENV_IN_PROJECT` is set, but the in-project `.venv` is a file. `#3134 `_ +- Hashes for remote and local non-PyPI artifacts will now be included in ``Pipfile.lock`` during resolution. `#3145 `_ +- Fix project path hashing logic in purpose to prevent collisions of virtual environments. `#3151 `_ +- Fix package installation when the virtual environment path contains parentheses. `#3158 `_ +- Azure Pipelines YAML files are updated to use the latest syntax and product name. `#3164 `_ +- Fixed new spinner success message to write only one success message during resolution. `#3183 `_ +- Pipenv will now correctly respect the ``--pre`` option when used with ``pipenv install``. `#3185 `_ +- Fix a bug where exception is raised when run pipenv graph in a project without created virtualenv `#3201 `_ +- When sources are missing names, names will now be derived from the supplied URL. `#3216 `_ + +Vendored Libraries +------------------ + +- Updated ``pythonfinder`` to correct an issue with unnesting of nested paths when searching for python versions. `#3061 `_, + `#3121 `_ +- Updated vendored dependencies: + - ``certifi 2018.08.24 => 2018.10.15`` + - ``urllib3 1.23 => 1.24`` + - ``requests 2.19.1 => 2.20.0`` + - ``shellingham ``1.2.6 => 1.2.7`` + - ``tomlkit 0.4.4. => 0.4.6`` + - ``vistir 0.1.6 => 0.1.8`` + - ``pythonfinder 0.1.2 => 0.1.3`` + - ``requirementslib 1.1.9 => 1.1.10`` + - ``backports.functools_lru_cache 1.5.0 (new)`` + - ``cursor 1.2.0 (new)`` `#3089 `_ +- Updated vendored dependencies: + - ``requests 2.19.1 => 2.20.1`` + - ``tomlkit 0.4.46 => 0.5.2`` + - ``vistir 0.1.6 => 0.2.4`` + - ``pythonfinder 1.1.2 => 1.1.8`` + - ``requirementslib 1.1.10 => 1.3.0`` `#3096 `_ +- Switch to ``tomlkit`` for parsing and writing. Drop ``prettytoml`` and ``contoml`` from vendors. `#3191 `_ +- Updated ``requirementslib`` to aid in resolution of local and remote archives. `#3196 `_ + +Improved Documentation +---------------------- + +- Expanded development and testing documentation for contributors to get started. `#3074 `_ + + 2018.10.13 (2018-10-13) ======================= diff --git a/MANIFEST.in b/MANIFEST.in index a8d08c6c..3c8eb1d4 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -28,6 +28,7 @@ recursive-exclude pipenv *.pyi recursive-exclude pipenv *.typed prune peeps +prune .azure-pipelines prune .buildkite prune .github prune .vsts-ci diff --git a/Pipfile.lock b/Pipfile.lock index 3990a451..f5e02ec8 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -83,10 +83,10 @@ }, "bleach": { "hashes": [ - "sha256:9c471c0dd9c820f6bf4ee5ca3e348ceccefbc1475d9a40c397ed5d04e0b42c54", - "sha256:b407b2612b37e6cdc6704f84cec18c1f140b78e6c625652a844e89d6b9855f6b" + "sha256:48d39675b80a75f6d1c3bdbffec791cf0bbbab665cf01e20da701c77de278718", + "sha256:73d26f018af5d5adcdabf5c1c974add4361a9c76af215fe32fdec8a6fc5fb9b9" ], - "version": "==3.0.0" + "version": "==3.0.2" }, "cerberus": { "hashes": [ @@ -96,10 +96,10 @@ }, "certifi": { "hashes": [ - "sha256:376690d6f16d32f9d1fe8932551d80b23e9d393a8578c5633a2ed39a64861638", - "sha256:456048c7e371c089d0a77a5212fb37a2c2dce1e24146e3b7e0261736aaeaa22a" + "sha256:339dc09518b07e2fa7eda5450740925974815557727d6bd35d319c1524a04a4c", + "sha256:6d58c986d22b038c8c0df30d639f23a3e6d172a05c3583e766f4c0b785c0986a" ], - "version": "==2018.8.24" + "version": "==2018.10.15" }, "cffi": { "hashes": [ @@ -188,11 +188,10 @@ }, "colorama": { "hashes": [ - "sha256:463f8483208e921368c9f306094eb6f725c6ca42b0f97e313cb5d5512459feda", - "sha256:48eb22f4f8461b1df5734a074b57042430fb06e1d61bd1e11b078c0fe6d7a1f1" + "sha256:a3d89af5db9e9806a779a50296b5fdb466e281147c2c235e8225ecc6dbf7bbf3", + "sha256:c9b54bebe91a6a803e0772c8561d53f2926bfeb17cd141fbabcb08424086595c" ], - "markers": "sys_platform == 'win32'", - "version": "==0.3.9" + "version": "==0.4.0" }, "configparser": { "hashes": [ @@ -201,6 +200,12 @@ "markers": "python_version < '3.2'", "version": "==3.5.0" }, + "cursor": { + "hashes": [ + "sha256:8ee9fe5b925e1001f6ae6c017e93682583d2b4d1ef7130a26cfcdf1651c0032c" + ], + "version": "==1.2.0" + }, "distlib": { "hashes": [ "sha256:57977cd7d9ea27986ec62f425630e4ddb42efe651ff80bc58ed8dbc3c7c21f19" @@ -270,6 +275,12 @@ "markers": "python_version < '3.3'", "version": "==1.0.2" }, + "future": { + "hashes": [ + "sha256:e39ced1ab767b5936646cedba8bcce582398233d6a627067d4c6a454c90cfedb" + ], + "version": "==0.16.0" + }, "futures": { "hashes": [ "sha256:9ec02aa7d674acb8618afb127e27fde7fc68994c0437ad759fa094a574adb265", @@ -278,12 +289,6 @@ "markers": "python_version < '3' and python_version >= '2.6'", "version": "==3.2.0" }, - "future": { - "hashes": [ - "sha256:e39ced1ab767b5936646cedba8bcce582398233d6a627067d4c6a454c90cfedb" - ], - "version": "==0.16.0" - }, "idna": { "hashes": [ "sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e", @@ -325,9 +330,10 @@ }, "itsdangerous": { "hashes": [ - "sha256:cbb3fcf8d3e33df861709ecaf89d9e6629cff0a217bc2848f1b41cd30d360519" + "sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19", + "sha256:b12271b2047cb23eeb98c8b5622e2e5c5e9abd9784a153e9d8ef9cb4dd09d749" ], - "version": "==0.24" + "version": "==1.1.0" }, "jedi": { "hashes": [ @@ -346,9 +352,36 @@ }, "markupsafe": { "hashes": [ - "sha256:a6be69091dac236ea9c6bc7d012beab42010fa914c459791d627dad4910eb665" + "sha256:048ef924c1623740e70204aa7143ec592504045ae4429b59c30054cb31e3c432", + "sha256:130f844e7f5bdd8e9f3f42e7102ef1d49b2e6fdf0d7526df3f87281a532d8c8b", + "sha256:19f637c2ac5ae9da8bfd98cef74d64b7e1bb8a63038a3505cd182c3fac5eb4d9", + "sha256:1b8a7a87ad1b92bd887568ce54b23565f3fd7018c4180136e1cf412b405a47af", + "sha256:1c25694ca680b6919de53a4bb3bdd0602beafc63ff001fea2f2fc16ec3a11834", + "sha256:1f19ef5d3908110e1e891deefb5586aae1b49a7440db952454b4e281b41620cd", + "sha256:1fa6058938190ebe8290e5cae6c351e14e7bb44505c4a7624555ce57fbbeba0d", + "sha256:31cbb1359e8c25f9f48e156e59e2eaad51cd5242c05ed18a8de6dbe85184e4b7", + "sha256:3e835d8841ae7863f64e40e19477f7eb398674da6a47f09871673742531e6f4b", + "sha256:4e97332c9ce444b0c2c38dd22ddc61c743eb208d916e4265a2a3b575bdccb1d3", + "sha256:525396ee324ee2da82919f2ee9c9e73b012f23e7640131dd1b53a90206a0f09c", + "sha256:52b07fbc32032c21ad4ab060fec137b76eb804c4b9a1c7c7dc562549306afad2", + "sha256:52ccb45e77a1085ec5461cde794e1aa037df79f473cbc69b974e73940655c8d7", + "sha256:5c3fbebd7de20ce93103cb3183b47671f2885307df4a17a0ad56a1dd51273d36", + "sha256:5e5851969aea17660e55f6a3be00037a25b96a9b44d2083651812c99d53b14d1", + "sha256:5edfa27b2d3eefa2210fb2f5d539fbed81722b49f083b2c6566455eb7422fd7e", + "sha256:7d263e5770efddf465a9e31b78362d84d015cc894ca2c131901a4445eaa61ee1", + "sha256:83381342bfc22b3c8c06f2dd93a505413888694302de25add756254beee8449c", + "sha256:857eebb2c1dc60e4219ec8e98dfa19553dae33608237e107db9c6078b1167856", + "sha256:98e439297f78fca3a6169fd330fbe88d78b3bb72f967ad9961bcac0d7fdd1550", + "sha256:bf54103892a83c64db58125b3f2a43df6d2cb2d28889f14c78519394feb41492", + "sha256:d9ac82be533394d341b41d78aca7ed0e0f4ba5a2231602e2f05aa87f25c51672", + "sha256:e982fe07ede9fada6ff6705af70514a52beb1b2c3d25d4e873e82114cf3c5401", + "sha256:edce2ea7f3dfc981c4ddc97add8a61381d9642dc3273737e756517cc03e84dd6", + "sha256:efdc45ef1afc238db84cb4963aa689c0408912a0239b0721cb172b4016eb31d6", + "sha256:f137c02498f8b935892d5c0172560d7ab54bc45039de8805075e19079c639a9c", + "sha256:f82e347a72f955b7017a39708a3667f106e6ad4d10b25f237396a7115d8ed5fd", + "sha256:fb7c206e01ad85ce57feeaaa0bf784b97fa3cad0d4a5737bc5295785f5c613a1" ], - "version": "==1.0" + "version": "==1.1.0" }, "mccabe": { "hashes": [ @@ -402,17 +435,24 @@ }, "pbr": { "hashes": [ - "sha256:1be135151a0da949af8c5d0ee9013d9eafada71237eb80b3ba8896b4f12ec5dc", - "sha256:cf36765bf2218654ae824ec8e14257259ba44e43b117fd573c8d07a9895adbdd" + "sha256:f59d71442f9ece3dffc17bc36575768e1ee9967756e6b6535f0ee1f0054c3d68", + "sha256:f6d5b23f226a2ba58e14e49aa3b1bfaf814d0199144b95d78458212444de1387" ], - "version": "==4.3.0" + "version": "==5.1.1" + }, + "pep517": { + "hashes": [ + "sha256:cc663a438fdfe2e88d8d3c5ef2203ac858de34e31b6609b1fc505d611490a926", + "sha256:f79bb08fb064dfc5b141204bfeb56a4141a6d504677fab4723036a464fc25cc1" + ], + "version": "==0.3" }, "pip-shims": { "hashes": [ - "sha256:164b93bc94b207613d9632f28f4d55eba9301f9454aaaba335de36c24d92d106", - "sha256:27e2439aa93af8c1b8e58cf63a40cbcd26959b26424904f2e6d57837af8f76c5" + "sha256:3bc24ec050a6b9eea35419467237e4f47eaf806dadc9999bf887355c377edea7", + "sha256:edb4cf3c509eab2f36b55c1ac1a59a4c485ccd537cc87934d74950880f641256" ], - "version": "==0.3.0" + "version": "==0.3.2" }, "pipenv": { "editable": true, @@ -426,6 +466,9 @@ "version": "==1.4.2" }, "plette": { + "extras": [ + "validation" + ], "hashes": [ "sha256:c0e3553c1e581d8423daccbd825789c6e7f29b7d9e00e5331b12e1642a1a26d3", "sha256:dde5d525cf5f0cbad4d938c83b93db17887918daf63c13eafed257c4f61b07b4" @@ -434,24 +477,24 @@ }, "pluggy": { "hashes": [ - "sha256:6e3836e39f4d36ae72840833db137f7b7d35105079aee6ec4a62d9f80d594dd1", - "sha256:95eb8364a4708392bae89035f45341871286a333f749c3141c20573d2b3876e1" + "sha256:447ba94990e8014ee25ec853339faf7b0fc8050cdc3289d4d71f7f410fb90095", + "sha256:bde19360a8ec4dfd8a20dcb811780a30998101f078fc7ded6162f0076f50508f" ], - "version": "==0.7.1" + "version": "==0.8.0" }, "py": { "hashes": [ - "sha256:06a30435d058473046be836d3fc4f27167fd84c45b99704f2fb5509ef61f9af1", - "sha256:50402e9d1c9005d759426988a492e0edaadb7f4e68bcddfea586bc7432d009c6" + "sha256:bf92637198836372b520efcba9e020c330123be8ce527e535d185ed4b6f45694", + "sha256:e76826342cefe3c3d5f7e8ee4316b80d1dd8a300781612ddbc765c17ba25a6c6" ], - "version": "==1.6.0" + "version": "==1.7.0" }, "pycodestyle": { "hashes": [ - "sha256:682256a5b318149ca0d2a9185d365d8864a768a28db66a84a2ea946bcc426766", - "sha256:6c4245ade1edfad79c3446fadfc96b0de2759662dc29d07d80a6f27ad1ca6ba9" + "sha256:cbc619d09254895b0d12c2c691e237b2e91e9b2ecf5e84c26b35400f93dcfb83", + "sha256:cbfca99bd594a10f674d0cd97a3d802a1fdef635d4361e1a2658de47ed261e3a" ], - "version": "==2.3.1" + "version": "==2.4.0" }, "pycparser": { "hashes": [ @@ -461,10 +504,10 @@ }, "pyflakes": { "hashes": [ - "sha256:08bd6a50edf8cffa9fa09a463063c425ecaaf10d1eb0335a7e8b1401aef89e6f", - "sha256:8d616a382f243dbf19b54743f280b80198be0bca3a5396f1d2e1fca6223e8805" + "sha256:9a7662ec724d0120012f6e29d6248ae3727d821bba522a0e6b356eff19126a49", + "sha256:f661252913bc1dbe7fcfcbf0af0db3f42ab65aabd1a6ca68fe5d466bace94dae" ], - "version": "==1.6.0" + "version": "==2.0.0" }, "pygments": { "hashes": [ @@ -475,10 +518,10 @@ }, "pyparsing": { "hashes": [ - "sha256:bc6c7146b91af3f567cf6daeaec360bc07d45ffec4cf5353f4d7a208ce7ca30a", - "sha256:d29593d8ebe7b57d6967b62494f8c72b03ac0262b1eed63826c6f788b3606401" + "sha256:40856e74d4987de5d01761a22d1621ae1c7f8774585acae358aa5c5936c6c90b", + "sha256:f353aab21fd474459d97b709e527b5571314ee5f067441dc9f88e33eecd96592" ], - "version": "==2.2.2" + "version": "==2.3.0" }, "pytest": { "hashes": [ @@ -515,6 +558,12 @@ "index": "pypi", "version": "==1.23.2" }, + "pytoml": { + "hashes": [ + "sha256:ca2d0cb127c938b8b76a9a0d0f855cf930c1d50cc3a0af6d3595b566519a1013" + ], + "version": "==0.1.20" + }, "pytz": { "hashes": [ "sha256:a061aa0a9e06881eb8b3b2b43f05b9439d6583c206d0a6c340ff72a7b6669053", @@ -525,17 +574,17 @@ }, "readme-renderer": { "hashes": [ - "sha256:237ca8705ffea849870de41101dba41543561da05c0ae45b2f1c547efa9843d2", - "sha256:f75049a3a7afa57165551e030dd8f9882ebf688b9600535a3f7e23596651875d" + "sha256:bb16f55b259f27f75f640acf5e00cf897845a8b3e4731b5c1a436e4b8529202f", + "sha256:c8532b79afc0375a85f10433eca157d6b50f7d6990f337fa498c96cd4bfc203d" ], - "version": "==22.0" + "version": "==24.0" }, "requests": { "hashes": [ - "sha256:63b52e3c866428a224f97cab011de738c36aec0185aa91cfacd418b5d58911d1", - "sha256:ec22d826a36ed72a7358ff3fe56cbd4ba69dd7a6718ffd450ff0e9df7a47ce6a" + "sha256:65b3a120e4329e33c9889db89c80976c5272f56ea92d3e74da8a463992e3ff54", + "sha256:ea881206e59f41dbd0bd445437d792e43906703fff75ca8ff43ccdb11f33f263" ], - "version": "==2.19.1" + "version": "==2.20.1" }, "requests-toolbelt": { "hashes": [ @@ -546,10 +595,10 @@ }, "requirementslib": { "hashes": [ - "sha256:39fb4aab3ebd7f46b266ddc98a3ac731127ee35fe6cf1b3e11be7c6551cc2c9b", - "sha256:810d8961f333d8fef92400f58b25f80003151fb424a545e244073fc3d95ae2dd" + "sha256:441a5bfa487d3f3f5fd5d81c27071d9fd36bb385f538b3a87d20556a80b76f76", + "sha256:89e1e02ff0b52ce9c610124eb990ae706e0aee08beef8c718e7b87e470cdceeb" ], - "version": "==1.1.7" + "version": "==1.3.1.post1" }, "resolvelib": { "hashes": [ @@ -626,10 +675,10 @@ }, "tomlkit": { "hashes": [ - "sha256:8ab16e93162fc44d3ad83d2aa29a7140b8f7d996ae1790a73b9a7aed6fb504ac", - "sha256:ca181cee7aee805d455628f7c94eb8ae814763769a93e69157f250fe4ebe1926" + "sha256:82a8fbb8d8c6af72e96ba00b9db3e20ef61be6c79082552c9363f4559702258b", + "sha256:a43e0195edc9b3c198cd4b5f0f3d427a395d47c4a76ceba7cc875ed030756c39" ], - "version": "==0.4.4" + "version": "==0.5.2" }, "towncrier": { "editable": true, @@ -638,10 +687,10 @@ }, "tqdm": { "hashes": [ - "sha256:18f1818ce951aeb9ea162ae1098b43f583f7d057b34d706f66939353d1208889", - "sha256:df02c0650160986bac0218bb07952245fc6960d23654648b5d5526ad5a4128c9" + "sha256:3c4d4a5a41ef162dd61f1edb86b0e1c7859054ab656b2e7c7b77e7fbf6d9f392", + "sha256:5b4d5549984503050883bc126280b386f5f4ca87e6c023c5d015655ad75bdebb" ], - "version": "==4.26.0" + "version": "==4.28.1" }, "twine": { "hashes": [ @@ -662,36 +711,37 @@ }, "urllib3": { "hashes": [ - "sha256:a68ac5e15e76e7e5dd2b8f94007233e01effe3e50e8daddf69acfd81cb686baf", - "sha256:b5725a0bd4ba422ab0e66e89e030c806576753ea3ee08554382c14e685d117b5" + "sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39", + "sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22" ], - "version": "==1.23" + "version": "==1.24.1" }, "virtualenv": { "hashes": [ - "sha256:2ce32cd126117ce2c539f0134eb89de91a8413a29baac49cbab3eb50e2026669", - "sha256:ca07b4c0b54e14a91af9f34d0919790b016923d157afda5efdde55c96718f752" + "sha256:686176c23a538ecc56d27ed9d5217abd34644823d6391cbeb232f42bf722baad", + "sha256:f899fafcd92e1150f40c8215328be38ff24b519cd95357fa6e78e006c7638208" ], - "version": "==16.0.0" + "version": "==16.1.0" }, "virtualenv-clone": { "hashes": [ - "sha256:4507071d81013fd03ea9930ec26bc8648b997927a11fa80e8ee81198b57e0ac7", - "sha256:b5cfe535d14dc68dfc1d1bb4ac1209ea28235b91156e2bba8e250d291c3fb4f8" + "sha256:afce268508aa5596c90dda234abe345deebc401a57d287bcbd76baa140a1aa58" ], - "version": "==0.3.0" + "version": "==0.4.0" }, "vistir": { - "hashes": [ - "sha256:8a360ac20cbcc0863d6dbbe7a52e8b2c9ebf48abd6833c3813a82c70708244af", - "sha256:bc6e10284792485c10585536e6aede9e38996c841cc9d2a67238cd05742c2d0b" + "extras": [ + "spinner" ], - "version": "==0.1.6" + "hashes": [ + "sha256:851bd783f2b85a372e563db741dc689cb9263ce2e067e387facdca0c36b6a6ea", + "sha256:b38ffc8ef83f85d81b4efa4cd31ea3bcd37bdb2bc9e8da9f20a40859bc44b57e" + ], + "version": "==0.2.4" }, "webencodings": { "hashes": [ - "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78", - "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923" + "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78" ], "version": "==0.5.1" }, @@ -704,10 +754,10 @@ }, "wheel": { "hashes": [ - "sha256:9fa1f772f1a2df2bd00ddb4fa57e1cc349301e1facb98fbe62329803a9ff1196", - "sha256:d215f4520a1ba1851a3c00ba2b4122665cd3d6b0834d2ba2816198b1e3024a0e" + "sha256:196c9842d79262bb66fcf59faa4bd0deb27da911dbc7c6cdca931080eb1f0783", + "sha256:c93e2d711f5f9841e17f53b0e6c0ff85593f3b416b6eec7a9452041a59a42688" ], - "version": "==0.32.1" + "version": "==0.32.2" }, "yaspin": { "hashes": [ diff --git a/docs/advanced.rst b/docs/advanced.rst index 6981abf5..236f8c12 100644 --- a/docs/advanced.rst +++ b/docs/advanced.rst @@ -240,6 +240,9 @@ Example:: API access throttling based on overall usage rather than individual client usage. + You can also use your own safety API key by setting the + environment variable ``PIPENV_PYUP_API_KEY``. + ☤ Community Integrations ------------------------ @@ -350,12 +353,12 @@ To prevent pipenv from loading the ``.env`` file, set the ``PIPENV_DONT_LOAD_ENV ☤ Custom Script Shortcuts ------------------------- -Pipenv supports creating custom shortcuts in the (optional) ``[scripts]`` section of your Pipfile. +Pipenv supports creating custom shortcuts in the (optional) ``[scripts]`` section of your Pipfile. You can then run ``pipenv run `` in your terminal to run the command in the -context of your pipenv virtual environment even if you have not activated the pipenv shell first. +context of your pipenv virtual environment even if you have not activated the pipenv shell first. -For example, in your Pipfile:: +For example, in your Pipfile:: [scripts] printspam = "python -c \"print('I am a silly example, no one would need to do this')\"" @@ -394,6 +397,7 @@ Pipenv supports the usage of environment variables in values. For example:: Environment variables may be specified as ``${MY_ENVAR}`` or ``$MY_ENVAR``. On Windows, ``%MY_ENVAR%`` is supported in addition to ``${MY_ENVAR}`` or ``$MY_ENVAR``. +.. _configuration-with-environment-variables: ☤ Configuration With Environment Variables ------------------------------------------ diff --git a/docs/basics.rst b/docs/basics.rst index ee2e6621..4ba6116d 100644 --- a/docs/basics.rst +++ b/docs/basics.rst @@ -182,12 +182,12 @@ in your ``Pipfile.lock`` for now, run ``pipenv lock --keep-outdated``. Make sur ☤ Specifying Versions of a Package ---------------------------------- -You can specify versions of a package using the `Semantic Versioning scheme `_ -(i.e. ``major.minor.micro``). +You can specify versions of a package using the `Semantic Versioning scheme `_ +(i.e. ``major.minor.micro``). For example, to install requests you can use: :: - $ pipenv install requests~=1.2 # equivalent to requests~=1.2.0 + $ pipenv install requests~=1.2 # equivalent to requests~=1.2.0 Pipenv will install version ``1.2`` and any minor update, but not ``2.0``. @@ -201,11 +201,11 @@ To make inclusive or exclusive version comparisons you can use: :: $ pipenv install "requests>=1.4" # will install a version equal or larger than 1.4.0 $ pipenv install "requests<=2.13" # will install a version equal or lower than 2.13.0 - $ pipenv install "requests>2.19" # will install 2.19.1 but not 2.19.0 + $ pipenv install "requests>2.19" # will install 2.19.1 but not 2.19.0 -.. note:: The use of ``" "`` around the package and version specification is highly recommended +.. note:: The use of ``" "`` around the package and version specification is highly recommended to avoid issues with `Input and output redirection `_ - in Unix-based operating systems. + in Unix-based operating systems. The use of ``~=`` is preferred over the ``==`` identifier as the former prevents pipenv from updating the packages: :: @@ -367,18 +367,18 @@ The only optional section is the ``@`` section. When using git o Note that it is **strongly recommended** that you install any version-controlled dependencies in editable mode, using ``pipenv install -e``, in order to ensure that dependency resolution can be performed with an up to date copy of the repository each time it is performed, and that it includes all known dependencies. -Below is an example usage which installs the git repository located at ``https://github.com/requests/requests.git`` from tag ``v2.19.1`` as package name ``requests``:: +Below is an example usage which installs the git repository located at ``https://github.com/requests/requests.git`` from tag ``v2.20.1`` as package name ``requests``:: - $ pipenv install -e git+https://github.com/requests/requests.git@v2.19#egg=requests + $ pipenv install -e git+https://github.com/requests/requests.git@v2.20.1#egg=requests Creating a Pipfile for this project... - Installing -e git+https://github.com/requests/requests.git@v2.19.1#egg=requests... + Installing -e git+https://github.com/requests/requests.git@v2.20.1#egg=requests... [...snipped...] - Adding -e git+https://github.com/requests/requests.git@v2.19.1#egg=requests to Pipfile's [packages]... + Adding -e git+https://github.com/requests/requests.git@v2.20.1#egg=requests to Pipfile's [packages]... [...] $ cat Pipfile [packages] - requests = {git = "https://github.com/requests/requests.git", editable = true, ref = "2.19.1"} + requests = {git = "https://github.com/requests/requests.git", editable = true, ref = "v2.20.1"} Valid values for ```` include ``git``, ``bzr``, ``svn``, and ``hg``. Valid values for ```` include ``http``, ``https``, ``ssh``, and ``file``. In specific cases you also have access to other schemes: ``svn`` may be combined with ``svn`` as a scheme, and ``bzr`` can be combined with ``sftp`` and ``lp``. @@ -398,10 +398,8 @@ environment into production. You can use ``pipenv lock`` to compile your depende your development environment and deploy the compiled ``Pipfile.lock`` to all of your production environments for reproducible builds. -.. note: +.. note:: If you'd like a ``requirements.txt`` output of the lockfile, run ``$ pipenv lock -r``. This will include all hashes, however (which is great!). To get a ``requirements.txt`` without hashes, use ``$ pipenv run pip freeze``. - -.. _configuration-with-environment-variables:https://docs.pipenv.org/advanced/#configuration-with-environment-variables diff --git a/docs/dev/contributing.rst b/docs/dev/contributing.rst index 0a8e2c9f..f7895600 100644 --- a/docs/dev/contributing.rst +++ b/docs/dev/contributing.rst @@ -235,3 +235,38 @@ be aware of the following things when filing bug reports: If you do not provide all of these things, it will take us much longer to fix your problem. If we ask you to clarify these and you never respond, we will close your issue without fixing it. + +Run the tests +------------- + +Three ways of running the tests are as follows: + +1. ``make test`` (which uses ``docker``) +2. ``./run-tests.sh`` or ``run-tests.bat`` +3. Using pipenv:: + + pipenv install --dev + pipenv run pytest + +For the last two, it is important that your environment is setup correctly, and +this may take some work, for example, on a specific Mac installation, the following +steps may be needed:: + + # Make sure the tests can access github + if [ "$SSH_AGENT_PID" = "" ] + then + eval `ssh-agent` + ssh-add + fi + + # Use unix like utilities, installed with brew, + # e.g. brew install coreutils + for d in /usr/local/opt/*/libexec/gnubin /usr/local/opt/python/libexec/bin + do + [[ ":$PATH:" != *":$d:"* ]] && PATH="$d:${PATH}" + done + + export PATH + + # PIP_FIND_LINKS currently breaks test_uninstall.py + unset PIP_FIND_LINKS diff --git a/docs/requirements.txt b/docs/requirements.txt index 71c58644..d35a9f32 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -15,7 +15,7 @@ pip-tools==1.9.0 Pygments==2.2.0 pythonz-bd==1.11.4 pytz==2017.2 -requests==2.12.0 +requests==2.20.0 resumable-urlretrieve==0.1.5 semver==2.7.8 six==1.10.0 diff --git a/news/3020.feature b/news/3020.feature deleted file mode 100644 index aebfab57..00000000 --- a/news/3020.feature +++ /dev/null @@ -1 +0,0 @@ -Updated ``crayons`` patch to enable colors on native powershell but swap native blue for magenta. diff --git a/news/3026.bugfix b/news/3026.bugfix deleted file mode 100644 index 6a194d1f..00000000 --- a/news/3026.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fixed unnecessary extras are added when translating markers diff --git a/news/3069.trivial b/news/3069.trivial deleted file mode 100644 index 733a3cac..00000000 --- a/news/3069.trivial +++ /dev/null @@ -1 +0,0 @@ -- Fix PIPENV_VERBOSITY parsing logic diff --git a/news/3222.bugfix.rst b/news/3222.bugfix.rst new file mode 100644 index 00000000..2f71f552 --- /dev/null +++ b/news/3222.bugfix.rst @@ -0,0 +1 @@ +Pipenv will no longer disable user-mode installation when the ``--system`` flag is passed in. diff --git a/news/3223.bugfix.rst b/news/3223.bugfix.rst new file mode 100644 index 00000000..44f4a83e --- /dev/null +++ b/news/3223.bugfix.rst @@ -0,0 +1 @@ +Fixed an issue with attempting to render unicode output in non-unicode locales. diff --git a/news/3224.bugfix.rst b/news/3224.bugfix.rst new file mode 100644 index 00000000..e6eed912 --- /dev/null +++ b/news/3224.bugfix.rst @@ -0,0 +1 @@ +Fixed a bug which could cause failures to occur when parsing python entries from global pyenv version files. diff --git a/news/towncrier_template.rst b/news/towncrier_template.rst index 37d89a60..55505f82 100644 --- a/news/towncrier_template.rst +++ b/news/towncrier_template.rst @@ -13,9 +13,7 @@ {% if definitions[category]['showcontent'] %} {% for text, values in sections[section][category]|dictsort(by='value') %} -- {{ text }}{% if category != 'process' %} - {{ values|sort|join(',\n ') }} - {% endif %} +- {{ text }}{% if category != 'process' %}{{ values|sort|join(',\n ') }}{% endif %} {% endfor %} {% else %} diff --git a/pipenv/__init__.py b/pipenv/__init__.py index 471dcb94..4d137e7f 100644 --- a/pipenv/__init__.py +++ b/pipenv/__init__.py @@ -1,25 +1,49 @@ +# -*- coding=utf-8 -*- # |~~\' |~~ # |__/||~~\|--|/~\\ / # | ||__/|__| |\/ # | + import os import sys +import warnings + from .__version__ import __version__ - -PIPENV_ROOT = os.path.dirname(os.path.realpath(__file__)) +PIPENV_ROOT = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) PIPENV_VENDOR = os.sep.join([PIPENV_ROOT, "vendor"]) PIPENV_PATCHED = os.sep.join([PIPENV_ROOT, "patched"]) # Inject vendored directory into system path. sys.path.insert(0, PIPENV_VENDOR) # Inject patched directory into system path. sys.path.insert(0, PIPENV_PATCHED) + +from pipenv.vendor.urllib3.exceptions import DependencyWarning +from pipenv.vendor.vistir.compat import ResourceWarning, fs_str +warnings.filterwarnings("ignore", category=DependencyWarning) +warnings.filterwarnings("ignore", category=ResourceWarning) +warnings.filterwarnings("ignore", category=UserWarning) + +if sys.version_info >= (3, 1) and sys.version_info <= (3, 6): + if sys.stdout.isatty() and sys.stderr.isatty(): + import io + import atexit + stdout_wrapper = io.TextIOWrapper(sys.stdout.buffer, encoding='utf8') + atexit.register(stdout_wrapper.close) + stderr_wrapper = io.TextIOWrapper(sys.stderr.buffer, encoding='utf8') + atexit.register(stderr_wrapper.close) + sys.stdout = stdout_wrapper + sys.stderr = stderr_wrapper + +os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = fs_str("1") + # Hack to make things work better. try: if "concurrency" in sys.modules: del sys.modules["concurrency"] except Exception: pass + from .cli import cli from . import resolver diff --git a/pipenv/__main__.py b/pipenv/__main__.py index 98dcca0c..56494106 100644 --- a/pipenv/__main__.py +++ b/pipenv/__main__.py @@ -1,4 +1,4 @@ from .cli import cli if __name__ == "__main__": - cli() + cli(auto_envvar_prefix="PIPENV") diff --git a/pipenv/__version__.py b/pipenv/__version__.py index b2df6ad1..802c3b9c 100644 --- a/pipenv/__version__.py +++ b/pipenv/__version__.py @@ -2,4 +2,4 @@ # // ) ) / / // ) ) //___) ) // ) ) || / / # //___/ / / / //___/ / // // / / || / / # // / / // ((____ // / / ||/ / -__version__ = "2018.10.14.dev0" +__version__ = "2018.11.15.dev0" diff --git a/pipenv/_compat.py b/pipenv/_compat.py index 558df3b8..09f65669 100644 --- a/pipenv/_compat.py +++ b/pipenv/_compat.py @@ -4,6 +4,7 @@ Exposes a standard API that enables compatibility across python versions, operating systems, etc. """ + import functools import importlib import io @@ -11,8 +12,8 @@ import os import six import sys import warnings +import vistir from tempfile import _bin_openflags, gettempdir, _mkstemp_inner, mkdtemp -from .utils import logging, rmtree try: from tempfile import _infer_return_type @@ -53,33 +54,23 @@ except ImportError: class finalize(object): def __init__(self, *args, **kwargs): + from .utils import logging logging.warn("weakref.finalize unavailable, not cleaning...") def detach(self): return False -if six.PY2: - - class ResourceWarning(Warning): - pass +from vistir.compat import ResourceWarning -def pip_import(module_path, subimport=None, old_path=None): - internal = "pip._internal.{0}".format(module_path) - old_path = old_path or module_path - pip9 = "pip.{0}".format(old_path) - try: - _tmp = importlib.import_module(internal) - except ImportError: - _tmp = importlib.import_module(pip9) - if subimport: - return getattr(_tmp, subimport, _tmp) - return _tmp +warnings.filterwarnings("ignore", category=ResourceWarning) class TemporaryDirectory(object): - """Create and return a temporary directory. This has the same + + """ + Create and return a temporary directory. This has the same behavior as mkdtemp but can be used as a context manager. For example: @@ -109,7 +100,7 @@ class TemporaryDirectory(object): @classmethod def _cleanup(cls, name, warn_message): - rmtree(name) + vistir.path.rmtree(name) warnings.warn(warn_message, ResourceWarning) def __repr__(self): @@ -123,7 +114,7 @@ class TemporaryDirectory(object): def cleanup(self): if self._finalizer.detach(): - rmtree(self.name) + vistir.path.rmtree(self.name) def _sanitize_params(prefix, suffix, dir): @@ -145,9 +136,11 @@ def _sanitize_params(prefix, suffix, dir): class _TemporaryFileCloser: - """A separate object allowing proper closing of a temporary file's + """ + A separate object allowing proper closing of a temporary file's underlying file object, without adding a __del__ method to the - temporary file.""" + temporary file. + """ file = None # Set here since __del__ checks it close_called = False @@ -191,7 +184,9 @@ class _TemporaryFileCloser: class _TemporaryFileWrapper: - """Temporary file wrapper + + """ + Temporary file wrapper This class provides a wrapper around files opened for temporary use. In particular, it seeks to automatically remove the file when it is no longer needed. @@ -267,7 +262,8 @@ def NamedTemporaryFile( dir=None, delete=True, ): - """Create and return a temporary file. + """ + Create and return a temporary file. Arguments: 'prefix', 'suffix', 'dir' -- as for mkstemp. 'mode' -- the mode argument to io.open (default "w+b"). @@ -300,3 +296,104 @@ def NamedTemporaryFile( os.unlink(name) os.close(fd) raise + + +def getpreferredencoding(): + import locale + # Borrowed from Invoke + # (see https://github.com/pyinvoke/invoke/blob/93af29d/invoke/runners.py#L881) + _encoding = locale.getpreferredencoding(False) + if six.PY2 and not sys.platform == "win32": + _default_encoding = locale.getdefaultlocale()[1] + if _default_encoding is not None: + _encoding = _default_encoding + return _encoding + + +DEFAULT_ENCODING = getpreferredencoding() + + +# From https://github.com/CarlFK/veyepar/blob/5c5de47/dj/scripts/fixunicode.py +# MIT LIcensed, thanks Carl! +def force_encoding(): + try: + stdout_isatty = sys.stdout.isatty + stderr_isatty = sys.stderr.isatty + except AttributeError: + return DEFAULT_ENCODING, DEFAULT_ENCODING + else: + if not (stdout_isatty() and stderr_isatty()): + return DEFAULT_ENCODING, DEFAULT_ENCODING + stdout_encoding = sys.stdout.encoding + stderr_encoding = sys.stderr.encoding + if sys.platform == "win32" and sys.version_info >= (3, 1): + return DEFAULT_ENCODING, DEFAULT_ENCODING + if stdout_encoding.lower() != "utf-8" or stderr_encoding.lower() != "utf-8": + + from ctypes import pythonapi, py_object, c_char_p + try: + PyFile_SetEncoding = pythonapi.PyFile_SetEncoding + except AttributeError: + return DEFAULT_ENCODING, DEFAULT_ENCODING + else: + PyFile_SetEncoding.argtypes = (py_object, c_char_p) + if stdout_encoding.lower() != "utf-8": + try: + was_set = PyFile_SetEncoding(sys.stdout, "utf-8") + except OSError: + was_set = False + if not was_set: + stdout_encoding = DEFAULT_ENCODING + else: + stdout_encoding = "utf-8" + + if stderr_encoding.lower() != "utf-8": + try: + was_set = PyFile_SetEncoding(sys.stderr, "utf-8") + except OSError: + was_set = False + if not was_set: + stderr_encoding = DEFAULT_ENCODING + else: + stderr_encoding = "utf-8" + + return stdout_encoding, stderr_encoding + + +OUT_ENCODING, ERR_ENCODING = force_encoding() + + +UNICODE_TO_ASCII_TRANSLATION_MAP = { + 8230: u"...", + 8211: u"-", + 10004: u"x", + 10008: u"Ok" +} + + +def decode_output(output): + if not isinstance(output, six.string_types): + return output + try: + output = output.encode(DEFAULT_ENCODING) + except (AttributeError, UnicodeDecodeError, UnicodeEncodeError): + if six.PY2: + output = unicode.translate(vistir.misc.to_text(output), + UNICODE_TO_ASCII_TRANSLATION_MAP) + else: + output = output.translate(UNICODE_TO_ASCII_TRANSLATION_MAP) + output = output.encode(DEFAULT_ENCODING, "replace") + return vistir.misc.to_text(output, encoding=DEFAULT_ENCODING, errors="replace") + return output + + +def fix_utf8(text): + if not isinstance(text, six.string_types): + return text + from ._compat import decode_output + try: + text = decode_output(text) + except UnicodeDecodeError: + if six.PY2: + text = unicode.translate(vistir.misc.to_text(text), UNICODE_TO_ASCII_TRANSLATION_MAP) + return text diff --git a/pipenv/cli/command.py b/pipenv/cli/command.py index 1ce9fee9..85f0b65a 100644 --- a/pipenv/cli/command.py +++ b/pipenv/cli/command.py @@ -15,7 +15,6 @@ import click_completion from click_didyoumean import DYMCommandCollection -from .. import environments from ..__version__ import __version__ from .options import ( CONTEXT_SETTINGS, PipenvGroup, code_option, common_options, deploy_option, @@ -98,16 +97,16 @@ def cli( warn_in_virtualenv, do_where, project, - spinner, cleanup_virtualenv, ensure_project, format_help, do_clear, ) + from ..utils import create_spinner if man: if system_which("man"): - path = os.sep.join([os.path.dirname(__file__), "pipenv.1"]) + path = os.path.join(os.path.dirname(os.path.dirname(__file__)), "pipenv.1") os.execle(system_which("man"), "man", path, os.environ) return 0 else: @@ -115,6 +114,7 @@ def cli( return 1 if envs: echo("The following environment variables can be set, to do various things:\n") + from .. import environments for key in environments.__dict__: if key.startswith("PIPENV"): echo(" - {0}".format(crayons.normal(key, bold=True))) @@ -161,7 +161,8 @@ def cli( # --rm was passed… elif rm: # Abort if --system (or running in a virtualenv). - if environments.PIPENV_USE_SYSTEM: + from ..environments import PIPENV_USE_SYSTEM + if PIPENV_USE_SYSTEM: echo( crayons.red( "You are attempting to remove a virtualenv that " @@ -179,7 +180,7 @@ def cli( ) ) ) - with spinner(): + with create_spinner(text="Running..."): # Remove the virtualenv. cleanup_virtualenv(bare=True) return 0 @@ -253,7 +254,7 @@ def install( @cli.command(short_help="Un-installs a provided package and removes it from Pipfile.") -@option("--lock", is_flag=True, default=True, help="Lock afterwards.") +@option("--skip-lock/--lock", is_flag=True, default=False, help="Lock afterwards.") @option( "--all-dev", is_flag=True, @@ -268,27 +269,29 @@ def install( ) @uninstall_options @pass_state +@pass_context def uninstall( + ctx, state, - lock=False, + skip_lock=False, all_dev=False, all=False, **kwargs ): """Un-installs a provided package and removes it from Pipfile.""" from ..core import do_uninstall - retcode = do_uninstall( packages=state.installstate.packages, editable_packages=state.installstate.editables, three=state.three, python=state.python, system=state.system, - lock=lock, + lock=not state.installstate.skip_lock, all_dev=all_dev, all=all, keep_outdated=state.installstate.keep_outdated, pypi_mirror=state.pypi_mirror, + ctx=ctx ) if retcode: sys.exit(retcode) @@ -297,7 +300,9 @@ def uninstall( @cli.command(short_help="Generates Pipfile.lock.") @lock_options @pass_state +@pass_context def lock( + ctx, state, **kwargs ): @@ -307,9 +312,14 @@ def lock( # Ensure that virtualenv is available. ensure_project(three=state.three, python=state.python, pypi_mirror=state.pypi_mirror) if state.installstate.requirementstxt: - do_init(dev=state.installstate.dev, requirements=state.installstate.requirementstxt, - pypi_mirror=state.pypi_mirror, pre=state.installstate.pre) + do_init( + dev=state.installstate.dev, + requirements=state.installstate.requirementstxt, + pypi_mirror=state.pypi_mirror, + pre=state.installstate.pre, + ) do_lock( + ctx=ctx, clear=state.clear, pre=state.installstate.pre, keep_outdated=state.installstate.keep_outdated, @@ -533,12 +543,20 @@ def graph(bare=False, json=False, json_tree=False, reverse=False): @argument("module", nargs=1) @pass_state def run_open(state, module, *args, **kwargs): - """View a given module in your editor.""" + """View a given module in your editor. + + This uses the EDITOR environment variable. You can temporarily override it, + for example: + + EDITOR=atom pipenv open requests + """ from ..core import which, ensure_project # Ensure that virtualenv is available. - ensure_project(three=state.three, python=state.python, validate=False, - pypi_mirror=state.pypi_mirror) + ensure_project( + three=state.three, python=state.python, + validate=False, pypi_mirror=state.pypi_mirror, + ) c = delegator.run( '{0} -c "import {1}; print({1}.__file__);"'.format(which("python"), module) ) @@ -590,6 +608,7 @@ def sync( @cli.command(short_help="Uninstalls all packages not specified in Pipfile.lock.") +@option("--bare", is_flag=True, default=False, help="Minimal output.") @option("--dry-run", is_flag=True, default=False, help="Just output unneeded packages.") @verbose_option @three_option diff --git a/pipenv/cli/options.py b/pipenv/cli/options.py index 99fc5344..606454dd 100644 --- a/pipenv/cli/options.py +++ b/pipenv/cli/options.py @@ -3,7 +3,7 @@ from __future__ import absolute_import import os -from click import BOOL as click_booltype +import click.types from click import ( BadParameter, Group, Option, argument, echo, make_pass_decorator, option ) @@ -86,8 +86,8 @@ def index_option(f): state.index = value return value return option('-i', '--index', expose_value=False, envvar="PIP_INDEX_URL", - help='Target PyPI-compatible package index url.', nargs=1, - callback=callback)(f) + help='Target PyPI-compatible package index url.', nargs=1, + callback=callback)(f) def extra_index_option(f): @@ -96,8 +96,8 @@ def extra_index_option(f): state.extra_index_urls.extend(list(value)) return value return option("--extra-index-url", multiple=True, expose_value=False, - help=u"URLs to the extra PyPI compatible indexes to query for package lookups.", - callback=callback, envvar="PIP_EXTRA_INDEX_URL")(f) + help=u"URLs to the extra PyPI compatible indexes to query for package lookups.", + callback=callback, envvar="PIP_EXTRA_INDEX_URL")(f) def editable_option(f): @@ -106,8 +106,8 @@ def editable_option(f): state.installstate.editables.extend(value) return value return option('-e', '--editable', expose_value=False, multiple=True, - help='An editable python package URL or path, often to a VCS repo.', - callback=callback)(f) + help='An editable python package URL or path, often to a VCS repo.', + callback=callback, type=click.types.STRING)(f) def sequential_option(f): @@ -116,8 +116,8 @@ def sequential_option(f): state.installstate.sequential = value return value return option("--sequential", is_flag=True, default=False, expose_value=False, - help="Install dependencies one-at-a-time, instead of concurrently.", - callback=callback, type=click_booltype)(f) + help="Install dependencies one-at-a-time, instead of concurrently.", + callback=callback, type=click.types.BOOL)(f) def skip_lock_option(f): @@ -126,8 +126,8 @@ def skip_lock_option(f): state.installstate.skip_lock = value return value return option("--skip-lock", is_flag=True, default=False, expose_value=False, - help=u"Ignore locking mechanisms when installing—use the Pipfile, instead.", - callback=callback, type=click_booltype)(f) + help=u"Skip locking mechanisms and use the Pipfile instead during operation.", + envvar="PIPENV_SKIP_LOCK", callback=callback, type=click.types.BOOL)(f) def keep_outdated_option(f): @@ -136,8 +136,8 @@ def keep_outdated_option(f): state.installstate.keep_outdated = value return value return option("--keep-outdated", is_flag=True, default=False, expose_value=False, - help=u"Keep out-dated dependencies from being updated in Pipfile.lock.", - callback=callback, type=click_booltype)(f) + help=u"Keep out-dated dependencies from being updated in Pipfile.lock.", + callback=callback, type=click.types.BOOL)(f) def selective_upgrade_option(f): @@ -145,9 +145,9 @@ def selective_upgrade_option(f): state = ctx.ensure_object(State) state.installstate.selective_upgrade = value return value - return option("--selective-upgrade", is_flag=True, default=False, type=click_booltype, - help="Update specified packages.", callback=callback, - expose_value=False)(f) + return option("--selective-upgrade", is_flag=True, default=False, type=click.types.BOOL, + help="Update specified packages.", callback=callback, + expose_value=False)(f) def ignore_pipfile_option(f): @@ -156,8 +156,8 @@ def ignore_pipfile_option(f): state.installstate.ignore_pipfile = value return value return option("--ignore-pipfile", is_flag=True, default=False, expose_value=False, - help="Ignore Pipfile when installing, using the Pipfile.lock.", - callback=callback)(f) + help="Ignore Pipfile when installing, using the Pipfile.lock.", + callback=callback, type=click.types.BOOL)(f) def dev_option(f): @@ -165,9 +165,9 @@ def dev_option(f): state = ctx.ensure_object(State) state.installstate.dev = value return value - return option("--dev", "-d", is_flag=True, default=False, type=click_booltype, - help="Install both develop and default packages.", callback=callback, - expose_value=False)(f) + return option("--dev", "-d", is_flag=True, default=False, type=click.types.BOOL, + help="Install both develop and default packages.", callback=callback, + expose_value=False)(f) def pre_option(f): @@ -176,7 +176,7 @@ def pre_option(f): state.installstate.pre = value return value return option("--pre", is_flag=True, default=False, help=u"Allow pre-releases.", - callback=callback, type=click_booltype, expose_value=False)(f) + callback=callback, type=click.types.BOOL, expose_value=False)(f) def package_arg(f): @@ -184,7 +184,8 @@ def package_arg(f): state = ctx.ensure_object(State) state.installstate.packages.extend(value) return value - return argument('packages', nargs=-1, callback=callback, expose_value=False,)(f) + return argument('packages', nargs=-1, callback=callback, expose_value=False, + type=click.types.STRING)(f) def three_option(f): @@ -195,8 +196,8 @@ def three_option(f): state.two = not value return value return option("--three/--two", is_flag=True, default=None, - help="Use Python 3/2 when creating virtualenv.", callback=callback, - expose_value=False)(f) + help="Use Python 3/2 when creating virtualenv.", callback=callback, + expose_value=False)(f) def python_option(f): @@ -206,8 +207,8 @@ def python_option(f): state.python = validate_python_path(ctx, param, value) return value return option("--python", default=False, nargs=1, callback=callback, - help="Specify which version of Python virtualenv should use.", - expose_value=False)(f) + help="Specify which version of Python virtualenv should use.", + expose_value=False)(f) def pypi_mirror_option(f): @@ -217,7 +218,7 @@ def pypi_mirror_option(f): state.pypi_mirror = validate_pypi_mirror(ctx, param, value) return value return option("--pypi-mirror", default=environments.PIPENV_PYPI_MIRROR, nargs=1, - callback=callback, help="Specify a PyPI mirror.", expose_value=False)(f) + callback=callback, help="Specify a PyPI mirror.", expose_value=False)(f) def verbose_option(f): @@ -227,7 +228,7 @@ def verbose_option(f): state.verbose = True setup_verbosity(ctx, param, value) return option("--verbose", "-v", is_flag=True, expose_value=False, - callback=callback, help="Verbose mode.")(f) + callback=callback, help="Verbose mode.", type=click.types.BOOL)(f) def site_packages_option(f): @@ -235,9 +236,9 @@ def site_packages_option(f): state = ctx.ensure_object(State) state.site_packages = value return value - return option("--site-packages", is_flag=True, default=False, type=click_booltype, - help="Enable site-packages for the virtualenv.", callback=callback, - expose_value=False)(f) + return option("--site-packages", is_flag=True, default=False, type=click.types.BOOL, + help="Enable site-packages for the virtualenv.", callback=callback, + expose_value=False)(f) def clear_option(f): @@ -245,9 +246,9 @@ def clear_option(f): state = ctx.ensure_object(State) state.clear = value return value - return option("--clear", is_flag=True, callback=callback, type=click_booltype, - help="Clears caches (pipenv, pip, and pip-tools).", - expose_value=False)(f) + return option("--clear", is_flag=True, callback=callback, type=click.types.BOOL, + help="Clears caches (pipenv, pip, and pip-tools).", + expose_value=False)(f) def system_option(f): @@ -257,7 +258,7 @@ def system_option(f): state.system = value return value return option("--system", is_flag=True, default=False, help="System pip management.", - callback=callback, type=click_booltype, expose_value=False)(f) + callback=callback, type=click.types.BOOL, expose_value=False)(f) def requirementstxt_option(f): @@ -267,7 +268,7 @@ def requirementstxt_option(f): state.installstate.requirementstxt = value return value return option("--requirements", "-r", nargs=1, default=False, expose_value=False, - help="Import a requirements.txt file.", callback=callback)(f) + help="Import a requirements.txt file.", callback=callback)(f) def requirements_flag(f): @@ -277,7 +278,7 @@ def requirements_flag(f): state.installstate.requirementstxt = value return value return option("--requirements", "-r", default=False, is_flag=True, expose_value=False, - help="Generate output in requirements.txt format.", callback=callback)(f) + help="Generate output in requirements.txt format.", callback=callback)(f) def code_option(f): @@ -295,9 +296,9 @@ def deploy_option(f): state = ctx.ensure_object(State) state.installstate.deploy = value return value - return option("--deploy", is_flag=True, default=False, type=click_booltype, - help=u"Abort if the Pipfile.lock is out-of-date, or Python version is" - " wrong.", callback=callback, expose_value=False)(f) + return option("--deploy", is_flag=True, default=False, type=click.types.BOOL, + help=u"Abort if the Pipfile.lock is out-of-date, or Python version is" + " wrong.", callback=callback, expose_value=False)(f) def setup_verbosity(ctx, param, value): @@ -338,6 +339,7 @@ def common_options(f): def install_base_options(f): f = common_options(f) f = dev_option(f) + f = pre_option(f) f = keep_outdated_option(f) return f @@ -353,7 +355,6 @@ def uninstall_options(f): def lock_options(f): f = install_base_options(f) f = requirements_flag(f) - f = pre_option(f) return f diff --git a/pipenv/cmdparse.py b/pipenv/cmdparse.py index 087a95b1..cec19273 100644 --- a/pipenv/cmdparse.py +++ b/pipenv/cmdparse.py @@ -1,3 +1,4 @@ +import itertools import re import shlex @@ -8,6 +9,12 @@ class ScriptEmptyError(ValueError): pass +def _quote_if_contains(value, pattern): + if next(re.finditer(pattern, value), None): + return '"{0}"'.format(re.sub(r'(\\*)"', r'\1\1\\"', value)) + return value + + class Script(object): """Parse a script line (in Pipfile's [scripts] section). @@ -56,17 +63,21 @@ class Script(object): The result is then quoted into a pair of double quotes to be grouped. An argument is intentionally not quoted if it does not contain - whitespaces. This is done to be compatible with Windows built-in + foul characters. This is done to be compatible with Windows built-in commands that don't work well with quotes, e.g. everything with `echo`, and DOS-style (forward slash) switches. + Foul characters include: + + * Whitespaces. + * Parentheses in the command. (pypa/pipenv#3168) + The intended use of this function is to pre-process an argument list before passing it into ``subprocess.Popen(..., shell=True)``. See also: https://docs.python.org/3/library/subprocess.html#converting-argument-sequence """ - return " ".join( - arg if not next(re.finditer(r'\s', arg), None) - else '"{0}"'.format(re.sub(r'(\\*)"', r'\1\1\\"', arg)) - for arg in self._parts - ) + return " ".join(itertools.chain( + [_quote_if_contains(self.command, r'[\s()]')], + (_quote_if_contains(arg, r'\s') for arg in self.args), + )) diff --git a/pipenv/core.py b/pipenv/core.py index ca37bd5c..c4bb1c5e 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1,11 +1,10 @@ # -*- coding=utf-8 -*- -import contextlib + import logging import os import sys import shutil import time -import tempfile import json as simplejson import click import click_completion @@ -13,10 +12,12 @@ import crayons import dotenv import delegator import pipfile -from blindspin import spinner import vistir +import warnings import six +import urllib3.util as urllib3_util + from .cmdparse import Script from .project import Project, SourceNotFound from .utils import ( @@ -24,8 +25,6 @@ from .utils import ( is_required_version, proper_case, pep423_name, - split_file, - merge_deps, venv_resolve_deps, escape_grouped_arguments, python_version, @@ -37,17 +36,15 @@ from .utils import ( download_file, is_pinned, is_star, - rmtree, - clean_resolved_dep, parse_indexes, - escape_cmd + escape_cmd, + create_spinner, + get_canonical_names ) from . import environments, pep508checker, progress from .environments import ( PIPENV_COLORBLIND, - PIPENV_NOSPIN, PIPENV_SHELL_FANCY, - PIPENV_TIMEOUT, PIPENV_SKIP_VALIDATION, PIPENV_HIDE_EMOJIS, PIPENV_YES, @@ -56,7 +53,10 @@ from .environments import ( PIPENV_DONT_USE_PYENV, SESSION_IS_INTERACTIVE, PIPENV_CACHE_DIR, + PIPENV_PYUP_API_KEY, ) +from ._compat import fix_utf8 +from . import exceptions # Packages that should be ignored later. BAD_PACKAGES = ( @@ -96,12 +96,6 @@ click_completion.init() # Disable colors, for the color blind and others who do not prefer colors. if PIPENV_COLORBLIND: crayons.disable() -# Disable spinner, for cleaner build logs (the unworthy). -if PIPENV_NOSPIN: - - @contextlib.contextmanager # noqa: F811 - def spinner(): - yield def which(command, location=None, allow_global=False): @@ -132,15 +126,15 @@ project = Project(which=which) def do_clear(): - click.echo(crayons.white("Clearing caches…", bold=True)) + click.echo(crayons.white(fix_utf8("Clearing caches…"), bold=True)) try: from pip._internal import locations except ImportError: # pip 9. from pip import locations try: - shutil.rmtree(PIPENV_CACHE_DIR) - shutil.rmtree(locations.USER_CACHE_DIR) + vistir.path.rmtree(PIPENV_CACHE_DIR) + vistir.path.rmtree(locations.USER_CACHE_DIR) except OSError as e: # Ignore FileNotFoundError. This is needed for Python 2.7. import errno @@ -161,7 +155,7 @@ def load_dot_env(): if os.path.isfile(dotenv_file): click.echo( - crayons.normal("Loading .env environment variables…", bold=True), + crayons.normal(fix_utf8("Loading .env environment variables…"), bold=True), err=True, ) else: @@ -190,7 +184,7 @@ def cleanup_virtualenv(bare=True): click.echo(crayons.red("Environment creation aborted.")) try: # Delete the virtualenv. - rmtree(project.virtualenv_location) + vistir.path.rmtree(project.virtualenv_location) except OSError as e: click.echo( "{0} An error occurred while removing {1}!".format( @@ -204,7 +198,7 @@ def cleanup_virtualenv(bare=True): def import_requirements(r=None, dev=False): from .patched.notpip._vendor import requests as pip_requests - from .patched.notpip._internal.req.req_file import parse_requirements + from .vendor.pip_shims.shims import parse_requirements # Parse requirements.txt file with Pip's parser. # Pip requires a `PipSession` which is a subclass of requests.Session. @@ -283,27 +277,29 @@ def ensure_pipfile(validate=True, skip_requirements=False, system=False): if project.pipfile_is_empty: # Show an error message and exit if system is passed and no pipfile exists if system and not PIPENV_VIRTUALENV: - click.echo( - "{0}: --system is intended to be used for pre-existing Pipfile " - "installation, not installation of specific packages. Aborting.".format( - crayons.red("Warning", bold=True) - ), - err=True, + raise exceptions.PipenvOptionsError( + "--system", + "--system is intended to be used for pre-existing Pipfile " + "installation, not installation of specific packages. Aborting." ) - sys.exit(1) # If there's a requirements file, but no Pipfile… if project.requirements_exists and not skip_requirements: click.echo( crayons.normal( - u"requirements.txt found, instead of Pipfile! Converting…", + fix_utf8("requirements.txt found, instead of Pipfile! Converting…"), bold=True, ) ) # Create a Pipfile… project.create_pipfile(python=python) - with spinner(): + with create_spinner("Importing requirements...") as sp: # Import requirements.txt. - import_requirements() + try: + import_requirements() + except Exception: + sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format("Failed...")) + else: + sp.ok(environments.PIPENV_SPINNER_OK_TEXT.format("Success!")) # Warn the user of side-effects. click.echo( u"{0}: Your {1} now contains pinned versions, if your {2} did. \n" @@ -317,7 +313,7 @@ def ensure_pipfile(validate=True, skip_requirements=False, system=False): ) else: click.echo( - crayons.normal(u"Creating a Pipfile for this project…", bold=True), + crayons.normal(fix_utf8("Creating a Pipfile for this project…"), bold=True), err=True, ) # Create the pipfile if it doesn't exist. @@ -405,7 +401,7 @@ def ensure_python(three=None, python=None): u"{0}: Python {1} {2}".format( crayons.red("Warning", bold=True), crayons.blue(python), - u"was not found on your system…", + fix_utf8("was not found on your system…"), ), err=True, ) @@ -424,7 +420,7 @@ def ensure_python(three=None, python=None): except ValueError: abort() except PyenvError as e: - click.echo(u"Something went wrong…") + click.echo(fix_utf8("Something went wrong…")) click.echo(crayons.blue(e.err), err=True) abort() s = "{0} {1} {2}".format( @@ -443,17 +439,22 @@ def ensure_python(three=None, python=None): crayons.green(u"CPython {0}".format(version), bold=True), crayons.normal(u"with pyenv", bold=True), crayons.normal(u"(this may take a few minutes)"), - crayons.normal(u"…", bold=True), + crayons.normal(fix_utf8("…"), bold=True), ) ) - with spinner(): + with create_spinner("Installing python...") as sp: try: c = pyenv.install(version) except PyenvError as e: - click.echo(u"Something went wrong…") + sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format( + "Failed...") + ) + click.echo(fix_utf8("Something went wrong…"), err=True) click.echo(crayons.blue(e.err), err=True) - # Print the results, in a beautiful blue… - click.echo(crayons.blue(c.out), err=True) + else: + sp.ok(environments.PIPENV_SPINNER_OK_TEXT.format("Success!")) + # Print the results, in a beautiful blue… + click.echo(crayons.blue(c.out), err=True) # Find the newly installed Python, hopefully. version = str(version) path_to_python = find_a_system_python(version) @@ -518,7 +519,7 @@ def ensure_virtualenv(three=None, python=None, site_packages=False, pypi_mirror= ): abort() click.echo( - crayons.normal(u"Removing existing virtualenv…", bold=True), err=True + crayons.normal(fix_utf8("Removing existing virtualenv…"), bold=True), err=True ) # Remove the virtualenv. cleanup_virtualenv(bare=True) @@ -554,8 +555,11 @@ def ensure_project( # Automatically use an activated virtualenv. if PIPENV_USE_SYSTEM: system = True - if not project.pipfile_exists and not deploy: - project.touch_pipfile() + if not project.pipfile_exists: + if deploy is True: + raise exceptions.PipfileNotFound + else: + project.touch_pipfile() # Skip virtualenv creation when --system was used. if not system: ensure_virtualenv( @@ -594,8 +598,7 @@ def ensure_project( err=True, ) else: - click.echo(crayons.red("Deploy aborted."), err=True) - sys.exit(1) + raise exceptions.DeployException # Ensure the Pipfile exists. ensure_pipfile( validate=validate, skip_requirements=skip_requirements, system=system @@ -649,6 +652,103 @@ def do_where(virtualenv=False, bare=True): click.echo(location) +def _cleanup_procs(procs, concurrent, failed_deps_queue, retry=True): + while not procs.empty(): + c = procs.get() + if concurrent: + c.block() + failed = False + if c.return_code != 0: + failed = True + if "Ignoring" in c.out: + click.echo(crayons.yellow(c.out.strip())) + elif environments.is_verbose(): + click.echo(crayons.blue(c.out.strip() or c.err.strip())) + # The Installation failed… + if failed: + if not retry: + # The Installation failed… + # We echo both c.out and c.err because pip returns error details on out. + err = c.err.strip().splitlines() if c.err else [] + out = c.out.strip().splitlines() if c.out else [] + err_lines = [line for line in [out, err]] + # Return the subprocess' return code. + raise exceptions.InstallError(c.dep.name, extra=err_lines) + # Save the Failed Dependency for later. + dep = c.dep.copy() + failed_deps_queue.put(dep) + # Alert the user. + click.echo( + "{0} {1}! Will try again.".format( + crayons.red("An error occurred while installing"), + crayons.green(dep.as_line()), + ), err=True + ) + + +def batch_install(deps_list, procs, failed_deps_queue, + requirements_dir, no_deps=False, ignore_hashes=False, + allow_global=False, blocking=False, pypi_mirror=None, + nprocs=PIPENV_MAX_SUBPROCESS, retry=True): + + failed = (not retry) + if not failed: + label = INSTALL_LABEL if os.name != "nt" else "" + else: + label = INSTALL_LABEL2 + + deps_list_bar = progress.bar( + deps_list, width=32, + label=label + ) + indexes = [] + trusted_hosts = [] + # Install these because + for dep in deps_list_bar: + index = None + if dep.index: + index = project.find_source(dep.index) + indexes.append(index) + if not index.get("verify_ssl", False): + trusted_hosts.append(urllib3_util.parse_url(index.get("url")).host) + # Install the module. + is_artifact = False + if dep.is_file_or_url and (dep.is_direct_url or any( + dep.req.uri.endswith(ext) for ext in ["zip", "tar.gz"] + )): + is_artifact = True + + extra_indexes = [] + if not index and indexes: + index = next(iter(indexes)) + if len(indexes) > 1: + extra_indexes = indexes[1:] + + with vistir.contextmanagers.temp_environ(): + if not allow_global: + os.environ["PIP_USER"] = vistir.compat.fs_str("0") + c = pip_install( + dep, + ignore_hashes=any([ignore_hashes, dep.editable, dep.is_vcs]), + allow_global=allow_global, + no_deps=False if is_artifact else no_deps, + block=any([dep.editable, dep.is_vcs, blocking]), + index=index, + requirements_dir=requirements_dir, + pypi_mirror=pypi_mirror, + trusted_hosts=trusted_hosts, + extra_indexes=extra_indexes + ) + if dep.is_vcs: + c.block() + if procs.qsize() < nprocs: + c.dep = dep + procs.put(c) + + if procs.full() or procs.qsize() == len(deps_list): + _cleanup_procs(procs, not blocking, failed_deps_queue, retry=retry) + + def do_install_dependencies( dev=False, only=False, @@ -661,32 +761,13 @@ def do_install_dependencies( requirements_dir=None, pypi_mirror=False, ): - """"Executes the install functionality. + """" + Executes the install functionality. If requirements is True, simply spits out a requirements format to stdout. """ - from .vendor.requirementslib.models.requirements import Requirement - - def cleanup_procs(procs, concurrent): - for c in procs: - if concurrent: - c.block() - if "Ignoring" in c.out: - click.echo(crayons.yellow(c.out.strip())) - elif environments.is_verbose(): - click.echo(crayons.blue(c.out or c.err)) - # The Installation failed… - if c.return_code != 0: - # Save the Failed Dependency for later. - failed_deps_list.append((c.dep, c.ignore_hash)) - # Alert the user. - click.echo( - "{0} {1}! Will try again.".format( - crayons.red("An error occurred while installing"), - crayons.green(c.dep.as_line()), - ) - ) + from six.moves import queue if requirements: bare = True blocking = not concurrent @@ -694,143 +775,74 @@ def do_install_dependencies( if skip_lock or only or not project.lockfile_exists: if not bare: click.echo( - crayons.normal(u"Installing dependencies from Pipfile…", bold=True) + crayons.normal(fix_utf8("Installing dependencies from Pipfile…"), bold=True) ) - lockfile = split_file(project._lockfile) + lockfile = project.get_or_create_lockfile() else: - with open(project.lockfile_location) as f: - lockfile = split_file(simplejson.load(f)) + lockfile = project.get_or_create_lockfile() if not bare: click.echo( crayons.normal( - u"Installing dependencies from Pipfile.lock ({0})…".format( + fix_utf8("Installing dependencies from Pipfile.lock ({0})…".format( lockfile["_meta"].get("hash", {}).get("sha256")[-6:] - ), + )), bold=True, ) ) # Allow pip to resolve dependencies when in skip-lock mode. no_deps = not skip_lock - deps_list, dev_deps_list = merge_deps( - lockfile, - project, - dev=dev, - requirements=requirements, - ignore_hashes=ignore_hashes, - blocking=blocking, - only=only, - ) - failed_deps_list = [] + deps_list = list(lockfile.get_requirements(dev=dev, only=requirements)) if requirements: index_args = prepare_pip_source_args(project.sources) index_args = " ".join(index_args).replace(" -", "\n-") - deps_list = [dep for dep, ignore_hash, block in deps_list] - dev_deps_list = [dep for dep, ignore_hash, block in dev_deps_list] + deps = [ + req.as_line(sources=False, include_hashes=False) for req in deps_list + ] # Output only default dependencies click.echo(index_args) - if not dev: - click.echo( - "\n".join(d.partition("--hash")[0].strip() for d in sorted(deps_list)) - ) - sys.exit(0) - # Output only dev dependencies - if dev: - click.echo( - "\n".join( - d.partition("--hash")[0].strip() for d in sorted(dev_deps_list) - ) - ) - sys.exit(0) - procs = [] - deps_list_bar = progress.bar( - deps_list, label=INSTALL_LABEL if os.name != "nt" else "" - ) - for dep, ignore_hash, block in deps_list_bar: - if len(procs) < PIPENV_MAX_SUBPROCESS: - # Use a specific index, if specified. - indexes, trusted_hosts, dep = parse_indexes(dep) - index = None - extra_indexes = [] - if indexes: - index = indexes[0] - if len(indexes) > 0: - extra_indexes = indexes[1:] - dep = Requirement.from_line(" ".join(dep)) - if index: - _index = None - try: - _index = project.find_source(index).get("name") - except SourceNotFound: - _index = None - dep.index = _index - dep._index = index - dep.extra_indexes = extra_indexes - # Install the module. - prev_no_deps_setting = no_deps - if dep.is_file_or_url and any( - dep.req.uri.endswith(ext) for ext in ["zip", "tar.gz"] - ): - no_deps = False - c = pip_install( - dep, - ignore_hashes=ignore_hash, - allow_global=allow_global, - no_deps=no_deps, - block=block, - index=index, - requirements_dir=requirements_dir, - extra_indexes=extra_indexes, - pypi_mirror=pypi_mirror, - trusted_hosts=trusted_hosts - ) - c.dep = dep - c.ignore_hash = ignore_hash - c.index = index - c.extra_indexes = extra_indexes - procs.append(c) - no_deps = prev_no_deps_setting - if len(procs) >= PIPENV_MAX_SUBPROCESS or len(procs) == len(deps_list): - cleanup_procs(procs, concurrent) - procs = [] - cleanup_procs(procs, concurrent) - # Iterate over the hopefully-poorly-packaged dependencies… - if failed_deps_list: click.echo( - crayons.normal(u"Installing initially failed dependencies…", bold=True) + "\n".join(sorted(deps)) ) - for dep, ignore_hash in progress.bar(failed_deps_list, label=INSTALL_LABEL2): - # Use a specific index, if specified. - # Install the module. - prev_no_deps_setting = no_deps - if dep.is_file_or_url and any( - dep.req.uri.endswith(ext) for ext in ["zip", "tar.gz"] - ): - no_deps = False - c = pip_install( - dep, - ignore_hashes=ignore_hash, - allow_global=allow_global, - no_deps=no_deps, - index=getattr(dep, "_index", None), - requirements_dir=requirements_dir, - extra_indexes=getattr(dep, "extra_indexes", None), - ) - no_deps = prev_no_deps_setting - # The Installation failed… - if c.return_code != 0: - # We echo both c.out and c.err because pip returns error details on out. - click.echo(crayons.blue(format_pip_output(c.out))) - click.echo(crayons.blue(format_pip_error(c.err)), err=True) - # Return the subprocess' return code. - sys.exit(c.return_code) - else: - click.echo( - "{0} {1}{2}".format( - crayons.green("Success installing"), - crayons.green(dep.name), - crayons.green("!"), - ) - ) + sys.exit(0) + + procs = queue.Queue(maxsize=PIPENV_MAX_SUBPROCESS) + failed_deps_queue = queue.Queue() + + install_kwargs = { + "no_deps": no_deps, "ignore_hashes": ignore_hashes, "allow_global": allow_global, + "blocking": blocking, "pypi_mirror": pypi_mirror + } + if concurrent: + install_kwargs["nprocs"] = PIPENV_MAX_SUBPROCESS + else: + install_kwargs["nprocs"] = 1 + + batch_install( + deps_list, procs, failed_deps_queue, requirements_dir, **install_kwargs + ) + + if not procs.empty(): + _cleanup_procs(procs, concurrent, failed_deps_queue) + + # Iterate over the hopefully-poorly-packaged dependencies… + if not failed_deps_queue.empty(): + click.echo( + crayons.normal(fix_utf8("Installing initially failed dependencies…"), bold=True) + ) + retry_list = [] + while not failed_deps_queue.empty(): + failed_dep = failed_deps_queue.get() + retry_list.append(failed_dep) + install_kwargs.update({ + "nprocs": 1, + "retry": False, + "blocking": True, + }) + batch_install( + retry_list, procs, failed_deps_queue, requirements_dir, **install_kwargs + ) + if not procs.empty(): + _cleanup_procs(procs, False, failed_deps_queue, retry=False) def convert_three_to_python(three, python): @@ -851,7 +863,7 @@ def convert_three_to_python(three, python): def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None): """Creates a virtualenv.""" click.echo( - crayons.normal(u"Creating a virtualenv for this project…", bold=True), err=True + crayons.normal(fix_utf8("Creating a virtualenv for this project…"), bold=True), err=True ) click.echo( u"Pipfile: {0}".format(crayons.red(project.pipfile_location, bold=True)), @@ -865,14 +877,14 @@ def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None): u"{0} {1} {3} {2}".format( crayons.normal("Using", bold=True), crayons.red(python, bold=True), - crayons.normal(u"to create virtualenv…", bold=True), + crayons.normal(fix_utf8("to create virtualenv…"), bold=True), crayons.green("({0})".format(python_version(python))), ), err=True, ) cmd = [ - sys.executable, + vistir.compat.Path(sys.executable).absolute().as_posix(), "-m", "virtualenv", "--prompt=({0}) ".format(project.name), @@ -883,7 +895,7 @@ def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None): # Pass site-packages flag to virtualenv, if desired… if site_packages: click.echo( - crayons.normal(u"Making site-packages available…", bold=True), err=True + crayons.normal(fix_utf8("Making site-packages available…"), bold=True), err=True ) cmd.append("--system-site-packages") @@ -893,26 +905,33 @@ def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None): pip_config = {} # Actually create the virtualenv. - with spinner(): - c = delegator.run(cmd, block=False, timeout=PIPENV_TIMEOUT, env=pip_config) - c.block() + nospin = environments.PIPENV_NOSPIN + c = vistir.misc.run( + cmd, verbose=False, return_object=True, + spinner_name=environments.PIPENV_SPINNER, combine_stderr=False, + block=False, nospin=nospin, env=pip_config, + ) click.echo(crayons.blue("{0}".format(c.out)), err=True) - if c.return_code != 0: - click.echo(crayons.blue("{0}".format(c.err)), err=True) - click.echo( - u"{0}: Failed to create virtual environment.".format( - crayons.red("Warning", bold=True) - ), - err=True, + if c.returncode != 0: + raise exceptions.VirtualenvCreationException( + extra=[crayons.blue("{0}".format(c.err)),] ) - sys.exit(1) # Associate project directory with the environment. # This mimics Pew's "setproject". project_file_name = os.path.join(project.virtualenv_location, ".project") with open(project_file_name, "w") as f: f.write(vistir.misc.fs_str(project.project_directory)) - + from .environment import Environment + sources = project.pipfile_sources + project._environment = Environment( + prefix=project.get_location_for_virtualenv(), + is_venv=True, + sources=sources, + pipfile=project.parsed_pipfile, + project=project + ) + project._environment.add_dist("pipenv") # Say where the virtualenv is. do_where(virtualenv=True, bare=False) @@ -956,7 +975,16 @@ def get_downloads_info(names_map, section): return info +def overwrite_dev(prod, dev): + dev_keys = set(list(dev.keys())) + prod_keys = set(list(prod.keys())) + for pkg in dev_keys & prod_keys: + dev[pkg] = prod[pkg] + return dev + + def do_lock( + ctx=None, system=False, clear=False, pre=False, @@ -965,19 +993,16 @@ def do_lock( pypi_mirror=None, ): """Executes the freeze functionality.""" - from .utils import get_vcs_deps cached_lockfile = {} if not pre: pre = project.settings.get("allow_prereleases") if keep_outdated: if not project.lockfile_exists: - click.echo( - "{0}: Pipfile.lock must exist to use --keep-outdated!".format( - crayons.red("Warning", bold=True) - ) + raise exceptions.PipenvOptionsError( + "--keep-outdated", ctx=ctx, + message="Pipfile.lock must exist to use --keep-outdated!" ) - sys.exit(1) cached_lockfile = project.lockfile_content # Create the lockfile. lockfile = project._lockfile @@ -988,120 +1013,57 @@ def do_lock( del lockfile[section][k] # Ensure that develop inherits from default. dev_packages = project.dev_packages.copy() - for dev_package in project.dev_packages: - if dev_package in project.packages: - dev_packages[dev_package] = project.packages[dev_package] + dev_packages = overwrite_dev(project.packages, dev_packages) # Resolve dev-package dependencies, with pip-tools. - pip_freeze = delegator.run( - "{0} freeze".format(escape_grouped_arguments(which_pip(allow_global=system))) - ).out - sections = { - "dev": { - "packages": project.dev_packages, - "vcs": project.vcs_dev_packages, - "pipfile_key": "dev_packages", - "lockfile_key": "develop", - "log_string": "dev-packages", - "dev": True, - }, - "default": { - "packages": project.packages, - "vcs": project.vcs_packages, - "pipfile_key": "packages", - "lockfile_key": "default", - "log_string": "packages", - "dev": False, - }, - } - for section_name in ["dev", "default"]: - settings = sections[section_name] + for is_dev in [True, False]: + pipfile_section = "dev_packages" if is_dev else "packages" + lockfile_section = "develop" if is_dev else "default" + packages = getattr(project, pipfile_section) + if write: # Alert the user of progress. click.echo( u"{0} {1} {2}".format( - crayons.normal("Locking"), - crayons.red("[{0}]".format(settings["log_string"])), - crayons.normal("dependencies…"), + crayons.normal(u"Locking"), + crayons.red(u"[{0}]".format(pipfile_section.replace("_", "-"))), + crayons.normal(fix_utf8("dependencies…")), ), err=True, ) deps = convert_deps_to_pip( - settings["packages"], project, r=False, include_index=True + packages, project, r=False, include_index=True ) - results = venv_resolve_deps( + # Mutates the lockfile + venv_resolve_deps( deps, which=which, project=project, + dev=is_dev, clear=clear, pre=pre, allow_global=system, pypi_mirror=pypi_mirror, + pipfile=packages, + lockfile=lockfile ) - # Add dependencies to lockfile. - for dep in results: - is_top_level = dep["name"] in settings["packages"] - pipfile_entry = settings["packages"][dep["name"]] if is_top_level else None - dep_lockfile = clean_resolved_dep( - dep, is_top_level=is_top_level, pipfile_entry=pipfile_entry - ) - lockfile[settings["lockfile_key"]].update(dep_lockfile) - # Add refs for VCS installs. - # TODO: be smarter about this. - vcs_reqs, vcs_lockfile = get_vcs_deps( - project, - pip_freeze, - which=which, - clear=clear, - pre=pre, - allow_global=system, - dev=settings["dev"], - ) - vcs_lines = [req.as_line() for req in vcs_reqs if req.editable] - vcs_results = venv_resolve_deps( - vcs_lines, - which=which, - project=project, - clear=clear, - pre=pre, - allow_global=system, - pypi_mirror=pypi_mirror, - ) - for dep in vcs_results: - normalized = pep423_name(dep["name"]) - if not hasattr(dep, "keys") or not hasattr(dep["name"], "keys"): - continue - is_top_level = dep["name"] in vcs_lockfile or normalized in vcs_lockfile - if is_top_level: - try: - pipfile_entry = vcs_lockfile[dep["name"]] - except KeyError: - pipfile_entry = vcs_lockfile[normalized] - else: - pipfile_entry = None - dep_lockfile = clean_resolved_dep( - dep, is_top_level=is_top_level, pipfile_entry=pipfile_entry - ) - vcs_lockfile.update(dep_lockfile) - lockfile[settings["lockfile_key"]].update(vcs_lockfile) # Support for --keep-outdated… if keep_outdated: + from pipenv.vendor.packaging.utils import canonicalize_name for section_name, section in ( ("default", project.packages), ("develop", project.dev_packages), ): - for package_specified in section: - norm_name = pep423_name(package_specified) + for package_specified in section.keys(): if not is_pinned(section[package_specified]): - if norm_name in cached_lockfile[section_name]: - lockfile[section_name][norm_name] = cached_lockfile[ + canonical_name = canonicalize_name(package_specified) + if canonical_name in cached_lockfile[section_name]: + lockfile[section_name][canonical_name] = cached_lockfile[ section_name - ][norm_name] + ][canonical_name].copy() # Overwrite any develop packages with default packages. - for default_package in lockfile["default"]: - if default_package in lockfile["develop"]: - lockfile["develop"][default_package] = lockfile["default"][default_package] + lockfile["develop"].update(overwrite_dev(lockfile.get("default", {}), lockfile["develop"])) if write: project.write_lockfile(lockfile) click.echo( @@ -1121,51 +1083,46 @@ def do_lock( def do_purge(bare=False, downloads=False, allow_global=False): """Executes the purge functionality.""" - from .vendor.requirementslib.models.requirements import Requirement if downloads: if not bare: - click.echo(crayons.normal(u"Clearing out downloads directory…", bold=True)) - shutil.rmtree(project.download_location) + click.echo(crayons.normal(fix_utf8("Clearing out downloads directory…"), bold=True)) + vistir.path.rmtree(project.download_location) return - freeze = delegator.run( - "{0} freeze".format( - escape_grouped_arguments(which_pip(allow_global=allow_global)) - ) - ).out # Remove comments from the output, if any. - installed = [ - line for line in freeze.splitlines() if not line.lstrip().startswith("#") - ] - # Remove setuptools and friends from installed, if present. - for package_name in BAD_PACKAGES: - for i, package in enumerate(installed): - if package.startswith(package_name): - del installed[i] - actually_installed = [] - for package in installed: - try: - dep = Requirement.from_line(package) - except AssertionError: - dep = None - if dep and not dep.is_vcs and not dep.editable: - dep = dep.name - actually_installed.append(dep) + installed = set([ + pep423_name(pkg.project_name) for pkg in project.environment.get_installed_packages() + ]) + bad_pkgs = set([pep423_name(pkg) for pkg in BAD_PACKAGES]) + # Remove setuptools, pip, etc from targets for removal + to_remove = installed - bad_pkgs + + # Skip purging if there is no packages which needs to be removed + if not to_remove: + if not bare: + click.echo("Found 0 installed package, skip purging.") + click.echo(crayons.green("Environment now purged and fresh!")) + return installed + if not bare: click.echo( - u"Found {0} installed package(s), purging…".format(len(actually_installed)) + fix_utf8("Found {0} installed package(s), purging…".format(len(to_remove))) ) + command = "{0} uninstall {1} -y".format( escape_grouped_arguments(which_pip(allow_global=allow_global)), - " ".join(actually_installed), + " ".join(to_remove), ) if environments.is_verbose(): click.echo("$ {0}".format(command)) c = delegator.run(command) + if c.return_code != 0: + raise exceptions.UninstallError(installed, command, c.out + c.err, c.return_code) if not bare: click.echo(crayons.blue(c.out)) click.echo(crayons.green("Environment now purged and fresh!")) + return installed def do_init( @@ -1185,7 +1142,6 @@ def do_init( """Executes the init functionality.""" from .environments import PIPENV_VIRTUALENV - cleanup_reqdir = False if not system: if not project.virtualenv_exists: try: @@ -1197,8 +1153,7 @@ def do_init( if not deploy: ensure_pipfile(system=system) if not requirements_dir: - cleanup_reqdir = True - requirements_dir = vistir.compat.TemporaryDirectory( + requirements_dir = vistir.path.create_tracked_tempdir( suffix="-requirements", prefix="pipenv-" ) # Write out the lockfile if it doesn't exist, but not if the Pipfile is being ignored @@ -1214,27 +1169,26 @@ def do_init( ) ) ) - click.echo(crayons.normal("Aborting deploy.", bold=True), err=True) - requirements_dir.cleanup() + raise exceptions.DeployException sys.exit(1) elif (system or allow_global) and not (PIPENV_VIRTUALENV): click.echo( - crayons.red( - u"Pipfile.lock ({0}) out of date, but installation " - u"uses {1}… re-building lockfile must happen in " - u"isolation. Please rebuild lockfile in a virtualenv. " - u"Continuing anyway…".format( + crayons.red(fix_utf8( + "Pipfile.lock ({0}) out of date, but installation " + "uses {1}… re-building lockfile must happen in " + "isolation. Please rebuild lockfile in a virtualenv. " + "Continuing anyway…".format( crayons.white(old_hash[-6:]), crayons.white("--system") - ), + )), bold=True, ), err=True, ) else: if old_hash: - msg = u"Pipfile.lock ({1}) out of date, updating to ({0})…" + msg = fix_utf8("Pipfile.lock ({1}) out of date, updating to ({0})…") else: - msg = u"Pipfile.lock is corrupted, replaced with ({0})…" + msg = fix_utf8("Pipfile.lock is corrupted, replaced with ({0})…") click.echo( crayons.red(msg.format(old_hash[-6:], new_hash[-6:]), bold=True), err=True, @@ -1251,19 +1205,15 @@ def do_init( # Unless we're in a virtualenv not managed by pipenv, abort if we're # using the system's python. if (system or allow_global) and not (PIPENV_VIRTUALENV): - click.echo( - "{0}: --system is intended to be used for Pipfile installation, " - "not installation of specific packages. Aborting.".format( - crayons.red("Warning", bold=True) - ), - err=True, + raise exceptions.PipenvOptionsError( + "--system", + "--system is intended to be used for Pipfile installation, " + "not installation of specific packages. Aborting.\n" + "See also: --deploy flag." ) - click.echo("See also: --deploy flag.", err=True) - requirements_dir.cleanup() - sys.exit(1) else: click.echo( - crayons.normal(u"Pipfile.lock not found, creating…", bold=True), + crayons.normal(fix_utf8("Pipfile.lock not found, creating…"), bold=True), err=True, ) do_lock( @@ -1279,11 +1229,9 @@ def do_init( allow_global=allow_global, skip_lock=skip_lock, concurrent=concurrent, - requirements_dir=requirements_dir.name, + requirements_dir=requirements_dir, pypi_mirror=pypi_mirror, ) - if cleanup_reqdir: - requirements_dir.cleanup() # Hint the user what to do to activate the virtualenv. if not allow_global and not deploy and "PIPENV_ACTIVE" not in os.environ: @@ -1311,15 +1259,16 @@ def pip_install( pypi_mirror=None, trusted_hosts=None ): - from notpip._internal import logger as piplogger + from pipenv.patched.notpip._internal import logger as piplogger + from .utils import Mapping from .vendor.urllib3.util import parse_url src = [] write_to_tmpfile = False if requirement: - editable_with_markers = requirement.editable and requirement.markers needs_hashes = not requirement.editable and not ignore_hashes and r is None - write_to_tmpfile = needs_hashes or editable_with_markers + has_subdir = requirement.is_vcs and requirement.req.subdirectory + write_to_tmpfile = needs_hashes or has_subdir if not trusted_hosts: trusted_hosts = [] @@ -1333,12 +1282,16 @@ def pip_install( ) # Create files for hash mode. if write_to_tmpfile: - with vistir.compat.NamedTemporaryFile( + if not requirements_dir: + requirements_dir = vistir.path.create_tracked_tempdir( + prefix="pipenv", suffix="requirements") + f = vistir.compat.NamedTemporaryFile( prefix="pipenv-", suffix="-requirement.txt", dir=requirements_dir, delete=False - ) as f: - f.write(vistir.misc.to_bytes(requirement.as_line())) - r = f.name + ) + f.write(vistir.misc.to_bytes(requirement.as_line())) + r = f.name + f.close() # Install dependencies when a package is a VCS dependency. if requirement and requirement.vcs: no_deps = False @@ -1348,21 +1301,27 @@ def pip_install( # Try installing for each source in project.sources. if index: - try: - index_source = project.find_source(index) - index_source = index_source.copy() - except SourceNotFound: - src_name = project.src_name_from_url(index) - index_url = parse_url(index) - verify_ssl = index_url.host not in trusted_hosts - index_source = {"url": index, "verify_ssl": verify_ssl, "name": src_name} + if isinstance(index, (Mapping, dict)): + index_source = index + else: + try: + index_source = project.find_source(index) + index_source = index_source.copy() + except SourceNotFound: + src_name = project.src_name_from_url(index) + index_url = parse_url(index) + verify_ssl = index_url.host not in trusted_hosts + index_source = {"url": index, "verify_ssl": verify_ssl, "name": src_name} sources = [index_source.copy(),] if extra_indexes: if isinstance(extra_indexes, six.string_types): extra_indexes = [extra_indexes,] for idx in extra_indexes: + extra_src = None + if isinstance(idx, (Mapping, dict)): + extra_src = idx try: - extra_src = project.find_source(idx) + extra_src = project.find_source(idx) if not extra_src else extra_src except SourceNotFound: src_name = project.src_name_from_url(idx) src_url = parse_url(idx) @@ -1382,12 +1341,15 @@ def pip_install( for source in sources ] if (requirement and requirement.editable) and not r: - install_reqs = requirement.as_line(as_list=True) + line_kwargs = {"as_list": True} + if requirement.markers: + line_kwargs["include_markers"] = False + install_reqs = requirement.as_line(**line_kwargs) if requirement.editable and install_reqs[0].startswith("-e "): req, install_reqs = install_reqs[0], install_reqs[1:] editable_opt, req = req.split(" ", 1) install_reqs = [editable_opt, req] + install_reqs - if not any(item.startswith("--hash") for item in install_reqs): + if not all(item.startswith("--hash") for item in install_reqs): ignore_hashes = True elif r: install_reqs = ["-r", r] @@ -1396,7 +1358,7 @@ def pip_install( ignore_hashes = True else: ignore_hashes = True if not requirement.hashes else False - install_reqs = requirement.as_line(as_list=True) + install_reqs = [escape_cmd(r) for r in requirement.as_line(as_list=True)] pip_command = [which_pip(allow_global=allow_global), "install"] if pre: pip_command.append("--pre") @@ -1409,7 +1371,6 @@ def pip_install( pip_command.append("--upgrade-strategy=only-if-needed") if no_deps: pip_command.append("--no-deps") - install_reqs = [escape_cmd(req) for req in install_reqs] pip_command.extend(install_reqs) pip_command.extend(prepare_pip_source_args(sources)) if not ignore_hashes: @@ -1431,7 +1392,8 @@ def pip_install( pip_config.update( {"PIP_SRC": vistir.misc.fs_str(project.virtualenv_src_location)} ) - pip_command = Script.parse(pip_command).cmdify() + cmd = Script.parse(pip_command) + pip_command = cmd.cmdify() c = delegator.run(pip_command, block=block, env=pip_config) return c @@ -1476,22 +1438,34 @@ def which_pip(allow_global=False): def system_which(command, mult=False): """Emulates the system's which. Returns None if not found.""" _which = "which -a" if not os.name == "nt" else "where" - c = delegator.run("{0} {1}".format(_which, command)) + os.environ = { + vistir.compat.fs_str(k): vistir.compat.fs_str(val) + for k, val in os.environ.items() + } try: - # Which Not found… - if c.return_code == 127: - click.echo( - "{}: the {} system utility is required for Pipenv to find Python installations properly." - "\n Please install it.".format( - crayons.red("Warning", bold=True), crayons.red(_which) - ), - err=True, - ) - assert c.return_code == 0 - except AssertionError: - return None if not mult else [] - - result = c.out.strip() or c.err.strip() + c = delegator.run("{0} {1}".format(_which, command)) + try: + # Which Not found… + if c.return_code == 127: + click.echo( + "{}: the {} system utility is required for Pipenv to find Python installations properly." + "\n Please install it.".format( + crayons.red("Warning", bold=True), crayons.red(_which) + ), + err=True, + ) + assert c.return_code == 0 + except AssertionError: + return None if not mult else [] + except TypeError: + from .vendor.pythonfinder import Finder + finder = Finder() + result = finder.which(command) + if result: + return result.path.as_posix() + return + else: + result = c.out.strip() or c.err.strip() if mult: return result.split("\n") @@ -1623,9 +1597,9 @@ def ensure_lockfile(keep_outdated=False, pypi_mirror=None): if new_hash != old_hash: click.echo( crayons.red( - u"Pipfile.lock ({0}) out of date, updating to ({1})…".format( + fix_utf8("Pipfile.lock ({0}) out of date, updating to ({1})…".format( old_hash[-6:], new_hash[-6:] - ), + )), bold=True, ), err=True, @@ -1643,13 +1617,22 @@ def do_py(system=False): def do_outdated(pypi_mirror=None): + # TODO: Allow --skip-lock here? from .vendor.requirementslib.models.requirements import Requirement + from .vendor.packaging.utils import canonicalize_name + from collections import namedtuple packages = {} - results = delegator.run("{0} freeze".format(which("pip"))).out.strip().split("\n") - results = filter(bool, results) - for result in results: - dep = Requirement.from_line(result) + package_info = namedtuple("PackageInfo", ["name", "installed", "available"]) + + installed_packages = project.environment.get_installed_packages() + outdated_packages = { + canonicalize_name(pkg.project_name): package_info + (pkg.project_name, pkg.parsed_version, pkg.latest_version) + for pkg in project.environment.get_outdated_packages() + } + for result in installed_packages: + dep = Requirement.from_line(str(result.as_requirement())) packages.update(dep.as_pipfile()) updated_packages = {} lockfile = do_lock(write=False, pypi_mirror=pypi_mirror) @@ -1660,13 +1643,25 @@ def do_outdated(pypi_mirror=None): except KeyError: pass outdated = [] + skipped = [] for package in packages: norm_name = pep423_name(package) if norm_name in updated_packages: if updated_packages[norm_name] != packages[package]: outdated.append( - (package, updated_packages[norm_name], packages[package]) + package_info(package, updated_packages[norm_name], packages[package]) ) + elif canonicalize_name(package) in outdated_packages: + skipped.append(outdated_packages[canonicalize_name(package)]) + for package, old_version, new_version in skipped: + click.echo(crayons.yellow( + "Skipped Update of Package {0!s}: {1!s} installed, {2!s} available.".format( + package, old_version, new_version + )), err=True + ) + if not outdated: + click.echo(crayons.green("All packages are up to date!", bold=True)) + sys.exit(0) for package, new_version, old_version in outdated: click.echo( "Package {0!r} out-of-date: {1!r} installed, {2!r} available.".format( @@ -1698,11 +1693,12 @@ def do_install( selective_upgrade=False, ): from .environments import PIPENV_VIRTUALENV, PIPENV_USE_SYSTEM - from notpip._internal.exceptions import PipError + from .vendor.pip_shims.shims import PipError - requirements_directory = vistir.compat.TemporaryDirectory( + requirements_directory = vistir.path.create_tracked_tempdir( suffix="-requirements", prefix="pipenv-" ) + warnings.filterwarnings("default", category=vistir.compat.ResourceWarning) if selective_upgrade: keep_outdated = True packages = packages if packages else [] @@ -1712,6 +1708,12 @@ def do_install( # Don't search for requirements.txt files if the user provides one if requirements or package_args or project.pipfile_exists: skip_requirements = True + # Don't attempt to install develop and default packages if Pipfile is missing + if not project.pipfile_exists and not (package_args or dev) and not code: + if not (ignore_pipfile or deploy): + raise exceptions.PipfileNotFound(project.path_to("Pipfile")) + elif ((skip_lock and deploy) or ignore_pipfile) and not project.lockfile_exists: + raise exceptions.LockfileNotFound(project.path_to("Pipfile.lock")) concurrent = not sequential # Ensure that virtualenv is available. ensure_project( @@ -1731,35 +1733,29 @@ def do_install( remote = requirements and is_valid_url(requirements) # Warn and exit if --system is used without a pipfile. if (system and package_args) and not (PIPENV_VIRTUALENV): - click.echo( - "{0}: --system is intended to be used for Pipfile installation, " - "not installation of specific packages. Aborting.".format( - crayons.red("Warning", bold=True) - ), - err=True, - ) - click.echo("See also: --deploy flag.", err=True) - requirements_directory.cleanup() - sys.exit(1) + raise exceptions.SystemUsageError # Automatically use an activated virtualenv. if PIPENV_USE_SYSTEM: system = True # Check if the file is remote or not if remote: - fd, temp_reqs = tempfile.mkstemp( - prefix="pipenv-", suffix="-requirement.txt", dir=requirements_directory.name - ) - requirements_url = requirements - # Download requirements file click.echo( crayons.normal( - u"Remote requirements file provided! Downloading…", bold=True + fix_utf8("Remote requirements file provided! Downloading…"), bold=True ), err=True, ) + fd = vistir.path.create_tracked_tempfile( + prefix="pipenv-", suffix="-requirement.txt", dir=requirements_directory + ) + temp_reqs = fd.name + requirements_url = requirements + # Download requirements file try: download_file(requirements, temp_reqs) except IOError: + fd.close() + os.unlink(temp_reqs) click.echo( crayons.red( u"Unable to find requirements file at {0}.".format( @@ -1768,8 +1764,9 @@ def do_install( ), err=True, ) - requirements_directory.cleanup() sys.exit(1) + finally: + fd.close() # Replace the url with the temporary requirements file requirements = temp_reqs remote = True @@ -1777,7 +1774,7 @@ def do_install( error, traceback = None, None click.echo( crayons.normal( - u"Requirements file provided! Importing into Pipfile…", bold=True + fix_utf8("Requirements file provided! Importing into Pipfile…"), bold=True ), err=True, ) @@ -1800,16 +1797,15 @@ def do_install( finally: # If requirements file was provided by remote url delete the temporary file if remote: - os.close(fd) # Close for windows to allow file cleanup. - os.remove(project.path_to(temp_reqs)) + fd.close() # Close for windows to allow file cleanup. + os.remove(temp_reqs) if error and traceback: click.echo(crayons.red(error)) click.echo(crayons.blue(str(traceback)), err=True) - requirements_directory.cleanup() sys.exit(1) if code: click.echo( - crayons.normal(u"Discovering imports from local codebase…", bold=True) + crayons.normal(fix_utf8("Discovering imports from local codebase…"), bold=True) ) for req in import_from_code(code): click.echo(" Found {0}!".format(crayons.green(req))) @@ -1859,7 +1855,7 @@ def do_install( # Install all dependencies, if none was provided. # This basically ensures that we have a pipfile and lockfile, then it locks and # installs from the lockfile - if packages is False and editable_packages is False: + if not packages and not editable_packages: # Update project settings with pre preference. if pre: project.update_settings({"allow_prereleases": pre}) @@ -1874,6 +1870,7 @@ def do_install( pre=pre, requirements_dir=requirements_directory, pypi_mirror=pypi_mirror, + keep_outdated=keep_outdated ) # This is for if the user passed in dependencies, then we want to maek sure we @@ -1882,39 +1879,60 @@ def do_install( # make a tuple of (display_name, entry) pkg_list = packages + ["-e {0}".format(pkg) for pkg in editable_packages] - + if not system and not project.virtualenv_exists: + do_init( + dev=dev, + system=system, + allow_global=system, + concurrent=concurrent, + keep_outdated=keep_outdated, + requirements_dir=requirements_directory, + deploy=deploy, + pypi_mirror=pypi_mirror, + skip_lock=skip_lock, + ) for pkg_line in pkg_list: click.echo( crayons.normal( - u"Installing {0}…".format(crayons.green(pkg_line, bold=True)), + fix_utf8("Installing {0}…".format(crayons.green(pkg_line, bold=True))), bold=True, ) ) # pip install: - with spinner(): + with vistir.contextmanagers.temp_environ(), create_spinner("Installing...") as sp: + if not system: + os.environ["PIP_USER"] = vistir.compat.fs_str("0") try: pkg_requirement = Requirement.from_line(pkg_line) except ValueError as e: - click.echo("{0}: {1}".format(crayons.red("WARNING"), e)) - requirements_directory.cleanup() + sp.write_err(vistir.compat.fs_str("{0}: {1}".format(crayons.red("WARNING"), e))) + sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format("Installation Failed")) sys.exit(1) if index_url: pkg_requirement.index = index_url - c = pip_install( - pkg_requirement, - ignore_hashes=True, - allow_global=system, - selective_upgrade=selective_upgrade, - no_deps=False, - pre=pre, - requirements_dir=requirements_directory.name, - index=index_url, - extra_indexes=extra_index_url, - pypi_mirror=pypi_mirror, - ) + try: + c = pip_install( + pkg_requirement, + ignore_hashes=True, + allow_global=system, + selective_upgrade=selective_upgrade, + no_deps=False, + pre=pre, + requirements_dir=requirements_directory, + index=index_url, + extra_indexes=extra_index_url, + pypi_mirror=pypi_mirror, + ) + except (ValueError, RuntimeError) as e: + sp.write_err(vistir.compat.fs_str( + "{0}: {1}".format(crayons.red("WARNING"), e), + )) + sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format( + "Installation Failed", + )) # Warn if --editable wasn't passed. if pkg_requirement.is_vcs and not pkg_requirement.editable: - click.echo( + sp.write_err( "{0}: You installed a VCS dependency in non-editable mode. " "This will work fine, but sub-dependencies will not be resolved by {1}." "\n To enable this sub-dependency functionality, specify that this dependency is editable." @@ -1923,44 +1941,39 @@ def do_install( crayons.red("$ pipenv lock"), ) ) - click.echo(crayons.blue(format_pip_output(c.out))) - # Ensure that package was successfully installed. - try: - assert c.return_code == 0 - except AssertionError: - click.echo( - "{0} An error occurred while installing {1}!".format( - crayons.red("Error: ", bold=True), crayons.green(pkg_line) - ), - err=True, - ) - click.echo(crayons.blue(format_pip_error(c.err)), err=True) - if "setup.py egg_info" in c.err: - click.echo( - "This is likely caused by a bug in {0}. " - "Report this to its maintainers.".format( - crayons.green(pkg_requirement.name) - ), - err=True, + click.echo(crayons.blue(format_pip_output(c.out))) + # Ensure that package was successfully installed. + if c.return_code != 0: + sp.write_err(vistir.compat.fs_str( + "{0} An error occurred while installing {1}!".format( + crayons.red("Error: ", bold=True), crayons.green(pkg_line) + ), + )) + sp.write_err(vistir.compat.fs_str(crayons.blue(format_pip_error(c.err)))) + if "setup.py egg_info" in c.err: + sp.write_err(vistir.compat.fs_str( + "This is likely caused by a bug in {0}. " + "Report this to its maintainers.".format( + crayons.green(pkg_requirement.name) + ) + )) + sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format("Installation Failed")) + sys.exit(1) + sp.write(vistir.compat.fs_str( + u"{0} {1} {2} {3}{4}".format( + crayons.normal(u"Adding", bold=True), + crayons.green(u"{0}".format(pkg_requirement.name), bold=True), + crayons.normal(u"to Pipfile's", bold=True), + crayons.red(u"[dev-packages]" if dev else u"[packages]", bold=True), + crayons.normal(fix_utf8("…"), bold=True), ) - requirements_directory.cleanup() - sys.exit(1) - click.echo( - "{0} {1} {2} {3}{4}".format( - crayons.normal("Adding", bold=True), - crayons.green(pkg_requirement.name, bold=True), - crayons.normal("to Pipfile's", bold=True), - crayons.red("[dev-packages]" if dev else "[packages]", bold=True), - crayons.normal("…", bold=True), - ) - ) + )) + sp.ok(environments.PIPENV_SPINNER_OK_TEXT.format("Installation Succeeded")) # Add the package to the Pipfile. try: project.add_package_to_pipfile(pkg_requirement, dev) except ValueError as e: - click.echo( - "{0} {1}".format(crayons.red("ERROR (PACKAGE NOT INSTALLED):"), e) - ) + raise exceptions.PipfileException(e) # Update project settings with pre preference. if pre: project.update_settings({"allow_prereleases": pre}) @@ -1975,7 +1988,6 @@ def do_install( pypi_mirror=pypi_mirror, skip_lock=skip_lock, ) - requirements_directory.cleanup() sys.exit(0) @@ -1990,9 +2002,11 @@ def do_uninstall( all=False, keep_outdated=False, pypi_mirror=None, + ctx=None ): from .environments import PIPENV_USE_SYSTEM from .vendor.requirementslib.models.requirements import Requirement + from .vendor.packaging.utils import canonicalize_name # Automatically use an activated virtualenv. if PIPENV_USE_SYSTEM: @@ -2001,21 +2015,31 @@ def do_uninstall( # TODO: We probably shouldn't ensure a project exists if the outcome will be to just # install things in order to remove them... maybe tell the user to install first? ensure_project(three=three, python=python, pypi_mirror=pypi_mirror) + # Un-install all dependencies, if --all was provided. + if not any([packages, editable_packages, all_dev, all]): + raise exceptions.MissingParameter( + crayons.red("No package provided!"), + ctx=ctx, param_type="parameter", + ) editable_pkgs = [ Requirement.from_line("-e {0}".format(p)).name for p in editable_packages if p ] - package_names = [p for p in packages if p] + editable_pkgs + packages = packages + editable_pkgs + package_names = [p for p in packages if p] + package_map = { + canonicalize_name(p): p for p in packages if p + } + installed_package_names = project.installed_package_names + # Intelligently detect if --dev should be used or not. + lockfile_packages = set() + if project.lockfile_exists: + project_pkg_names = project.lockfile_package_names + else: + project_pkg_names = project.pipfile_package_names pipfile_remove = True - # Un-install all dependencies, if --all was provided. - if all is True: - click.echo( - crayons.normal(u"Un-installing all packages from virtualenv…", bold=True) - ) - do_purge(allow_global=system) - return # Uninstall [dev-packages], if --dev was provided. if all_dev: - if "dev-packages" not in project.parsed_pipfile: + if "dev-packages" not in project.parsed_pipfile and not project_pkg_names["dev"]: click.echo( crayons.normal( "No {0} to uninstall.".format(crayons.red("[dev-packages]")), @@ -2025,28 +2049,84 @@ def do_uninstall( return click.echo( crayons.normal( - u"Un-installing {0}…".format(crayons.red("[dev-packages]")), bold=True + fix_utf8("Un-installing {0}…".format(crayons.red("[dev-packages]"))), bold=True ) ) - package_names = project.dev_packages.keys() - if packages is False and editable_packages is False and not all_dev: - click.echo(crayons.red("No package provided!"), err=True) - return 1 - for package_name in package_names: - click.echo(u"Un-installing {0}…".format(crayons.green(package_name))) - cmd = "{0} uninstall {1} -y".format( - escape_grouped_arguments(which_pip(allow_global=system)), package_name - ) + package_names = project_pkg_names["dev"] + + # Remove known "bad packages" from the list. + bad_pkgs = get_canonical_names(BAD_PACKAGES) + ignored_packages = bad_pkgs & set(list(package_map.keys())) + for ignored_pkg in ignored_packages: if environments.is_verbose(): - click.echo("$ {0}".format(cmd)) - c = delegator.run(cmd) - click.echo(crayons.blue(c.out)) - if pipfile_remove: + click.echo("Ignoring {0}.".format(ignored_pkg), err=True) + pkg_name_index = package_names.index(package_map[ignored_pkg]) + del package_names[pkg_name_index] + + used_packages = project_pkg_names["combined"] & installed_package_names + failure = False + packages_to_remove = set() + if all: + click.echo( + crayons.normal( + fix_utf8("Un-installing all {0} and {1}…".format( + crayons.red("[dev-packages]"), + crayons.red("[packages]"), + )), bold=True + ) + ) + do_purge(bare=False, allow_global=system) + sys.exit(0) + if all_dev: + package_names = project_pkg_names["dev"] + else: + package_names = set([pkg_name for pkg_name in package_names]) + selected_pkg_map = { + canonicalize_name(p): p for p in package_names + } + packages_to_remove = [ + p for normalized, p in selected_pkg_map.items() + if normalized in (used_packages - bad_pkgs) + ] + for normalized, package_name in selected_pkg_map.items(): + click.echo( + crayons.white( + fix_utf8("Uninstalling {0}…".format(package_name)), bold=True + ) + ) + # Uninstall the package. + if package_name in packages_to_remove: + with project.environment.activated(): + cmd = "{0} uninstall {1} -y".format( + escape_grouped_arguments(which_pip(allow_global=system)), package_name, + ) + if environments.is_verbose(): + click.echo("$ {0}".format(cmd)) + c = delegator.run(cmd) + click.echo(crayons.blue(c.out)) + if c.return_code != 0: + failure = True + if not failure and pipfile_remove: in_packages = project.get_package_name_in_pipfile(package_name, dev=False) in_dev_packages = project.get_package_name_in_pipfile( package_name, dev=True ) - if not in_dev_packages and not in_packages: + if normalized in lockfile_packages: + click.echo("{0} {1} {2} {3}".format( + crayons.blue("Removing"), + crayons.green(package_name), + crayons.blue("from"), + crayons.white(fix_utf8("Pipfile.lock…"))) + ) + lockfile = project.get_or_create_lockfile() + if normalized in lockfile.default: + del lockfile.default[normalized] + if normalized in lockfile.develop: + del lockfile.develop[normalized] + lockfile.write() + if not (in_dev_packages or in_packages): + if normalized in lockfile_packages: + continue click.echo( "No package {0} to remove from Pipfile.".format( crayons.green(package_name) @@ -2055,20 +2135,29 @@ def do_uninstall( continue click.echo( - u"Removing {0} from Pipfile…".format(crayons.green(package_name)) + fix_utf8("Removing {0} from Pipfile…".format(crayons.green(package_name))) ) # Remove package from both packages and dev-packages. - project.remove_package_from_pipfile(package_name, dev=True) - project.remove_package_from_pipfile(package_name, dev=False) + if in_dev_packages: + project.remove_package_from_pipfile(package_name, dev=True) + if in_packages: + project.remove_package_from_pipfile(package_name, dev=False) if lock: do_lock(system=system, keep_outdated=keep_outdated, pypi_mirror=pypi_mirror) + sys.exit(int(failure)) def do_shell(three=None, python=False, fancy=False, shell_args=None, pypi_mirror=None): # Ensure that virtualenv is available. - ensure_project(three=three, python=python, validate=False, pypi_mirror=pypi_mirror) + ensure_project( + three=three, python=python, validate=False, pypi_mirror=pypi_mirror, + ) + # Set an environment variable, so we know we're in the environment. os.environ["PIPENV_ACTIVE"] = vistir.misc.fs_str("1") + + os.environ.pop("PIP_SHIMS_BASE_MODULE", None) + # Support shell compatibility mode. if PIPENV_SHELL_FANCY: fancy = True @@ -2076,9 +2165,13 @@ def do_shell(three=None, python=False, fancy=False, shell_args=None, pypi_mirror from .shells import choose_shell shell = choose_shell() - click.echo("Launching subshell in virtual environment…", err=True) + click.echo(fix_utf8("Launching subshell in virtual environment…"), err=True) - fork_args = (project.virtualenv_location, project.project_directory, shell_args) + fork_args = ( + project.virtualenv_location, + project.project_directory, + shell_args, + ) if fancy: shell.fork(*fork_args) @@ -2087,9 +2180,9 @@ def do_shell(three=None, python=False, fancy=False, shell_args=None, pypi_mirror try: shell.fork_compat(*fork_args) except (AttributeError, ImportError): - click.echo( - u"Compatibility mode not supported. " - u"Trying to continue as well-configured shell…", + click.echo(fix_utf8( + "Compatibility mode not supported. " + "Trying to continue as well-configured shell…"), err=True, ) shell.fork(*fork_args) @@ -2099,13 +2192,7 @@ def _inline_activate_virtualenv(): try: activate_this = which("activate_this.py") if not activate_this or not os.path.exists(activate_this): - click.echo( - u"{0}: activate_this.py not found. Your environment is most " - u"certainly not activated. Continuing anyway…" - u"".format(crayons.red("Warning", bold=True)), - err=True, - ) - return + raise exceptions.VirtualenvActivationException() with open(activate_this) as f: code = compile(f.read(), activate_this, "exec") exec(code, dict(__file__=activate_this)) @@ -2214,8 +2301,16 @@ def do_run(command, args, three=None, python=False, pypi_mirror=None): from .cmdparse import ScriptEmptyError # Ensure that virtualenv is available. - ensure_project(three=three, python=python, validate=False, pypi_mirror=pypi_mirror) + ensure_project( + three=three, python=python, validate=False, pypi_mirror=pypi_mirror, + ) + + # Set an environment variable, so we know we're in the environment. + os.environ["PIPENV_ACTIVE"] = vistir.misc.fs_str("1") + + os.environ.pop("PIP_SHIMS_BASE_MODULE", None) load_dot_env() + # Activate virtualenv under the current interpreter's environment inline_activate_virtual_environment() try: @@ -2267,7 +2362,7 @@ def do_check( sys.exit(1) else: sys.exit(0) - click.echo(crayons.normal(u"Checking PEP 508 requirements…", bold=True)) + click.echo(crayons.normal(fix_utf8("Checking PEP 508 requirements…"), bold=True)) if system: python = system_which("python") else: @@ -2303,7 +2398,7 @@ def do_check( sys.exit(1) else: click.echo(crayons.green("Passed!")) - click.echo(crayons.normal(u"Checking installed package safety…", bold=True)) + click.echo(crayons.normal(fix_utf8("Checking installed package safety…"), bold=True)) path = pep508checker.__file__.rstrip("cdo") path = os.sep.join(__file__.split(os.sep)[:-1] + ["patched", "safety.zip"]) if not system: @@ -2321,15 +2416,15 @@ def do_check( else: ignored = "" c = delegator.run( - '"{0}" {1} check --json --key=1ab8d58f-5122e025-83674263-bc1e79e0 {2}'.format( - python, escape_grouped_arguments(path), ignored + '"{0}" {1} check --json --key={2} {3}'.format( + python, escape_grouped_arguments(path), PIPENV_PYUP_API_KEY, ignored ) ) try: results = simplejson.loads(c.out) except ValueError: click.echo("An error occurred:", err=True) - click.echo(c.err, err=True) + click.echo(c.err if len(c.err) > 0 else c.out, err=True) sys.exit(1) for (package, resolved, installed, description, vuln) in results: click.echo( @@ -2362,6 +2457,9 @@ def do_graph(bare=False, json=False, json_tree=False, reverse=False): err=True, ) sys.exit(1) + except RuntimeError: + pass + if reverse and json: click.echo( u"{0}: {1}".format( @@ -2441,7 +2539,7 @@ def do_graph(bare=False, json=False, json_tree=False, reverse=False): click.echo(simplejson.dumps(data, indent=4)) sys.exit(0) else: - for line in c.out.split("\n"): + for line in c.out.strip().split("\n"): # Ignore bad packages as top level. if line.split("==")[0] in BAD_PACKAGES and not reverse: continue @@ -2483,13 +2581,7 @@ def do_sync( ): # The lock file needs to exist because sync won't write to it. if not project.lockfile_exists: - click.echo( - "{0}: Pipfile.lock is missing! You need to run {1} first.".format( - crayons.red("Error", bold=True), crayons.red("$ pipenv lock", bold=True) - ), - err=True, - ) - return 1 + raise exceptions.LockfileNotFound(project.lockfile_location) # Ensure that virtualenv is available if not system. ensure_project( @@ -2501,7 +2593,7 @@ def do_sync( ) # Install everything. - requirements_dir = vistir.compat.TemporaryDirectory( + requirements_dir = vistir.path.create_tracked_tempdir( suffix="-requirements", prefix="pipenv-" ) do_init( @@ -2513,8 +2605,8 @@ def do_sync( deploy=deploy, system=system, ) - requirements_dir.cleanup() - click.echo(crayons.green("All dependencies are now up-to-date!")) + if not bare: + click.echo(crayons.green("All dependencies are now up-to-date!")) def do_clean(ctx, three=None, python=None, dry_run=False, bare=False, pypi_mirror=None): @@ -2522,14 +2614,17 @@ def do_clean(ctx, three=None, python=None, dry_run=False, bare=False, pypi_mirro from packaging.utils import canonicalize_name ensure_project(three=three, python=python, validate=False, pypi_mirror=pypi_mirror) ensure_lockfile(pypi_mirror=pypi_mirror) + # Make sure that the virtualenv's site packages are configured correctly + # otherwise we may end up removing from the global site packages directory installed_package_names = [ - canonicalize_name(pkg.project_name) for pkg in project.get_installed_packages() + canonicalize_name(pkg.project_name) for pkg + in project.environment.get_installed_packages() ] # Remove known "bad packages" from the list. for bad_package in BAD_PACKAGES: if canonicalize_name(bad_package) in installed_package_names: if environments.is_verbose(): - click.echo("Ignoring {0}.".format(repr(bad_package)), err=True) + click.echo("Ignoring {0}.".format(bad_package), err=True) del installed_package_names[installed_package_names.index( canonicalize_name(bad_package) )] @@ -2543,14 +2638,15 @@ def do_clean(ctx, three=None, python=None, dry_run=False, bare=False, pypi_mirro )] failure = False for apparent_bad_package in installed_package_names: - if dry_run: + if dry_run and not bare: click.echo(apparent_bad_package) else: - click.echo( - crayons.white( - "Uninstalling {0}…".format(repr(apparent_bad_package)), bold=True + if not bare: + click.echo( + crayons.white( + fix_utf8("Uninstalling {0}…".format(apparent_bad_package)), bold=True + ) ) - ) # Uninstall the package. c = delegator.run( "{0} uninstall {1} -y".format(which_pip(), apparent_bad_package) diff --git a/pipenv/environment.py b/pipenv/environment.py new file mode 100644 index 00000000..7a7fd2ea --- /dev/null +++ b/pipenv/environment.py @@ -0,0 +1,630 @@ +# -*- coding=utf-8 -*- + +import contextlib +import importlib +import json +import os +import sys +import operator +import pkg_resources +import site +import six + +from distutils.sysconfig import get_python_lib +from sysconfig import get_paths + +from cached_property import cached_property + +import vistir +import pipenv + +from .utils import normalize_path + +BASE_WORKING_SET = pkg_resources.WorkingSet(sys.path) + + +class Environment(object): + def __init__(self, prefix=None, is_venv=False, base_working_set=None, pipfile=None, + sources=None, project=None): + super(Environment, self).__init__() + self._modules = {'pkg_resources': pkg_resources, 'pipenv': pipenv} + self.base_working_set = base_working_set if base_working_set else BASE_WORKING_SET + prefix = normalize_path(prefix) + self.is_venv = not prefix == normalize_path(sys.prefix) + if not sources: + sources = [] + self.project = project + if project and not sources: + sources = project.sources + self.sources = sources + if project and not pipfile: + pipfile = project.parsed_pipfile + self.pipfile = pipfile + self.extra_dists = [] + prefix = prefix if prefix else sys.prefix + self.prefix = vistir.compat.Path(prefix) + self.sys_paths = get_paths() + + def safe_import(self, name): + """Helper utility for reimporting previously imported modules while inside the env""" + module = None + if name not in self._modules: + self._modules[name] = importlib.import_module(name) + module = self._modules[name] + if not module: + dist = next(iter( + dist for dist in self.base_working_set if dist.project_name == name + ), None) + if dist: + dist.activate() + module = importlib.import_module(name) + if name in sys.modules: + try: + six.moves.reload_module(module) + six.moves.reload_module(sys.modules[name]) + except TypeError: + del sys.modules[name] + sys.modules[name] = self._modules[name] + return module + + @classmethod + def resolve_dist(cls, dist, working_set): + """Given a local distribution and a working set, returns all dependencies from the set. + + :param dist: A single distribution to find the dependencies of + :type dist: :class:`pkg_resources.Distribution` + :param working_set: A working set to search for all packages + :type working_set: :class:`pkg_resources.WorkingSet` + :return: A set of distributions which the package depends on, including the package + :rtype: set(:class:`pkg_resources.Distribution`) + """ + + deps = set() + deps.add(dist) + try: + reqs = dist.requires() + except (AttributeError, OSError, IOError): # The METADATA file can't be found + return deps + for req in reqs: + dist = working_set.find(req) + deps |= cls.resolve_dist(dist, working_set) + return deps + + def add_dist(self, dist_name): + dist = pkg_resources.get_distribution(pkg_resources.Requirement(dist_name)) + extras = self.resolve_dist(dist, self.base_working_set) + if extras: + self.extra_dists.extend(extras) + + @cached_property + def python_version(self): + with self.activated(): + from sysconfig import get_python_version + py_version = get_python_version() + return py_version + + @property + def python_info(self): + include_dir = self.prefix / "include" + python_path = next(iter(list(include_dir.iterdir())), None) + if python_path and python_path.name.startswith("python"): + python_version = python_path.name.replace("python", "") + py_version_short, abiflags = python_version[:3], python_version[3:] + return {"py_version_short": py_version_short, "abiflags": abiflags} + return {} + + @cached_property + def base_paths(self): + """ + Returns the context appropriate paths for the environment. + + :return: A dictionary of environment specific paths to be used for installation operations + :rtype: dict + + .. note:: The implementation of this is borrowed from a combination of pip and + virtualenv and is likely to change at some point in the future. + + >>> from pipenv.core import project + >>> from pipenv.environment import Environment + >>> env = Environment(prefix=project.virtualenv_location, is_venv=True, sources=project.sources) + >>> import pprint + >>> pprint.pprint(env.base_paths) + {'PATH': '/home/hawk/.virtualenvs/pipenv-MfOPs1lW/bin::/bin:/usr/bin', + 'PYTHONPATH': '/home/hawk/.virtualenvs/pipenv-MfOPs1lW/lib/python3.7/site-packages', + 'data': '/home/hawk/.virtualenvs/pipenv-MfOPs1lW', + 'include': '/home/hawk/.pyenv/versions/3.7.1/include/python3.7m', + 'libdir': '/home/hawk/.virtualenvs/pipenv-MfOPs1lW/lib/python3.7/site-packages', + 'platinclude': '/home/hawk/.pyenv/versions/3.7.1/include/python3.7m', + 'platlib': '/home/hawk/.virtualenvs/pipenv-MfOPs1lW/lib/python3.7/site-packages', + 'platstdlib': '/home/hawk/.virtualenvs/pipenv-MfOPs1lW/lib/python3.7', + 'prefix': '/home/hawk/.virtualenvs/pipenv-MfOPs1lW', + 'purelib': '/home/hawk/.virtualenvs/pipenv-MfOPs1lW/lib/python3.7/site-packages', + 'scripts': '/home/hawk/.virtualenvs/pipenv-MfOPs1lW/bin', + 'stdlib': '/home/hawk/.pyenv/versions/3.7.1/lib/python3.7'} + """ + + prefix = self.prefix.as_posix() + install_scheme = 'nt' if (os.name == 'nt') else 'posix_prefix' + paths = get_paths(install_scheme, vars={ + 'base': prefix, + 'platbase': prefix, + }) + paths["PATH"] = paths["scripts"] + os.pathsep + os.defpath + if "prefix" not in paths: + paths["prefix"] = prefix + purelib = get_python_lib(plat_specific=0, prefix=prefix) + platlib = get_python_lib(plat_specific=1, prefix=prefix) + if purelib == platlib: + lib_dirs = purelib + else: + lib_dirs = purelib + os.pathsep + platlib + paths["libdir"] = purelib + paths["purelib"] = purelib + paths["platlib"] = platlib + paths['PYTHONPATH'] = lib_dirs + paths["libdirs"] = lib_dirs + return paths + + @cached_property + def script_basedir(self): + """Path to the environment scripts dir""" + script_dir = self.base_paths["scripts"] + return script_dir + + @property + def python(self): + """Path to the environment python""" + py = vistir.compat.Path(self.base_paths["scripts"]).joinpath("python").as_posix() + if not py: + return vistir.compat.Path(sys.executable).as_posix() + return py + + @cached_property + def sys_path(self): + """The system path inside the environment + + :return: The :data:`sys.path` from the environment + :rtype: list + """ + + current_executable = vistir.compat.Path(sys.executable).as_posix() + if not self.python or self.python == current_executable: + return sys.path + elif any([sys.prefix == self.prefix, not self.is_venv]): + return sys.path + cmd_args = [self.python, "-c", "import json, sys; print(json.dumps(sys.path))"] + path, _ = vistir.misc.run(cmd_args, return_object=False, nospin=True, block=True, combine_stderr=False) + path = json.loads(path.strip()) + return path + + @cached_property + def sys_prefix(self): + """The prefix run inside the context of the environment + + :return: The python prefix inside the environment + :rtype: :data:`sys.prefix` + """ + + command = [self.python, "-c" "import sys; print(sys.prefix)"] + c = vistir.misc.run(command, return_object=True, block=True, nospin=True) + sys_prefix = vistir.compat.Path(vistir.misc.to_text(c.out).strip()).as_posix() + return sys_prefix + + @cached_property + def paths(self): + paths = {} + with vistir.contextmanagers.temp_environ(), vistir.contextmanagers.temp_path(): + os.environ["PYTHONIOENCODING"] = vistir.compat.fs_str("utf-8") + os.environ["PYTHONDONTWRITEBYTECODE"] = vistir.compat.fs_str("1") + paths = self.base_paths + os.environ["PATH"] = paths["PATH"] + os.environ["PYTHONPATH"] = paths["PYTHONPATH"] + if "headers" not in paths: + paths["headers"] = paths["include"] + return paths + + @property + def scripts_dir(self): + return self.paths["scripts"] + + @property + def libdir(self): + purelib = self.paths.get("purelib", None) + if purelib and os.path.exists(purelib): + return "purelib", purelib + return "platlib", self.paths["platlib"] + + def get_distributions(self): + """Retrives the distributions installed on the library path of the environment + + :return: A set of distributions found on the library path + :rtype: iterator + """ + + pkg_resources = self.safe_import("pkg_resources") + return pkg_resources.find_distributions(self.paths["PYTHONPATH"]) + + def find_egg(self, egg_dist): + """Find an egg by name in the given environment""" + site_packages = get_python_lib() + search_filename = "{0}.egg-link".format(egg_dist.project_name) + try: + user_site = site.getusersitepackages() + except AttributeError: + user_site = site.USER_SITE + search_locations = [site_packages, user_site] + for site_directory in search_locations: + egg = os.path.join(site_directory, search_filename) + if os.path.isfile(egg): + return egg + + def locate_dist(self, dist): + """Given a distribution, try to find a corresponding egg link first. + + If the egg - link doesn 't exist, return the supplied distribution.""" + + location = self.find_egg(dist) + if not location: + return dist.location + + def dist_is_in_project(self, dist): + """Determine whether the supplied distribution is in the environment.""" + from .project import _normalized + prefix = _normalized(self.base_paths["prefix"]) + location = self.locate_dist(dist) + if not location: + return False + return _normalized(location).startswith(prefix) + + def get_installed_packages(self): + """Returns all of the installed packages in a given environment""" + workingset = self.get_working_set() + packages = [pkg for pkg in workingset if self.dist_is_in_project(pkg)] + return packages + + @contextlib.contextmanager + def get_finder(self, pre=False): + from .vendor.pip_shims import Command, cmdoptions, index_group, PackageFinder + from .environments import PIPENV_CACHE_DIR + index_urls = [source.get("url") for source in self.sources] + + class PipCommand(Command): + name = "PipCommand" + + pip_command = PipCommand() + index_opts = cmdoptions.make_option_group( + index_group, pip_command.parser + ) + cmd_opts = pip_command.cmd_opts + pip_command.parser.insert_option_group(0, index_opts) + pip_command.parser.insert_option_group(0, cmd_opts) + pip_args = self._modules["pipenv"].utils.prepare_pip_source_args(self.sources) + pip_options, _ = pip_command.parser.parse_args(pip_args) + pip_options.cache_dir = PIPENV_CACHE_DIR + pip_options.pre = self.pipfile.get("pre", pre) + with pip_command._build_session(pip_options) as session: + finder = PackageFinder( + find_links=pip_options.find_links, + index_urls=index_urls, allow_all_prereleases=pip_options.pre, + trusted_hosts=pip_options.trusted_hosts, + process_dependency_links=pip_options.process_dependency_links, + session=session + ) + yield finder + + def get_package_info(self, pre=False): + dependency_links = [] + packages = self.get_installed_packages() + # This code is borrowed from pip's current implementation + for dist in packages: + if dist.has_metadata('dependency_links.txt'): + dependency_links.extend(dist.get_metadata_lines('dependency_links.txt')) + + with self.get_finder() as finder: + finder.add_dependency_links(dependency_links) + + for dist in packages: + typ = 'unknown' + all_candidates = finder.find_all_candidates(dist.key) + if not self.pipfile.get("pre", finder.allow_all_prereleases): + # Remove prereleases + all_candidates = [ + candidate for candidate in all_candidates + if not candidate.version.is_prerelease + ] + + if not all_candidates: + continue + best_candidate = max(all_candidates, key=finder._candidate_sort_key) + remote_version = best_candidate.version + if best_candidate.location.is_wheel: + typ = 'wheel' + else: + typ = 'sdist' + # This is dirty but makes the rest of the code much cleaner + dist.latest_version = remote_version + dist.latest_filetype = typ + yield dist + + def get_outdated_packages(self, pre=False): + return [ + pkg for pkg in self.get_package_info(pre=pre) + if pkg.latest_version._version > pkg.parsed_version._version + ] + + def get_package_requirements(self): + from .vendor.pipdeptree import flatten, sorted_tree, build_dist_index, construct_tree + dist_index = build_dist_index(self.get_installed_packages()) + tree = sorted_tree(construct_tree(dist_index)) + branch_keys = set(r.key for r in flatten(tree.values())) + nodes = [p for p in tree.keys() if p.key not in branch_keys] + key_tree = dict((k.key, v) for k, v in tree.items()) + get_children = lambda n: key_tree.get(n.key, []) + + def aux(node, parent=None, chain=None): + if chain is None: + chain = [node.project_name] + + d = node.as_dict() + if parent: + d['required_version'] = node.version_spec if node.version_spec else 'Any' + else: + d['required_version'] = d['installed_version'] + + d['dependencies'] = [ + aux(c, parent=node, chain=chain+[c.project_name]) + for c in get_children(node) + if c.project_name not in chain + ] + + return d + return [aux(p) for p in nodes] + + def get_working_set(self): + """Retrieve the working set of installed packages for the environment. + + :return: The working set for the environment + :rtype: :class:`pkg_resources.WorkingSet` + """ + + working_set = pkg_resources.WorkingSet(self.sys_path) + return working_set + + def is_installed(self, pkgname): + """Given a package name, returns whether it is installed in the environment + + :param str pkgname: The name of a package + :return: Whether the supplied package is installed in the environment + :rtype: bool + """ + + return any(d for d in self.get_distributions() if d.project_name == pkgname) + + def run(self, cmd, cwd=os.curdir): + """Run a command with :class:`~subprocess.Popen` in the context of the environment + + :param cmd: A command to run in the environment + :type cmd: str or list + :param str cwd: The working directory in which to execute the command, defaults to :data:`os.curdir` + :return: A finished command object + :rtype: :class:`~subprocess.Popen` + """ + + c = None + with self.activated(): + script = vistir.cmdparse.Script.parse(cmd) + c = vistir.misc.run(script._parts, return_object=True, nospin=True, cwd=cwd) + return c + + def run_py(self, cmd, cwd=os.curdir): + """Run a python command in the enviornment context. + + :param cmd: A command to run in the environment - runs with `python -c` + :type cmd: str or list + :param str cwd: The working directory in which to execute the command, defaults to :data:`os.curdir` + :return: A finished command object + :rtype: :class:`~subprocess.Popen` + """ + + c = None + if isinstance(cmd, six.string_types): + script = vistir.cmdparse.Script.parse("{0} -c {1}".format(self.python, cmd)) + else: + script = vistir.cmdparse.Script.parse([self.python, "-c"] + list(cmd)) + with self.activated(): + c = vistir.misc.run(script._parts, return_object=True, nospin=True, cwd=cwd) + return c + + def run_activate_this(self): + """Runs the environment's inline activation script""" + if self.is_venv: + activate_this = os.path.join(self.scripts_dir, "activate_this.py") + if not os.path.isfile(activate_this): + raise OSError("No such file: {0!s}".format(activate_this)) + with open(activate_this, "r") as f: + code = compile(f.read(), activate_this, "exec") + exec(code, dict(__file__=activate_this)) + + @contextlib.contextmanager + def activated(self, include_extras=True, extra_dists=None): + """Helper context manager to activate the environment. + + This context manager will set the following variables for the duration + of its activation: + + * sys.prefix + * sys.path + * os.environ["VIRTUAL_ENV"] + * os.environ["PATH"] + + In addition, it will make any distributions passed into `extra_dists` available + on `sys.path` while inside the context manager, as well as making `passa` itself + available. + + The environment's `prefix` as well as `scripts_dir` properties are both prepended + to `os.environ["PATH"]` to ensure that calls to `~Environment.run()` use the + environment's path preferentially. + """ + + if not extra_dists: + extra_dists = [] + original_path = sys.path + original_prefix = sys.prefix + parent_path = vistir.compat.Path(__file__).absolute().parent + vendor_dir = parent_path.joinpath("vendor").as_posix() + patched_dir = parent_path.joinpath("patched").as_posix() + parent_path = parent_path.as_posix() + prefix = self.prefix.as_posix() + with vistir.contextmanagers.temp_environ(), vistir.contextmanagers.temp_path(): + os.environ["PATH"] = os.pathsep.join([ + vistir.compat.fs_str(self.scripts_dir), + vistir.compat.fs_str(self.prefix.as_posix()), + os.environ.get("PATH", "") + ]) + os.environ["PYTHONIOENCODING"] = vistir.compat.fs_str("utf-8") + os.environ["PYTHONDONTWRITEBYTECODE"] = vistir.compat.fs_str("1") + os.environ["PATH"] = self.base_paths["PATH"] + os.environ["PYTHONPATH"] = self.base_paths["PYTHONPATH"] + if self.is_venv: + os.environ["VIRTUAL_ENV"] = vistir.compat.fs_str(prefix) + sys.path = self.sys_path + sys.prefix = self.sys_prefix + site.addsitedir(self.base_paths["purelib"]) + if include_extras: + site.addsitedir(parent_path) + sys.path.extend([parent_path, patched_dir, vendor_dir]) + extra_dists = list(self.extra_dists) + extra_dists + for extra_dist in extra_dists: + if extra_dist not in self.get_working_set(): + extra_dist.activate(self.sys_path) + try: + yield + finally: + sys.path = original_path + sys.prefix = original_prefix + six.moves.reload_module(pkg_resources) + + @cached_property + def finders(self): + from pipenv.vendor.pythonfinder import Finder + finders = [ + Finder(path=self.base_paths["scripts"], global_search=gs, system=False) + for gs in (False, True) + ] + return finders + + @property + def finder(self): + return next(iter(self.finders), None) + + def which(self, search, as_path=True): + find = operator.methodcaller("which", search) + result = next(iter(filter(None, (find(finder) for finder in self.finders))), None) + if not result: + result = self._which(search) + else: + if as_path: + result = str(result.path) + return result + + def get_install_args(self, editable=False, setup_path=None): + install_arg = "install" if not editable else "develop" + install_keys = ["headers", "purelib", "platlib", "scripts", "data"] + install_args = [ + self.environment.python, "-u", "-c", SETUPTOOLS_SHIM % setup_path, + install_arg, "--single-version-externally-managed", "--no-deps", + "--prefix={0}".format(self.base_paths["prefix"]), "--no-warn-script-location" + ] + for key in install_keys: + install_args.append( + "--install-{0}={1}".format(key, self.base_paths[key]) + ) + return install_args + + def install(self, requirements): + if not isinstance(requirements, (tuple, list)): + requirements = [requirements,] + with self.get_finder() as finder: + args = [] + for format_control in ('no_binary', 'only_binary'): + formats = getattr(finder.format_control, format_control) + args.extend(('--' + format_control.replace('_', '-'), + ','.join(sorted(formats or {':none:'})))) + if finder.index_urls: + args.extend(['-i', finder.index_urls[0]]) + for extra_index in finder.index_urls[1:]: + args.extend(['--extra-index-url', extra_index]) + else: + args.append('--no-index') + for link in finder.find_links: + args.extend(['--find-links', link]) + for _, host, _ in finder.secure_origins: + args.extend(['--trusted-host', host]) + if finder.allow_all_prereleases: + args.append('--pre') + if finder.process_dependency_links: + args.append('--process-dependency-links') + args.append('--') + args.extend(requirements) + out, _ = vistir.misc.run(args, return_object=False, nospin=True, block=True, + combine_stderr=False) + + @contextlib.contextmanager + def uninstall(self, pkgname, *args, **kwargs): + """A context manager which allows uninstallation of packages from the environment + + :param str pkgname: The name of a package to uninstall + + >>> env = Environment("/path/to/env/root") + >>> with env.uninstall("pytz", auto_confirm=True, verbose=False) as uninstaller: + cleaned = uninstaller.paths + >>> if cleaned: + print("uninstalled packages: %s" % cleaned) + """ + + auto_confirm = kwargs.pop("auto_confirm", True) + verbose = kwargs.pop("verbose", False) + with self.activated(): + monkey_patch = next(iter( + dist for dist in self.base_working_set + if dist.project_name == "recursive-monkey-patch" + ), None) + if monkey_patch: + monkey_patch.activate() + pip_shims = self.safe_import("pip_shims") + pathset_base = pip_shims.UninstallPathSet + import recursive_monkey_patch + recursive_monkey_patch.monkey_patch( + PatchedUninstaller, pathset_base + ) + dist = next( + iter(filter(lambda d: d.project_name == pkgname, self.get_working_set())), + None + ) + pathset = pathset_base.from_dist(dist) + if pathset is not None: + pathset.remove(auto_confirm=auto_confirm, verbose=verbose) + try: + yield pathset + except Exception as e: + if pathset is not None: + pathset.rollback() + else: + if pathset is not None: + pathset.commit() + if pathset is None: + return + + +class PatchedUninstaller(object): + def _permitted(self, path): + return True + + +SETUPTOOLS_SHIM = ( + "import setuptools, tokenize;__file__=%r;" + "f=getattr(tokenize, 'open', open)(__file__);" + "code=f.read().replace('\\r\\n', '\\n');" + "f.close();" + "exec(compile(code, __file__, 'exec'))" +) diff --git a/pipenv/environments.py b/pipenv/environments.py index 4bfd937d..066c296d 100644 --- a/pipenv/environments.py +++ b/pipenv/environments.py @@ -1,13 +1,18 @@ +# -*- coding=utf-8 -*- + import os import sys from appdirs import user_cache_dir from .vendor.vistir.misc import fs_str +from ._compat import fix_utf8 # HACK: avoid resolver.py uses the wrong byte code files. # I hope I can remove this one day. os.environ["PYTHONDONTWRITEBYTECODE"] = fs_str("1") +PIPENV_IS_CI = bool("CI" in os.environ or "TF_BUILD" in os.environ) + # HACK: Prevent invalid shebangs with Homebrew-installed Python: # https://bugs.python.org/issue22490 os.environ.pop("__PYVENV_LAUNCHER__", None) @@ -68,7 +73,7 @@ PIPENV_HIDE_EMOJIS = bool(os.environ.get("PIPENV_HIDE_EMOJIS")) Default is to show emojis. This is automatically set on Windows. """ -if os.name == "nt": +if os.name == "nt" or PIPENV_IS_CI: PIPENV_HIDE_EMOJIS = True PIPENV_IGNORE_VIRTUALENVS = bool(os.environ.get("PIPENV_IGNORE_VIRTUALENVS")) @@ -94,7 +99,7 @@ Default is 3. See also ``PIPENV_NO_INHERIT``. PIPENV_MAX_RETRIES = int(os.environ.get( "PIPENV_MAX_RETRIES", - "1" if "CI" in os.environ else "0", + "1" if PIPENV_IS_CI else "0", )) """Specify how many retries Pipenv should attempt for network requests. @@ -128,9 +133,18 @@ PIPENV_NOSPIN = bool(os.environ.get("PIPENV_NOSPIN")) This can make the logs cleaner. Automatically set on Windows, and in CI environments. """ -if os.name == "nt" or "CI" in os.environ: +if PIPENV_IS_CI: PIPENV_NOSPIN = True +PIPENV_SPINNER = "dots" +"""Sets the default spinner type. + +Spinners are identitcal to the node.js spinners and can be found at +https://github.com/sindresorhus/cli-spinners +""" +if os.name == "nt": + PIPENV_SPINNER = "bouncingBar" + PIPENV_PIPFILE = os.environ.get("PIPENV_PIPFILE") """If set, this specifies a custom Pipfile location. @@ -197,6 +211,18 @@ Default is to prompt the user for an answer if the current command line session if interactive. """ +PIPENV_SKIP_LOCK = False +"""If set, Pipenv won't lock dependencies automatically. + +This might be desirable if a project has large number of dependencies, +because locking is an inherently slow operation. + +Default is to lock dependencies and update ``Pipfile.lock`` on each run. + +NOTE: This only affects the ``install`` and ``uninstall`` commands. +""" + +PIPENV_PYUP_API_KEY = os.environ.get("PIPENV_PYUP_API_KEY", "1ab8d58f-5122e025-83674263-bc1e79e0") # Internal, support running in a different Python from sys.executable. PIPENV_PYTHON = os.environ.get("PIPENV_PYTHON") @@ -248,3 +274,8 @@ def is_verbose(threshold=1): def is_quiet(threshold=-1): return PIPENV_VERBOSITY <= threshold + + +PIPENV_SPINNER_FAIL_TEXT = fix_utf8(u"✘ {0}") if not PIPENV_HIDE_EMOJIS else ("{0}") + +PIPENV_SPINNER_OK_TEXT = fix_utf8(u"✔ {0}") if not PIPENV_HIDE_EMOJIS else ("{0}") diff --git a/pipenv/exceptions.py b/pipenv/exceptions.py new file mode 100644 index 00000000..62e25d53 --- /dev/null +++ b/pipenv/exceptions.py @@ -0,0 +1,306 @@ +# -*- coding=utf-8 -*- + +import itertools +import sys + +from traceback import format_exception +from pprint import pformat + +import six + +from ._compat import fix_utf8 +from .patched import crayons +from . import environments +from .vendor.click.utils import echo as click_echo +from .vendor.click._compat import get_text_stderr +from .vendor.click.exceptions import ( + Abort, + BadOptionUsage, + BadParameter, + ClickException, + Exit, + FileError, + MissingParameter, + UsageError, +) +from .vendor.click.types import Path + + +def handle_exception(exc_type, exception, traceback, hook=sys.excepthook): + if environments.is_verbose() or not issubclass(exc_type, ClickException): + hook(exc_type, exception, traceback) + else: + exc = format_exception(exc_type, exception, traceback) + lines = itertools.chain.from_iterable([l.splitlines() for l in exc]) + lines = list(lines)[-11:-1] + for line in lines: + line = line.strip("'").strip('"').strip("\n").strip() + if not line.startswith("File"): + line = " {0}".format(line) + else: + line = " {0}".format(line) + line = "[pipenv.exceptions.{0!s}]: {1}".format( + exception.__class__.__name__, line + ) + click_echo(fix_utf8(line), err=True) + exception.show() + + +sys.excepthook = handle_exception + + +class PipenvException(ClickException): + message = "{0}: {{0}}".format(crayons.red("ERROR", bold=True)) + + def __init__(self, message=None, **kwargs): + if not message: + message = "Pipenv encountered a problem and had to exit." + extra = kwargs.pop("extra", []) + message = self.message.format(message) + ClickException.__init__(self, message) + self.extra = extra + + def show(self, file=None): + if file is None: + file = get_text_stderr() + if self.extra: + if isinstance(self.extra, six.string_types): + self.extra = [self.extra,] + for extra in self.extra: + extra = "[pipenv.exceptions.{0!s}]: {1}".format( + self.__class__.__name__, extra + ) + click_echo(extra, file=file) + click_echo(fix_utf8("{0}".format(self.message)), file=file) + + +class PipenvUsageError(UsageError): + + def __init__(self, message=None, ctx=None, **kwargs): + formatted_message = "{0}: {1}" + msg_prefix = crayons.red("ERROR:", bold=True) + if not message: + message = "Pipenv encountered a problem and had to exit." + message = formatted_message.format(msg_prefix, crayons.white(message, bold=True)) + self.message = message + extra = kwargs.pop("extra", []) + UsageError.__init__(self, fix_utf8(message), ctx) + self.extra = extra + + def show(self, file=None): + if file is None: + file = get_text_stderr() + color = None + if self.ctx is not None: + color = self.ctx.color + if self.extra: + if isinstance(self.extra, six.string_types): + self.extra = [self.extra,] + for extra in self.extra: + if color: + extra = getattr(crayons, color, "blue")(extra) + click_echo(fix_utf8(extra), file=file) + hint = '' + if (self.cmd is not None and + self.cmd.get_help_option(self.ctx) is not None): + hint = ('Try "%s %s" for help.\n' + % (self.ctx.command_path, self.ctx.help_option_names[0])) + if self.ctx is not None: + click_echo(self.ctx.get_usage() + '\n%s' % hint, file=file, color=color) + click_echo(self.message, file=file) + + +class PipenvFileError(FileError): + formatted_message = "{0} {{0}} {{1}}".format( + crayons.red("ERROR:", bold=True) + ) + + def __init__(self, filename, message=None, **kwargs): + extra = kwargs.pop("extra", []) + if not message: + message = crayons.white("Please ensure that the file exists!", bold=True) + message = self.formatted_message.format( + crayons.white("{0} not found!".format(filename), bold=True), + message + ) + FileError.__init__(self, filename=filename, hint=fix_utf8(message), **kwargs) + self.extra = extra + + def show(self, file=None): + if file is None: + file = get_text_stderr() + if self.extra: + if isinstance(self.extra, six.string_types): + self.extra = [self.extra,] + for extra in self.extra: + click_echo(fix_utf8(extra), file=file) + click_echo(self.message, file=file) + + +class PipfileNotFound(PipenvFileError): + def __init__(self, filename="Pipfile", extra=None, **kwargs): + extra = kwargs.pop("extra", []) + message = ("{0} {1}".format( + crayons.red("Aborting!", bold=True), + crayons.white("Please ensure that the file exists and is located in your" + " project root directory.", bold=True) + ) + ) + super(PipfileNotFound, self).__init__(filename, message=fix_utf8(message), extra=extra, **kwargs) + + +class LockfileNotFound(PipenvFileError): + def __init__(self, filename="Pipfile.lock", extra=None, **kwargs): + extra = kwargs.pop("extra", []) + message = "{0} {1} {2}".format( + crayons.white("You need to run", bold=True), + crayons.red("$ pipenv lock", bold=True), + crayons.white("before you can continue.", bold=True) + ) + super(LockfileNotFound, self).__init__(filename, message=fix_utf8(message), extra=extra, **kwargs) + + +class DeployException(PipenvUsageError): + def __init__(self, message=None, **kwargs): + if not message: + message = crayons.normal("Aborting deploy", bold=True) + extra = kwargs.pop("extra", []) + PipenvUsageError.__init__(message=fix_utf8(message), extra=extra, **kwargs) + + +class PipenvOptionsError(PipenvUsageError): + def __init__(self, option_name, message=None, ctx=None, **kwargs): + extra = kwargs.pop("extra", []) + PipenvUsageError.__init__(self, message=fix_utf8(message), ctx=ctx, **kwargs) + self.extra = extra + self.option_name = option_name + + +class SystemUsageError(PipenvOptionsError): + def __init__(self, option_name="system", message=None, ctx=None, **kwargs): + extra = kwargs.pop("extra", []) + extra += [ + "{0}: --system is intended to be used for Pipfile installation, " + "not installation of specific packages. Aborting.".format( + crayons.red("Warning", bold=True) + ), + ] + message = crayons.blue("See also: {0}".format(crayons.white("-deploy flag."))) + super(SystemUsageError, self).__init__(option_name, message=message, ctx=ctx, extra=extra, **kwargs) + + +class PipfileException(PipenvFileError): + def __init__(self, hint=None, **kwargs): + from .core import project + + if not hint: + hint = "{0} {1}".format(crayons.red("ERROR (PACKAGE NOT INSTALLED):"), hint) + filename = project.pipfile_location + extra = kwargs.pop("extra", []) + PipenvFileError.__init__(self, filename, fix_utf8(hint), extra=extra, **kwargs) + + +class SetupException(PipenvException): + def __init__(self, message=None, **kwargs): + PipenvException.__init__(self, message, **kwargs) + + +class VirtualenvException(PipenvException): + + def __init__(self, message=None, **kwargs): + if not message: + message = ( + "There was an unexpected error while activating your virtualenv. " + "Continuing anyway..." + ) + PipenvException.__init__(self, fix_utf8(message), **kwargs) + + +class VirtualenvActivationException(VirtualenvException): + def __init__(self, message=None, **kwargs): + if not message: + message = ( + "activate_this.py not found. Your environment is most certainly " + "not activated. Continuing anyway…" + ) + self.message = message + VirtualenvException.__init__(self, fix_utf8(message), **kwargs) + + +class VirtualenvCreationException(VirtualenvException): + def __init__(self, message=None, **kwargs): + if not message: + message = "Failed to create virtual environment." + self.message = message + VirtualenvException.__init__(self, fix_utf8(message), **kwargs) + + +class UninstallError(PipenvException): + def __init__(self, package, command, return_values, return_code, **kwargs): + extra = [crayons.blue("Attempted to run command: {0}".format( + crayons.yellow("$ {0}".format(command), bold=True) + )),] + extra.extend([crayons.blue(line.strip()) for line in return_values.splitlines()]) + if isinstance(package, (tuple, list)): + package = " ".join(package) + message = "{0} {1}...".format( + crayons.normal("Failed to uninstall package(s)"), + crayons.yellow(package, bold=True) + ) + self.exit_code = return_code + PipenvException.__init__(self, message=fix_utf8(message), extra=extra) + self.extra = extra + + +class InstallError(PipenvException): + def __init__(self, package, **kwargs): + message = "{0} {1}".format( + crayons.red("ERROR:", bold=True), + crayons.yellow("Package installation failed...") + ) + extra = kwargs.pop("extra", []) + PipenvException.__init__(self, message=fix_utf8(message), extra=extra, **kwargs) + + +class CacheError(PipenvException): + def __init__(self, path, **kwargs): + message = "{0} {1} {2}\n{0}".format( + crayons.red("ERROR:", bold=True), + crayons.blue("Corrupt cache file"), + crayons.white(path), + crayons.white('Consider trying "pipenv lock --clear" to clear the cache.') + ) + super(PipenvException, self).__init__(message=fix_utf8(message)) + + +class ResolutionFailure(PipenvException): + def __init__(self, message, no_version_found=False): + extra = ( + "{0}: Your dependencies could not be resolved. You likely have a " + "mismatch in your sub-dependencies.\n " + "First try clearing your dependency cache with {1}, then try the original command again.\n " + "Alternatively, you can use {2} to bypass this mechanism, then run " + "{3} to inspect the situation.\n " + "Hint: try {4} if it is a pre-release dependency." + "".format( + crayons.red("Warning", bold=True), + crayons.red("$ pipenv lock --clear"), + crayons.red("$ pipenv install --skip-lock"), + crayons.red("$ pipenv graph"), + crayons.red("$ pipenv lock --pre"), + ), + ) + if "no version found at all" in message: + no_version_found = True + message = "{0} {1}".format( + crayons.red("ERROR:", bold=True), crayons.yellow(message) + ) + if no_version_found: + message = "{0}\n{1}".format( + message, + crayons.blue( + "Please check your version specifier and version number. " + "See PEP440 for more information." + ) + ) + super(ResolutionFailure, self).__init__(fix_utf8(message), extra=extra) diff --git a/pipenv/patched/contoml/LICENSE b/pipenv/patched/contoml/LICENSE deleted file mode 100644 index 116fa4e5..00000000 --- a/pipenv/patched/contoml/LICENSE +++ /dev/null @@ -1,22 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2015 Jumpscale - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - diff --git a/pipenv/patched/contoml/__init__.py b/pipenv/patched/contoml/__init__.py deleted file mode 100644 index 9dba5e20..00000000 --- a/pipenv/patched/contoml/__init__.py +++ /dev/null @@ -1,48 +0,0 @@ -from ._version import VERSION - -__version__ = VERSION - - -def loads(text): - """ - Parses TOML text into a dict-like object and returns it. - """ - from prettytoml.parser import parse_tokens - from prettytoml.lexer import tokenize as lexer - from .file import TOMLFile - - tokens = tuple(lexer(text, is_top_level=True)) - elements = parse_tokens(tokens) - return TOMLFile(elements) - - -def load(file_path): - """ - Parses a TOML file into a dict-like object and returns it. - """ - return loads(open(file_path).read()) - - -def dumps(value): - """ - Dumps a data structure to TOML source code. - - The given value must be either a dict of dict values, a dict, or a TOML file constructed by this module. - """ - - from contoml.file.file import TOMLFile - - if not isinstance(value, TOMLFile): - raise RuntimeError("Can only dump a TOMLFile instance loaded by load() or loads()") - - return value.dumps() - - -def dump(obj, file_path, prettify=False): - """ - Dumps a data structure to the filesystem as TOML. - - The given value must be either a dict of dict values, a dict, or a TOML file constructed by this module. - """ - with open(file_path, 'w') as fp: - fp.write(dumps(obj)) diff --git a/pipenv/patched/contoml/_version.py b/pipenv/patched/contoml/_version.py deleted file mode 100644 index e0f15470..00000000 --- a/pipenv/patched/contoml/_version.py +++ /dev/null @@ -1 +0,0 @@ -VERSION = 'master' diff --git a/pipenv/patched/contoml/file/__init__.py b/pipenv/patched/contoml/file/__init__.py deleted file mode 100644 index 1aba5121..00000000 --- a/pipenv/patched/contoml/file/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ - - -from .file import TOMLFile diff --git a/pipenv/patched/contoml/file/array.py b/pipenv/patched/contoml/file/array.py deleted file mode 100644 index 40e5acb3..00000000 --- a/pipenv/patched/contoml/file/array.py +++ /dev/null @@ -1,40 +0,0 @@ -from prettytoml.elements.table import TableElement -from prettytoml.errors import InvalidValueError -from contoml.file.freshtable import FreshTable -from prettytoml import util - - -class ArrayOfTables(list): - - def __init__(self, toml_file, name, iterable=None): - if iterable: - list.__init__(self, iterable) - self._name = name - self._toml_file = toml_file - - def append(self, value): - if isinstance(value, dict): - table = FreshTable(parent=self, name=self._name, is_array=True) - table._append_to_parent() - index = len(self._toml_file[self._name]) - 1 - for key_seq, value in util.flatten_nested(value).items(): - # self._toml_file._setitem_with_key_seq((self._name, index) + key_seq, value) - self._toml_file._array_setitem_with_key_seq(self._name, index, key_seq, value) - # for k, v in value.items(): - # table[k] = v - else: - raise InvalidValueError('Can only append a dict to an array of tables') - - def __getitem__(self, item): - try: - return list.__getitem__(self, item) - except IndexError: - if item == len(self): - return FreshTable(parent=self, name=self._name, is_array=True) - else: - raise - - def append_fresh_table(self, fresh_table): - list.append(self, fresh_table) - if self._toml_file: - self._toml_file.append_fresh_table(fresh_table) diff --git a/pipenv/patched/contoml/file/cascadedict.py b/pipenv/patched/contoml/file/cascadedict.py deleted file mode 100644 index 4e97c060..00000000 --- a/pipenv/patched/contoml/file/cascadedict.py +++ /dev/null @@ -1,56 +0,0 @@ -import operator -from functools import reduce -from contoml.file import raw - - -class CascadeDict: - """ - A dict-like object made up of one or more other dict-like objects where querying for an item cascade-gets - it from all the internal dicts in order of their listing, and setting an item sets it on the first dict listed. - """ - - def __init__(self, *internal_dicts): - assert internal_dicts, 'internal_dicts cannot be empty' - self._internal_dicts = tuple(internal_dicts) - - def cascaded_with(self, one_more_dict): - """ - Returns another instance with one more dict cascaded at the end. - """ - return CascadeDict(self._internal_dicts, one_more_dict,) - - def __getitem__(self, item): - for d in self._internal_dicts: - try: - return d[item] - except KeyError: - pass - raise KeyError - - def __setitem__(self, key, value): - self._internal_dicts[0][key] = value - - def keys(self): - return set(reduce(operator.or_, (set(d.keys()) for d in self._internal_dicts))) - - def items(self): - all_items = reduce(operator.add, (list(d.items()) for d in reversed(self._internal_dicts))) - unique_items = {k: v for k, v in all_items}.items() - return tuple(unique_items) - - def __contains__(self, item): - for d in self._internal_dicts: - if item in d: - return True - return False - - @property - def neutralized(self): - return {k: raw.to_raw(v) for k, v in self.items()} - - @property - def primitive_value(self): - return self.neutralized - - def __repr__(self): - return repr(self.primitive_value) diff --git a/pipenv/patched/contoml/file/file.py b/pipenv/patched/contoml/file/file.py deleted file mode 100644 index 99ce1483..00000000 --- a/pipenv/patched/contoml/file/file.py +++ /dev/null @@ -1,293 +0,0 @@ -from prettytoml.errors import NoArrayFoundError, DuplicateKeysError, DuplicateTablesError -from contoml.file import structurer, toplevels, raw -from contoml.file.array import ArrayOfTables -from contoml.file.freshtable import FreshTable -import prettytoml.elements.factory as element_factory -import prettytoml.util as util - - -class TOMLFile: - """ - A TOMLFile object that tries its best to prserve formatting and order of mappings of the input source. - - Raises InvalidTOMLFileError on invalid input elements. - - Raises DuplicateKeysError, DuplicateTableError when appropriate. - """ - - def __init__(self, _elements): - self._elements = [] - self._navigable = {} - self.append_elements(_elements) - - def __getitem__(self, item): - try: - value = self._navigable[item] - if isinstance(value, (list, tuple)): - return ArrayOfTables(toml_file=self, name=item, iterable=value) - else: - return value - except KeyError: - return FreshTable(parent=self, name=item, is_array=False) - - def get(self, item, default=None): - """This was not here for who knows why.""" - - if item not in self: - return default - else: - return self.__getitem__(item) - - def __contains__(self, item): - return item in self.keys() - - def _setitem_with_key_seq(self, key_seq, value): - """ - Sets a the value in the TOML file located by the given key sequence. - - Example: - self._setitem(('key1', 'key2', 'key3'), 'text_value') - is equivalent to doing - self['key1']['key2']['key3'] = 'text_value' - """ - table = self - key_so_far = tuple() - for key in key_seq[:-1]: - key_so_far += (key,) - self._make_sure_table_exists(key_so_far) - table = table[key] - table[key_seq[-1]] = value - - def _array_setitem_with_key_seq(self, array_name, index, key_seq, value): - """ - Sets a the array value in the TOML file located by the given key sequence. - - Example: - self._array_setitem(array_name, index, ('key1', 'key2', 'key3'), 'text_value') - is equivalent to doing - self.array(array_name)[index]['key1']['key2']['key3'] = 'text_value' - """ - table = self.array(array_name)[index] - key_so_far = tuple() - for key in key_seq[:-1]: - key_so_far += (key,) - new_table = self._array_make_sure_table_exists(array_name, index, key_so_far) - if new_table is not None: - table = new_table - else: - table = table[key] - table[key_seq[-1]] = value - - def _make_sure_table_exists(self, name_seq): - """ - Makes sure the table with the full name comprising of name_seq exists. - """ - t = self - for key in name_seq[:-1]: - t = t[key] - name = name_seq[-1] - if name not in t: - self.append_elements([element_factory.create_table_header_element(name_seq), - element_factory.create_table({})]) - - def _array_make_sure_table_exists(self, array_name, index, name_seq): - """ - Makes sure the table with the full name comprising of name_seq exists. - """ - t = self[array_name][index] - for key in name_seq[:-1]: - t = t[key] - name = name_seq[-1] - if name not in t: - new_table = element_factory.create_table({}) - self.append_elements([element_factory.create_table_header_element((array_name,) + name_seq), new_table]) - return new_table - - def __delitem__(self, key): - table_element_index = self._elements.index(self._navigable[key]) - self._elements[table_element_index] = element_factory.create_table({}) - self._on_element_change() - - def __setitem__(self, key, value): - - # Setting an array-of-tables - if key and isinstance(value, (tuple, list)) and value and all(isinstance(v, dict) for v in value): - for table in value: - self.array(key).append(table) - - # Or setting a whole single table - elif isinstance(value, dict): - - if key and key in self: - del self[key] - - for key_seq, child_value in util.flatten_nested({key: value}).items(): - self._setitem_with_key_seq(key_seq, child_value) - - # if key in self._navigable: - # del self[key] - # index = self._elements.index(self._navigable[key]) - # self._elements = self._elements[:index] + [element_factory.create_table(value)] + self._elements[index+1:] - # else: - # if key: - # self._elements.append(element_factory.create_table_header_element(key)) - # self._elements.append(element_factory.create_table(value)) - - - # Or updating the anonymous section table - else: - # It's mea - self[''][key] = value - - self._on_element_change() - - def _detect_toplevels(self): - """ - Returns a sequence of TopLevel instances for the current state of this table. - """ - return tuple(e for e in toplevels.identify(self.elements) if isinstance(e, toplevels.Table)) - - def _update_table_fallbacks(self, table_toplevels): - """ - Updates the fallbacks on all the table elements to make relative table access possible. - - Raises DuplicateKeysError if appropriate. - """ - - if len(self.elements) <= 1: - return - - def parent_of(toplevel): - # Returns an TopLevel parent of the given entry, or None. - for parent_toplevel in table_toplevels: - if toplevel.name.sub_names[:-1] == parent_toplevel.name.sub_names: - return parent_toplevel - - for entry in table_toplevels: - if entry.name.is_qualified: - parent = parent_of(entry) - if parent: - child_name = entry.name.without_prefix(parent.name) - parent.table_element.set_fallback({child_name.sub_names[0]: entry.table_element}) - - def _recreate_navigable(self): - if self._elements: - self._navigable = structurer.structure(toplevels.identify(self._elements)) - - def array(self, name): - """ - Returns the array of tables with the given name. - """ - if name in self._navigable: - if isinstance(self._navigable[name], (list, tuple)): - return self[name] - else: - raise NoArrayFoundError - else: - return ArrayOfTables(toml_file=self, name=name) - - def _on_element_change(self): - self._recreate_navigable() - - table_toplevels = self._detect_toplevels() - self._update_table_fallbacks(table_toplevels) - - def append_elements(self, elements): - """ - Appends more elements to the contained internal elements. - """ - self._elements = self._elements + list(elements) - self._on_element_change() - - def prepend_elements(self, elements): - """ - Prepends more elements to the contained internal elements. - """ - self._elements = list(elements) + self._elements - self._on_element_change() - - def dumps(self): - """ - Returns the TOML file serialized back to str. - """ - return ''.join(element.serialized() for element in self._elements) - - def dump(self, file_path): - with open(file_path, mode='w') as fp: - fp.write(self.dumps()) - - def keys(self): - return set(self._navigable.keys()) | {''} - - def values(self): - return self._navigable.values() - - def items(self): - items = self._navigable.items() - - def has_anonymous_entry(): - return any(key == '' for (key, _) in items) - - if has_anonymous_entry(): - return items - else: - return list(items) + [('', self[''])] - - @property - def primitive(self): - """ - Returns a primitive object representation for this container (which is a dict). - - WARNING: The returned container does not contain any markup or formatting metadata. - """ - raw_container = raw.to_raw(self._navigable) - - # Collapsing the anonymous table onto the top-level container is present - if '' in raw_container: - raw_container.update(raw_container['']) - del raw_container[''] - - return raw_container - - def append_fresh_table(self, fresh_table): - """ - Gets called by FreshTable instances when they get written to. - """ - if fresh_table.name: - elements = [] - if fresh_table.is_array: - elements += [element_factory.create_array_of_tables_header_element(fresh_table.name)] - else: - elements += [element_factory.create_table_header_element(fresh_table.name)] - - elements += [fresh_table, element_factory.create_newline_element()] - self.append_elements(elements) - - else: - # It's an anonymous table - self.prepend_elements([fresh_table, element_factory.create_newline_element()]) - - @property - def elements(self): - return self._elements - - def __str__(self): - - is_empty = (not self['']) and (not tuple(k for k in self.keys() if k)) - - def key_name(key): - return '[ANONYMOUS]' if not key else key - - def pair(key, value): - return '%s = %s' % (key_name(key), str(value)) - - content_text = '' if is_empty else \ - '\n\t' + ',\n\t'.join(pair(k, v) for (k, v) in self.items() if v) + '\n' - - return "TOMLFile{%s}" % content_text - - def __repr__(self): - return str(self) - - - diff --git a/pipenv/patched/contoml/file/freshtable.py b/pipenv/patched/contoml/file/freshtable.py deleted file mode 100644 index f28e2f74..00000000 --- a/pipenv/patched/contoml/file/freshtable.py +++ /dev/null @@ -1,45 +0,0 @@ -from prettytoml.elements.table import TableElement - - -class FreshTable(TableElement): - """ - A fresh TableElement that appended itself to each of parents when it first gets written to at most once. - - parents is a sequence of objects providing an append_fresh_table(TableElement) method - """ - - def __init__(self, parent, name, is_array=False): - TableElement.__init__(self, sub_elements=[]) - - self._parent = parent - self._name = name - self._is_array = is_array - - # As long as this flag is false, setitem() operations will append the table header and this table - # to the toml_file's elements - self.__appended = False - - @property - def name(self): - return self._name - - @property - def is_array(self): - return self._is_array - - def _append_to_parent(self): - """ - Causes this ephemeral table to be persisted on the TOMLFile. - """ - - if self.__appended: - return - - if self._parent is not None: - self._parent.append_fresh_table(self) - - self.__appended = True - - def __setitem__(self, key, value): - TableElement.__setitem__(self, key, value) - self._append_to_parent() diff --git a/pipenv/patched/contoml/file/peekableit.py b/pipenv/patched/contoml/file/peekableit.py deleted file mode 100644 index b5658a71..00000000 --- a/pipenv/patched/contoml/file/peekableit.py +++ /dev/null @@ -1,30 +0,0 @@ -import itertools - - -class PeekableIterator: - - # Returned by peek() when the iterator is exhausted. Truthiness is False. - Nothing = tuple() - - def __init__(self, iter): - self._iter = iter - - def __next__(self): - return next(self._iter) - - def next(self): - return self.__next__() - - def __iter__(self): - return self - - def peek(self): - """ - Returns PeekableIterator.Nothing when the iterator is exhausted. - """ - try: - v = next(self._iter) - self._iter = itertools.chain((v,), self._iter) - return v - except StopIteration: - return PeekableIterator.Nothing diff --git a/pipenv/patched/contoml/file/raw.py b/pipenv/patched/contoml/file/raw.py deleted file mode 100644 index 8cffdb6e..00000000 --- a/pipenv/patched/contoml/file/raw.py +++ /dev/null @@ -1,16 +0,0 @@ -from prettytoml.elements.abstracttable import AbstractTable - - -def to_raw(x): - from contoml.file.cascadedict import CascadeDict - - if isinstance(x, AbstractTable): - return x.primitive_value - elif isinstance(x, CascadeDict): - return x.neutralized - elif isinstance(x, (list, tuple)): - return [to_raw(y) for y in x] - elif isinstance(x, dict): - return {k: to_raw(v) for (k, v) in x.items()} - else: - return x diff --git a/pipenv/patched/contoml/file/structurer.py b/pipenv/patched/contoml/file/structurer.py deleted file mode 100644 index 72d002cd..00000000 --- a/pipenv/patched/contoml/file/structurer.py +++ /dev/null @@ -1,116 +0,0 @@ -from contoml.file import toplevels -from contoml.file.cascadedict import CascadeDict - - -class NamedDict(dict): - """ - A dict that can use Name instances as keys. - """ - - def __init__(self, other_dict=None): - dict.__init__(self) - if other_dict: - for k, v in other_dict.items(): - self[k] = v - - def __setitem__(self, key, value): - """ - key can be an Name instance. - - When key is a path in the form of an Name instance, all the parents and grandparents of the value are - created along the way as instances of NamedDict. If the parent of the value exists, it is replaced with a - CascadeDict() that cascades the old parent value with a new NamedDict that contains the given child name - and value. - """ - if isinstance(key, toplevels.Name): - - if len(key.sub_names) == 1: - name = key.sub_names[0] - if name in self: - self[name] = CascadeDict(self[name], value) - else: - self[name] = value - - elif len(key.sub_names) > 1: - name = key.sub_names[0] - rest_of_key = key.drop(1) - if name in self: - named_dict = NamedDict() - named_dict[rest_of_key] = value - self[name] = CascadeDict(self[name], named_dict) - else: - self[name] = NamedDict() - self[name][rest_of_key] = value - else: - return dict.__setitem__(self, key, value) - - def __contains__(self, item): - try: - _ = self[item] - return True - except KeyError: - return False - - def append(self, key, value): - """ - Makes sure the value pointed to by key exists and is a list and appends the given value to it. - """ - if key in self: - self[key].append(value) - else: - self[key] = [value] - - def __getitem__(self, item): - - if isinstance(item, toplevels.Name): - d = self - for name in item.sub_names: - d = d[name] - return d - else: - return dict.__getitem__(self, item) - - -def structure(table_toplevels): - """ - Accepts an ordered sequence of TopLevel instances and returns a navigable object structure representation of the - TOML file. - """ - - table_toplevels = tuple(table_toplevels) - obj = NamedDict() - - last_array_of_tables = None # The Name of the last array-of-tables header - - for toplevel in table_toplevels: - - if isinstance(toplevel, toplevels.AnonymousTable): - obj[''] = toplevel.table_element - - elif isinstance(toplevel, toplevels.Table): - if last_array_of_tables and toplevel.name.is_prefixed_with(last_array_of_tables): - seq = obj[last_array_of_tables] - unprefixed_name = toplevel.name.without_prefix(last_array_of_tables) - - seq[-1] = CascadeDict(seq[-1], NamedDict({unprefixed_name: toplevel.table_element})) - else: - obj[toplevel.name] = toplevel.table_element - else: # It's an ArrayOfTables - - if last_array_of_tables and toplevel.name != last_array_of_tables and \ - toplevel.name.is_prefixed_with(last_array_of_tables): - - seq = obj[last_array_of_tables] - unprefixed_name = toplevel.name.without_prefix(last_array_of_tables) - - if unprefixed_name in seq[-1]: - seq[-1][unprefixed_name].append(toplevel.table_element) - else: - cascaded_with = NamedDict({unprefixed_name: [toplevel.table_element]}) - seq[-1] = CascadeDict(seq[-1], cascaded_with) - - else: - obj.append(toplevel.name, toplevel.table_element) - last_array_of_tables = toplevel.name - - return obj diff --git a/pipenv/patched/contoml/file/test_cascadedict.py b/pipenv/patched/contoml/file/test_cascadedict.py deleted file mode 100644 index d692711e..00000000 --- a/pipenv/patched/contoml/file/test_cascadedict.py +++ /dev/null @@ -1,25 +0,0 @@ -from contoml.file.cascadedict import CascadeDict - - -def test_cascadedict(): - - d1 = {'a': 1, 'b': 2, 'c': 3} - d2 = {'b': 12, 'e': 4, 'f': 5} - - cascade = CascadeDict(d1, d2) - - # Test querying - assert cascade['a'] == 1 - assert cascade['b'] == 2 - assert cascade['c'] == 3 - assert cascade['e'] == 4 - assert cascade.keys() == {'a', 'b', 'c', 'e', 'f'} - assert set(cascade.items()) == {('a', 1), ('b', 2), ('c', 3), ('e', 4), ('f', 5)} - - # Test mutating - cascade['a'] = 11 - cascade['f'] = 'fff' - cascade['super'] = 'man' - assert d1['a'] == 11 - assert d1['super'] == 'man' - assert d1['f'] == 'fff' diff --git a/pipenv/patched/contoml/file/test_entries.py b/pipenv/patched/contoml/file/test_entries.py deleted file mode 100644 index 25584e82..00000000 --- a/pipenv/patched/contoml/file/test_entries.py +++ /dev/null @@ -1,20 +0,0 @@ -from prettytoml import parser, lexer -from contoml.file import toplevels - - -def test_entry_extraction(): - text = open('sample.toml').read() - elements = parser.parse_tokens(lexer.tokenize(text)) - - e = tuple(toplevels.identify(elements)) - - assert len(e) == 13 - assert isinstance(e[0], toplevels.AnonymousTable) - - -def test_entry_names(): - name_a = toplevels.Name(('super', 'sub1')) - name_b = toplevels.Name(('super', 'sub1', 'sub2', 'sub3')) - - assert name_b.is_prefixed_with(name_a) - assert name_b.without_prefix(name_a).sub_names == ('sub2', 'sub3') diff --git a/pipenv/patched/contoml/file/test_peekableit.py b/pipenv/patched/contoml/file/test_peekableit.py deleted file mode 100644 index 5c053a38..00000000 --- a/pipenv/patched/contoml/file/test_peekableit.py +++ /dev/null @@ -1,12 +0,0 @@ -from contoml.file.peekableit import PeekableIterator - - -def test_peekable_iterator(): - - peekable = PeekableIterator(i for i in (1, 2, 3, 4)) - - assert peekable.peek() == 1 - assert peekable.peek() == 1 - assert peekable.peek() == 1 - - assert [next(peekable), next(peekable), next(peekable), next(peekable)] == [1, 2, 3, 4] diff --git a/pipenv/patched/contoml/file/test_structurer.py b/pipenv/patched/contoml/file/test_structurer.py deleted file mode 100644 index b3ea4b4e..00000000 --- a/pipenv/patched/contoml/file/test_structurer.py +++ /dev/null @@ -1,41 +0,0 @@ -from prettytoml import lexer, parser -from contoml.file import toplevels -from prettytoml.parser import elementsanitizer -from contoml.file.structurer import NamedDict, structure -from prettytoml.parser.tokenstream import TokenStream - - -def test_NamedDict(): - - d = NamedDict() - - d[toplevels.Name(('super', 'sub1', 'sub2'))] = {'sub3': 12} - d[toplevels.Name(('super', 'sub1', 'sub2'))]['sub4'] = 42 - - assert d[toplevels.Name(('super', 'sub1', 'sub2', 'sub3'))] == 12 - assert d[toplevels.Name(('super', 'sub1', 'sub2', 'sub4'))] == 42 - - -def test_structure(): - tokens = lexer.tokenize(open('sample.toml').read()) - elements = elementsanitizer.sanitize(parser.parse_tokens(tokens)) - entries_ = tuple(toplevels.identify(elements)) - - s = structure(entries_) - - assert s['']['title'] == 'TOML Example' - assert s['owner']['name'] == 'Tom Preston-Werner' - assert s['database']['ports'][1] == 8001 - assert s['servers']['alpha']['dc'] == 'eqdc10' - assert s['clients']['data'][1][0] == 1 - assert s['clients']['key3'] == 'The quick brown fox jumps over the lazy dog.' - - assert s['fruit'][0]['name'] == 'apple' - assert s['fruit'][0]['physical']['color'] == 'red' - assert s['fruit'][0]['physical']['shape'] == 'round' - assert s['fruit'][0]['variety'][0]['name'] == 'red delicious' - assert s['fruit'][0]['variety'][1]['name'] == 'granny smith' - - assert s['fruit'][1]['name'] == 'banana' - assert s['fruit'][1]['variety'][0]['name'] == 'plantain' - assert s['fruit'][1]['variety'][0]['points'][2]['y'] == 4 diff --git a/pipenv/patched/contoml/file/toplevels.py b/pipenv/patched/contoml/file/toplevels.py deleted file mode 100644 index 64038072..00000000 --- a/pipenv/patched/contoml/file/toplevels.py +++ /dev/null @@ -1,142 +0,0 @@ -""" - Top-level entries in a TOML file. -""" - -from prettytoml import elements -from prettytoml.elements import TableElement, TableHeaderElement -from .peekableit import PeekableIterator - - -class TopLevel: - """ - A abstract top-level entry. - """ - - def __init__(self, names, table_element): - self._table_element = table_element - self._names = Name(names) - - @property - def table_element(self): - return self._table_element - - @property - def name(self): - """ - The distinct name of a table entry as an Name instance. - """ - return self._names - - -class Name: - - def __init__(self, names): - self._names = names - - @property - def sub_names(self): - return self._names - - def drop(self, n=0): - """ - Returns the name after dropping the first n entries of it. - """ - return Name(names=self._names[n:]) - - def is_prefixed_with(self, names): - if isinstance(names, Name): - return self.is_prefixed_with(names.sub_names) - - for i, name in enumerate(names): - if self._names[i] != name: - return False - return True - - def without_prefix(self, names): - if isinstance(names, Name): - return self.without_prefix(names.sub_names) - - for i, name in enumerate(names): - if name != self._names[i]: - return Name(self._names[i:]) - return Name(names=self.sub_names[len(names):]) - - @property - def is_qualified(self): - return len(self._names) > 1 - - def __str__(self): - return '.'.join(self.sub_names) - - def __hash__(self): - return hash(str(self)) - - def __eq__(self, other): - return str(self) == str(other) - - def __ne__(self, other): - return not self.__eq__(other) - - -class AnonymousTable(TopLevel): - - def __init__(self, table_element): - TopLevel.__init__(self, ('',), table_element) - - -class Table(TopLevel): - - def __init__(self, names, table_element): - TopLevel.__init__(self, names=names, table_element=table_element) - - -class ArrayOfTables(TopLevel): - - def __init__(self, names, table_element): - TopLevel.__init__(self, names=names, table_element=table_element) - - -def _validate_file_elements(file_elements): - pass - - -def identify(file_elements): - """ - Outputs an ordered sequence of instances of TopLevel types. - - Elements start with an optional TableElement, followed by zero or more pairs of (TableHeaderElement, TableElement). - """ - - if not file_elements: - return - - _validate_file_elements(file_elements) - - # An iterator over enumerate(the non-metadata) elements - iterator = PeekableIterator((element_i, element) for (element_i, element) in enumerate(file_elements) - if element.type != elements.TYPE_METADATA) - - try: - _, first_element = iterator.peek() - if isinstance(first_element, TableElement): - iterator.next() - yield AnonymousTable(first_element) - except KeyError: - pass - except StopIteration: - return - - for element_i, element in iterator: - - if not isinstance(element, TableHeaderElement): - continue - - # If TableHeader of a regular table, return Table following it - if not element.is_array_of_tables: - table_element_i, table_element = next(iterator) - yield Table(names=element.names, table_element=table_element) - - # If TableHeader of an array of tables, do your thing - else: - table_element_i, table_element = next(iterator) - yield ArrayOfTables(names=element.names, table_element=table_element) diff --git a/pipenv/patched/notpip/__init__.py b/pipenv/patched/notpip/__init__.py index 9227d0ea..ae265fa7 100644 --- a/pipenv/patched/notpip/__init__.py +++ b/pipenv/patched/notpip/__init__.py @@ -1 +1 @@ -__version__ = "18.0" +__version__ = "18.1" diff --git a/pipenv/patched/notpip/_internal/__init__.py b/pipenv/patched/notpip/_internal/__init__.py index dcd0937e..6d223928 100644 --- a/pipenv/patched/notpip/_internal/__init__.py +++ b/pipenv/patched/notpip/_internal/__init__.py @@ -4,7 +4,6 @@ from __future__ import absolute_import import locale import logging import os -import optparse import warnings import sys @@ -38,17 +37,12 @@ else: else: securetransport.inject_into_urllib3() -from pipenv.patched.notpip import __version__ -from pipenv.patched.notpip._internal import cmdoptions -from pipenv.patched.notpip._internal.exceptions import CommandError, PipError -from pipenv.patched.notpip._internal.utils.misc import get_installed_distributions, get_prog +from pipenv.patched.notpip._internal.cli.autocompletion import autocomplete +from pipenv.patched.notpip._internal.cli.main_parser import parse_command +from pipenv.patched.notpip._internal.commands import commands_dict +from pipenv.patched.notpip._internal.exceptions import PipError from pipenv.patched.notpip._internal.utils import deprecation from pipenv.patched.notpip._internal.vcs import git, mercurial, subversion, bazaar # noqa -from pipenv.patched.notpip._internal.baseparser import ( - ConfigOptionParser, UpdatingDefaultsHelpFormatter, -) -from pipenv.patched.notpip._internal.commands import get_summaries, get_similar_commands -from pipenv.patched.notpip._internal.commands import commands_dict from pipenv.patched.notpip._vendor.urllib3.exceptions import InsecureRequestWarning logger = logging.getLogger(__name__) @@ -57,232 +51,6 @@ logger = logging.getLogger(__name__) warnings.filterwarnings("ignore", category=InsecureRequestWarning) -def autocomplete(): - """Command and option completion for the main option parser (and options) - and its subcommands (and options). - - Enable by sourcing one of the completion shell scripts (bash, zsh or fish). - """ - # Don't complete if user hasn't sourced bash_completion file. - if 'PIP_AUTO_COMPLETE' not in os.environ: - return - cwords = os.environ['COMP_WORDS'].split()[1:] - cword = int(os.environ['COMP_CWORD']) - try: - current = cwords[cword - 1] - except IndexError: - current = '' - - subcommands = [cmd for cmd, summary in get_summaries()] - options = [] - # subcommand - try: - subcommand_name = [w for w in cwords if w in subcommands][0] - except IndexError: - subcommand_name = None - - parser = create_main_parser() - # subcommand options - if subcommand_name: - # special case: 'help' subcommand has no options - if subcommand_name == 'help': - sys.exit(1) - # special case: list locally installed dists for show and uninstall - should_list_installed = ( - subcommand_name in ['show', 'uninstall'] and - not current.startswith('-') - ) - if should_list_installed: - installed = [] - lc = current.lower() - for dist in get_installed_distributions(local_only=True): - if dist.key.startswith(lc) and dist.key not in cwords[1:]: - installed.append(dist.key) - # if there are no dists installed, fall back to option completion - if installed: - for dist in installed: - print(dist) - sys.exit(1) - - subcommand = commands_dict[subcommand_name]() - - for opt in subcommand.parser.option_list_all: - if opt.help != optparse.SUPPRESS_HELP: - for opt_str in opt._long_opts + opt._short_opts: - options.append((opt_str, opt.nargs)) - - # filter out previously specified options from available options - prev_opts = [x.split('=')[0] for x in cwords[1:cword - 1]] - options = [(x, v) for (x, v) in options if x not in prev_opts] - # filter options by current input - options = [(k, v) for k, v in options if k.startswith(current)] - # get completion type given cwords and available subcommand options - completion_type = get_path_completion_type( - cwords, cword, subcommand.parser.option_list_all, - ) - # get completion files and directories if ``completion_type`` is - # ````, ```` or ```` - if completion_type: - options = auto_complete_paths(current, completion_type) - options = ((opt, 0) for opt in options) - for option in options: - opt_label = option[0] - # append '=' to options which require args - if option[1] and option[0][:2] == "--": - opt_label += '=' - print(opt_label) - else: - # show main parser options only when necessary - - opts = [i.option_list for i in parser.option_groups] - opts.append(parser.option_list) - opts = (o for it in opts for o in it) - if current.startswith('-'): - for opt in opts: - if opt.help != optparse.SUPPRESS_HELP: - subcommands += opt._long_opts + opt._short_opts - else: - # get completion type given cwords and all available options - completion_type = get_path_completion_type(cwords, cword, opts) - if completion_type: - subcommands = auto_complete_paths(current, completion_type) - - print(' '.join([x for x in subcommands if x.startswith(current)])) - sys.exit(1) - - -def get_path_completion_type(cwords, cword, opts): - """Get the type of path completion (``file``, ``dir``, ``path`` or None) - - :param cwords: same as the environmental variable ``COMP_WORDS`` - :param cword: same as the environmental variable ``COMP_CWORD`` - :param opts: The available options to check - :return: path completion type (``file``, ``dir``, ``path`` or None) - """ - if cword < 2 or not cwords[cword - 2].startswith('-'): - return - for opt in opts: - if opt.help == optparse.SUPPRESS_HELP: - continue - for o in str(opt).split('/'): - if cwords[cword - 2].split('=')[0] == o: - if any(x in ('path', 'file', 'dir') - for x in opt.metavar.split('/')): - return opt.metavar - - -def auto_complete_paths(current, completion_type): - """If ``completion_type`` is ``file`` or ``path``, list all regular files - and directories starting with ``current``; otherwise only list directories - starting with ``current``. - - :param current: The word to be completed - :param completion_type: path completion type(`file`, `path` or `dir`)i - :return: A generator of regular files and/or directories - """ - directory, filename = os.path.split(current) - current_path = os.path.abspath(directory) - # Don't complete paths if they can't be accessed - if not os.access(current_path, os.R_OK): - return - filename = os.path.normcase(filename) - # list all files that start with ``filename`` - file_list = (x for x in os.listdir(current_path) - if os.path.normcase(x).startswith(filename)) - for f in file_list: - opt = os.path.join(current_path, f) - comp_file = os.path.normcase(os.path.join(directory, f)) - # complete regular files when there is not ```` after option - # complete directories when there is ````, ```` or - # ````after option - if completion_type != 'dir' and os.path.isfile(opt): - yield comp_file - elif os.path.isdir(opt): - yield os.path.join(comp_file, '') - - -def create_main_parser(): - parser_kw = { - 'usage': '\n%prog [options]', - 'add_help_option': False, - 'formatter': UpdatingDefaultsHelpFormatter(), - 'name': 'global', - 'prog': get_prog(), - } - - parser = ConfigOptionParser(**parser_kw) - parser.disable_interspersed_args() - - pip_pkg_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - parser.version = 'pip %s from %s (python %s)' % ( - __version__, pip_pkg_dir, sys.version[:3], - ) - - # add the general options - gen_opts = cmdoptions.make_option_group(cmdoptions.general_group, parser) - parser.add_option_group(gen_opts) - - parser.main = True # so the help formatter knows - - # create command listing for description - command_summaries = get_summaries() - description = [''] + ['%-27s %s' % (i, j) for i, j in command_summaries] - parser.description = '\n'.join(description) - - return parser - - -def parseopts(args): - parser = create_main_parser() - - # Note: parser calls disable_interspersed_args(), so the result of this - # call is to split the initial args into the general options before the - # subcommand and everything else. - # For example: - # args: ['--timeout=5', 'install', '--user', 'INITools'] - # general_options: ['--timeout==5'] - # args_else: ['install', '--user', 'INITools'] - general_options, args_else = parser.parse_args(args) - - # --version - if general_options.version: - sys.stdout.write(parser.version) - sys.stdout.write(os.linesep) - sys.exit() - - # pip || pip help -> print_help() - if not args_else or (args_else[0] == 'help' and len(args_else) == 1): - parser.print_help() - sys.exit() - - # the subcommand name - cmd_name = args_else[0] - - if cmd_name not in commands_dict: - guess = get_similar_commands(cmd_name) - - msg = ['unknown command "%s"' % cmd_name] - if guess: - msg.append('maybe you meant "%s"' % guess) - - raise CommandError(' - '.join(msg)) - - # all the args without the subcommand - cmd_args = args[:] - cmd_args.remove(cmd_name) - - return cmd_name, cmd_args - - -def check_isolated(args): - isolated = False - - if "--isolated" in args: - isolated = True - - return isolated - - def main(args=None): if args is None: args = sys.argv[1:] @@ -293,7 +61,7 @@ def main(args=None): autocomplete() try: - cmd_name, cmd_args = parseopts(args) + cmd_name, cmd_args = parse_command(args) except PipError as exc: sys.stderr.write("ERROR: %s" % exc) sys.stderr.write(os.linesep) @@ -306,5 +74,5 @@ def main(args=None): except locale.Error as e: # setlocale can apparently crash if locale are uninitialized logger.debug("Ignoring error %s when setting locale", e) - command = commands_dict[cmd_name](isolated=check_isolated(cmd_args)) + command = commands_dict[cmd_name](isolated=("--isolated" in cmd_args)) return command.main(cmd_args) diff --git a/pipenv/patched/notpip/_internal/build_env.py b/pipenv/patched/notpip/_internal/build_env.py index 1d351b5c..6d696fbd 100644 --- a/pipenv/patched/notpip/_internal/build_env.py +++ b/pipenv/patched/notpip/_internal/build_env.py @@ -7,6 +7,8 @@ import sys from distutils.sysconfig import get_python_lib from sysconfig import get_paths +from pipenv.patched.notpip._vendor.pkg_resources import Requirement, VersionConflict, WorkingSet + from pipenv.patched.notpip._internal.utils.misc import call_subprocess from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory from pipenv.patched.notpip._internal.utils.ui import open_spinner @@ -75,6 +77,20 @@ class BuildEnvironment(object): def cleanup(self): self._temp_dir.cleanup() + def missing_requirements(self, reqs): + """Return a list of the requirements from reqs that are not present + """ + missing = [] + with self: + ws = WorkingSet(os.environ["PYTHONPATH"].split(os.pathsep)) + for req in reqs: + try: + if ws.find(Requirement.parse(req)) is None: + missing.append(req) + except VersionConflict: + missing.append(req) + return missing + def install_requirements(self, finder, requirements, message): args = [ sys.executable, '-m', 'pip', 'install', '--ignore-installed', diff --git a/pipenv/patched/notpip/_internal/cache.py b/pipenv/patched/notpip/_internal/cache.py index a3a28fd8..d91b8170 100644 --- a/pipenv/patched/notpip/_internal/cache.py +++ b/pipenv/patched/notpip/_internal/cache.py @@ -8,9 +8,9 @@ import os from pipenv.patched.notpip._vendor.packaging.utils import canonicalize_name -from pipenv.patched.notpip._internal import index -from pipenv.patched.notpip._internal.compat import expanduser from pipenv.patched.notpip._internal.download import path_to_url +from pipenv.patched.notpip._internal.models.link import Link +from pipenv.patched.notpip._internal.utils.compat import expanduser from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory from pipenv.patched.notpip._internal.wheel import InvalidWheelFilename, Wheel @@ -22,7 +22,7 @@ class Cache(object): :param cache_dir: The root of the cache. - :param format_control: A pip.index.FormatControl object to limit + :param format_control: An object of FormatControl class to limit binaries being read from the cache. :param allowed_formats: which formats of files the cache should store. ('binary' and 'source' are the only allowed values) @@ -72,8 +72,8 @@ class Cache(object): return [] canonical_name = canonicalize_name(package_name) - formats = index.fmt_ctl_formats( - self.format_control, canonical_name + formats = self.format_control.get_allowed_formats( + canonical_name ) if not self.allowed_formats.intersection(formats): return [] @@ -101,7 +101,7 @@ class Cache(object): root = self.get_path_for_link(link) path = os.path.join(root, candidate) - return index.Link(path_to_url(path)) + return Link(path_to_url(path)) def cleanup(self): pass diff --git a/pipenv/patched/notpip/_internal/cli/__init__.py b/pipenv/patched/notpip/_internal/cli/__init__.py new file mode 100644 index 00000000..e589bb91 --- /dev/null +++ b/pipenv/patched/notpip/_internal/cli/__init__.py @@ -0,0 +1,4 @@ +"""Subpackage containing all of pip's command line interface related code +""" + +# This file intentionally does not import submodules diff --git a/pipenv/patched/notpip/_internal/cli/autocompletion.py b/pipenv/patched/notpip/_internal/cli/autocompletion.py new file mode 100644 index 00000000..15b560a1 --- /dev/null +++ b/pipenv/patched/notpip/_internal/cli/autocompletion.py @@ -0,0 +1,152 @@ +"""Logic that powers autocompletion installed by ``pip completion``. +""" + +import optparse +import os +import sys + +from pipenv.patched.notpip._internal.cli.main_parser import create_main_parser +from pipenv.patched.notpip._internal.commands import commands_dict, get_summaries +from pipenv.patched.notpip._internal.utils.misc import get_installed_distributions + + +def autocomplete(): + """Entry Point for completion of main and subcommand options. + """ + # Don't complete if user hasn't sourced bash_completion file. + if 'PIP_AUTO_COMPLETE' not in os.environ: + return + cwords = os.environ['COMP_WORDS'].split()[1:] + cword = int(os.environ['COMP_CWORD']) + try: + current = cwords[cword - 1] + except IndexError: + current = '' + + subcommands = [cmd for cmd, summary in get_summaries()] + options = [] + # subcommand + try: + subcommand_name = [w for w in cwords if w in subcommands][0] + except IndexError: + subcommand_name = None + + parser = create_main_parser() + # subcommand options + if subcommand_name: + # special case: 'help' subcommand has no options + if subcommand_name == 'help': + sys.exit(1) + # special case: list locally installed dists for show and uninstall + should_list_installed = ( + subcommand_name in ['show', 'uninstall'] and + not current.startswith('-') + ) + if should_list_installed: + installed = [] + lc = current.lower() + for dist in get_installed_distributions(local_only=True): + if dist.key.startswith(lc) and dist.key not in cwords[1:]: + installed.append(dist.key) + # if there are no dists installed, fall back to option completion + if installed: + for dist in installed: + print(dist) + sys.exit(1) + + subcommand = commands_dict[subcommand_name]() + + for opt in subcommand.parser.option_list_all: + if opt.help != optparse.SUPPRESS_HELP: + for opt_str in opt._long_opts + opt._short_opts: + options.append((opt_str, opt.nargs)) + + # filter out previously specified options from available options + prev_opts = [x.split('=')[0] for x in cwords[1:cword - 1]] + options = [(x, v) for (x, v) in options if x not in prev_opts] + # filter options by current input + options = [(k, v) for k, v in options if k.startswith(current)] + # get completion type given cwords and available subcommand options + completion_type = get_path_completion_type( + cwords, cword, subcommand.parser.option_list_all, + ) + # get completion files and directories if ``completion_type`` is + # ````, ```` or ```` + if completion_type: + options = auto_complete_paths(current, completion_type) + options = ((opt, 0) for opt in options) + for option in options: + opt_label = option[0] + # append '=' to options which require args + if option[1] and option[0][:2] == "--": + opt_label += '=' + print(opt_label) + else: + # show main parser options only when necessary + + opts = [i.option_list for i in parser.option_groups] + opts.append(parser.option_list) + opts = (o for it in opts for o in it) + if current.startswith('-'): + for opt in opts: + if opt.help != optparse.SUPPRESS_HELP: + subcommands += opt._long_opts + opt._short_opts + else: + # get completion type given cwords and all available options + completion_type = get_path_completion_type(cwords, cword, opts) + if completion_type: + subcommands = auto_complete_paths(current, completion_type) + + print(' '.join([x for x in subcommands if x.startswith(current)])) + sys.exit(1) + + +def get_path_completion_type(cwords, cword, opts): + """Get the type of path completion (``file``, ``dir``, ``path`` or None) + + :param cwords: same as the environmental variable ``COMP_WORDS`` + :param cword: same as the environmental variable ``COMP_CWORD`` + :param opts: The available options to check + :return: path completion type (``file``, ``dir``, ``path`` or None) + """ + if cword < 2 or not cwords[cword - 2].startswith('-'): + return + for opt in opts: + if opt.help == optparse.SUPPRESS_HELP: + continue + for o in str(opt).split('/'): + if cwords[cword - 2].split('=')[0] == o: + if not opt.metavar or any( + x in ('path', 'file', 'dir') + for x in opt.metavar.split('/')): + return opt.metavar + + +def auto_complete_paths(current, completion_type): + """If ``completion_type`` is ``file`` or ``path``, list all regular files + and directories starting with ``current``; otherwise only list directories + starting with ``current``. + + :param current: The word to be completed + :param completion_type: path completion type(`file`, `path` or `dir`)i + :return: A generator of regular files and/or directories + """ + directory, filename = os.path.split(current) + current_path = os.path.abspath(directory) + # Don't complete paths if they can't be accessed + if not os.access(current_path, os.R_OK): + return + filename = os.path.normcase(filename) + # list all files that start with ``filename`` + file_list = (x for x in os.listdir(current_path) + if os.path.normcase(x).startswith(filename)) + for f in file_list: + opt = os.path.join(current_path, f) + comp_file = os.path.normcase(os.path.join(directory, f)) + # complete regular files when there is not ```` after option + # complete directories when there is ````, ```` or + # ````after option + if completion_type != 'dir' and os.path.isfile(opt): + yield comp_file + elif os.path.isdir(opt): + yield os.path.join(comp_file, '') diff --git a/pipenv/patched/notpip/_internal/basecommand.py b/pipenv/patched/notpip/_internal/cli/base_command.py similarity index 92% rename from pipenv/patched/notpip/_internal/basecommand.py rename to pipenv/patched/notpip/_internal/cli/base_command.py index 60199d55..229831f2 100644 --- a/pipenv/patched/notpip/_internal/basecommand.py +++ b/pipenv/patched/notpip/_internal/cli/base_command.py @@ -7,10 +7,14 @@ import optparse import os import sys -from pipenv.patched.notpip._internal import cmdoptions -from pipenv.patched.notpip._internal.baseparser import ( +from pipenv.patched.notpip._internal.cli import cmdoptions +from pipenv.patched.notpip._internal.cli.parser import ( ConfigOptionParser, UpdatingDefaultsHelpFormatter, ) +from pipenv.patched.notpip._internal.cli.status_codes import ( + ERROR, PREVIOUS_BUILD_DIR_ERROR, SUCCESS, UNKNOWN_ERROR, + VIRTUALENV_NOT_FOUND, +) from pipenv.patched.notpip._internal.download import PipSession from pipenv.patched.notpip._internal.exceptions import ( BadCommand, CommandError, InstallationError, PreviousBuildDirError, @@ -18,12 +22,10 @@ from pipenv.patched.notpip._internal.exceptions import ( ) from pipenv.patched.notpip._internal.index import PackageFinder from pipenv.patched.notpip._internal.locations import running_under_virtualenv -from pipenv.patched.notpip._internal.req.req_file import parse_requirements -from pipenv.patched.notpip._internal.req.req_install import InstallRequirement -from pipenv.patched.notpip._internal.status_codes import ( - ERROR, PREVIOUS_BUILD_DIR_ERROR, SUCCESS, UNKNOWN_ERROR, - VIRTUALENV_NOT_FOUND, +from pipenv.patched.notpip._internal.req.constructors import ( + install_req_from_editable, install_req_from_line, ) +from pipenv.patched.notpip._internal.req.req_file import parse_requirements from pipenv.patched.notpip._internal.utils.logging import setup_logging from pipenv.patched.notpip._internal.utils.misc import get_prog, normalize_path from pipenv.patched.notpip._internal.utils.outdated import pip_version_check @@ -168,12 +170,14 @@ class Command(object): return UNKNOWN_ERROR finally: - # Check if we're using the latest version of pip available - skip_version_check = ( - options.disable_pip_version_check or - getattr(options, "no_index", False) + allow_version_check = ( + # Does this command have the index_group options? + hasattr(options, "no_index") and + # Is this command allowed to perform this check? + not (options.disable_pip_version_check or options.no_index) ) - if not skip_version_check: + # Check if we're using the latest version of pip available + if allow_version_check: session = self._build_session( options, retries=0, @@ -208,7 +212,7 @@ class RequirementCommand(Command): requirement_set.add_requirement(req_to_add) for req in args: - req_to_add = InstallRequirement.from_line( + req_to_add = install_req_from_line( req, None, isolated=options.isolated_mode, wheel_cache=wheel_cache ) @@ -216,7 +220,7 @@ class RequirementCommand(Command): requirement_set.add_requirement(req_to_add) for req in options.editables: - req_to_add = InstallRequirement.from_editable( + req_to_add = install_req_from_editable( req, isolated=options.isolated_mode, wheel_cache=wheel_cache diff --git a/pipenv/patched/notpip/_internal/cmdoptions.py b/pipenv/patched/notpip/_internal/cli/cmdoptions.py similarity index 81% rename from pipenv/patched/notpip/_internal/cmdoptions.py rename to pipenv/patched/notpip/_internal/cli/cmdoptions.py index c25e769f..a075a67e 100644 --- a/pipenv/patched/notpip/_internal/cmdoptions.py +++ b/pipenv/patched/notpip/_internal/cli/cmdoptions.py @@ -13,10 +13,9 @@ import warnings from functools import partial from optparse import SUPPRESS_HELP, Option, OptionGroup -from pipenv.patched.notpip._internal.index import ( - FormatControl, fmt_ctl_handle_mutual_exclude, fmt_ctl_no_binary, -) +from pipenv.patched.notpip._internal.exceptions import CommandError from pipenv.patched.notpip._internal.locations import USER_CACHE_DIR, src_prefix +from pipenv.patched.notpip._internal.models.format_control import FormatControl from pipenv.patched.notpip._internal.models.index import PyPI from pipenv.patched.notpip._internal.utils.hashes import STRONG_HASHES from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING @@ -53,13 +52,52 @@ def check_install_build_global(options, check_options=None): names = ["build_options", "global_options", "install_options"] if any(map(getname, names)): control = options.format_control - fmt_ctl_no_binary(control) + control.disallow_binaries() warnings.warn( 'Disabling all use of wheels due to the use of --build-options ' '/ --global-options / --install-options.', stacklevel=2, ) +def check_dist_restriction(options, check_target=False): + """Function for determining if custom platform options are allowed. + + :param options: The OptionParser options. + :param check_target: Whether or not to check if --target is being used. + """ + dist_restriction_set = any([ + options.python_version, + options.platform, + options.abi, + options.implementation, + ]) + + binary_only = FormatControl(set(), {':all:'}) + sdist_dependencies_allowed = ( + options.format_control != binary_only and + not options.ignore_dependencies + ) + + # Installations or downloads using dist restrictions must not combine + # source distributions and dist-specific wheels, as they are not + # gauranteed to be locally compatible. + if dist_restriction_set and sdist_dependencies_allowed: + raise CommandError( + "When restricting platform and interpreter constraints using " + "--python-version, --platform, --abi, or --implementation, " + "either --no-deps must be set, or --only-binary=:all: must be " + "set and --no-binary must not be set (or must be set to " + ":none:)." + ) + + if check_target: + if dist_restriction_set and not options.target_dir: + raise CommandError( + "Can not use any platform or abi specific options unless " + "installing via '--target'" + ) + + ########### # options # ########### @@ -365,24 +403,25 @@ def _get_format_control(values, option): def _handle_no_binary(option, opt_str, value, parser): - existing = getattr(parser.values, option.dest) - fmt_ctl_handle_mutual_exclude( + existing = _get_format_control(parser.values, option) + FormatControl.handle_mutual_excludes( value, existing.no_binary, existing.only_binary, ) def _handle_only_binary(option, opt_str, value, parser): - existing = getattr(parser.values, option.dest) - fmt_ctl_handle_mutual_exclude( + existing = _get_format_control(parser.values, option) + FormatControl.handle_mutual_excludes( value, existing.only_binary, existing.no_binary, ) def no_binary(): + format_control = FormatControl(set(), set()) return Option( "--no-binary", dest="format_control", action="callback", callback=_handle_no_binary, type="str", - default=FormatControl(set(), set()), + default=format_control, help="Do not use binary packages. Can be supplied multiple times, and " "each time adds to the existing value. Accepts either :all: to " "disable all binary packages, :none: to empty the set, or one or " @@ -393,10 +432,11 @@ def no_binary(): def only_binary(): + format_control = FormatControl(set(), set()) return Option( "--only-binary", dest="format_control", action="callback", callback=_handle_only_binary, type="str", - default=FormatControl(set(), set()), + default=format_control, help="Do not use source packages. Can be supplied multiple times, and " "each time adds to the existing value. Accepts either :all: to " "disable all source packages, :none: to empty the set, or one or " @@ -406,6 +446,61 @@ def only_binary(): ) +platform = partial( + Option, + '--platform', + dest='platform', + metavar='platform', + default=None, + help=("Only use wheels compatible with . " + "Defaults to the platform of the running system."), +) + + +python_version = partial( + Option, + '--python-version', + dest='python_version', + metavar='python_version', + default=None, + help=("Only use wheels compatible with Python " + "interpreter version . If not specified, then the " + "current system interpreter minor version is used. A major " + "version (e.g. '2') can be specified to match all " + "minor revs of that major version. A minor version " + "(e.g. '34') can also be specified."), +) + + +implementation = partial( + Option, + '--implementation', + dest='implementation', + metavar='implementation', + default=None, + help=("Only use wheels compatible with Python " + "implementation , e.g. 'pp', 'jy', 'cp', " + " or 'ip'. If not specified, then the current " + "interpreter implementation is used. Use 'py' to force " + "implementation-agnostic wheels."), +) + + +abi = partial( + Option, + '--abi', + dest='abi', + metavar='abi', + default=None, + help=("Only use wheels compatible with Python " + "abi , e.g. 'pypy_41'. If not specified, then the " + "current interpreter abi tag is used. Generally " + "you will need to specify --implementation, " + "--platform, and --python-version when using " + "this option."), +) + + def prefer_binary(): return Option( "--prefer-binary", @@ -501,7 +596,7 @@ no_clean = partial( '--no-clean', action='store_true', default=False, - help="Don't clean up build directories)." + help="Don't clean up build directories." ) # type: Any pre = partial( diff --git a/pipenv/patched/notpip/_internal/cli/main_parser.py b/pipenv/patched/notpip/_internal/cli/main_parser.py new file mode 100644 index 00000000..abe2f69e --- /dev/null +++ b/pipenv/patched/notpip/_internal/cli/main_parser.py @@ -0,0 +1,96 @@ +"""A single place for constructing and exposing the main parser +""" + +import os +import sys + +from pipenv.patched.notpip import __version__ +from pipenv.patched.notpip._internal.cli import cmdoptions +from pipenv.patched.notpip._internal.cli.parser import ( + ConfigOptionParser, UpdatingDefaultsHelpFormatter, +) +from pipenv.patched.notpip._internal.commands import ( + commands_dict, get_similar_commands, get_summaries, +) +from pipenv.patched.notpip._internal.exceptions import CommandError +from pipenv.patched.notpip._internal.utils.misc import get_prog + +__all__ = ["create_main_parser", "parse_command"] + + +def create_main_parser(): + """Creates and returns the main parser for pip's CLI + """ + + parser_kw = { + 'usage': '\n%prog [options]', + 'add_help_option': False, + 'formatter': UpdatingDefaultsHelpFormatter(), + 'name': 'global', + 'prog': get_prog(), + } + + parser = ConfigOptionParser(**parser_kw) + parser.disable_interspersed_args() + + pip_pkg_dir = os.path.abspath(os.path.join( + os.path.dirname(__file__), "..", "..", + )) + parser.version = 'pip %s from %s (python %s)' % ( + __version__, pip_pkg_dir, sys.version[:3], + ) + + # add the general options + gen_opts = cmdoptions.make_option_group(cmdoptions.general_group, parser) + parser.add_option_group(gen_opts) + + parser.main = True # so the help formatter knows + + # create command listing for description + command_summaries = get_summaries() + description = [''] + ['%-27s %s' % (i, j) for i, j in command_summaries] + parser.description = '\n'.join(description) + + return parser + + +def parse_command(args): + parser = create_main_parser() + + # Note: parser calls disable_interspersed_args(), so the result of this + # call is to split the initial args into the general options before the + # subcommand and everything else. + # For example: + # args: ['--timeout=5', 'install', '--user', 'INITools'] + # general_options: ['--timeout==5'] + # args_else: ['install', '--user', 'INITools'] + general_options, args_else = parser.parse_args(args) + + # --version + if general_options.version: + sys.stdout.write(parser.version) + sys.stdout.write(os.linesep) + sys.exit() + + # pip || pip help -> print_help() + if not args_else or (args_else[0] == 'help' and len(args_else) == 1): + parser.print_help() + sys.exit() + + # the subcommand name + cmd_name = args_else[0] + + if cmd_name not in commands_dict: + guess = get_similar_commands(cmd_name) + + msg = ['unknown command "%s"' % cmd_name] + if guess: + msg.append('maybe you meant "%s"' % guess) + + raise CommandError(' - '.join(msg)) + + # all the args without the subcommand + cmd_args = args[:] + cmd_args.remove(cmd_name) + + return cmd_name, cmd_args diff --git a/pipenv/patched/notpip/_internal/baseparser.py b/pipenv/patched/notpip/_internal/cli/parser.py similarity index 88% rename from pipenv/patched/notpip/_internal/baseparser.py rename to pipenv/patched/notpip/_internal/cli/parser.py index f82093bf..2d2c8f4d 100644 --- a/pipenv/patched/notpip/_internal/baseparser.py +++ b/pipenv/patched/notpip/_internal/cli/parser.py @@ -9,8 +9,9 @@ from distutils.util import strtobool from pipenv.patched.notpip._vendor.six import string_types -from pipenv.patched.notpip._internal.compat import get_terminal_size +from pipenv.patched.notpip._internal.cli.status_codes import UNKNOWN_ERROR from pipenv.patched.notpip._internal.configuration import Configuration, ConfigurationError +from pipenv.patched.notpip._internal.utils.compat import get_terminal_size logger = logging.getLogger(__name__) @@ -192,7 +193,14 @@ class ConfigOptionParser(CustomOptionParser): continue if option.action in ('store_true', 'store_false', 'count'): - val = strtobool(val) + try: + val = strtobool(val) + except ValueError: + error_msg = invalid_config_error_message( + option.action, key, val + ) + self.error(error_msg) + elif option.action == 'append': val = val.split() val = [self.check_default(option, key, v) for v in val] @@ -225,7 +233,7 @@ class ConfigOptionParser(CustomOptionParser): try: self.config.load() except ConfigurationError as err: - self.exit(2, err.args[0]) + self.exit(UNKNOWN_ERROR, str(err)) defaults = self._update_defaults(self.defaults.copy()) # ours for option in self._get_all_options(): @@ -237,4 +245,17 @@ class ConfigOptionParser(CustomOptionParser): def error(self, msg): self.print_usage(sys.stderr) - self.exit(2, "%s\n" % msg) + self.exit(UNKNOWN_ERROR, "%s\n" % msg) + + +def invalid_config_error_message(action, key, val): + """Returns a better error message when invalid configuration option + is provided.""" + if action in ('store_true', 'store_false'): + return ("{0} is not a valid value for {1} option, " + "please specify a boolean value like yes/no, " + "true/false or 1/0 instead.").format(val, key) + + return ("{0} is not a valid value for {1} option, " + "please specify a numerical value like 1/0 " + "instead.").format(val, key) diff --git a/pipenv/patched/notpip/_internal/status_codes.py b/pipenv/patched/notpip/_internal/cli/status_codes.py similarity index 100% rename from pipenv/patched/notpip/_internal/status_codes.py rename to pipenv/patched/notpip/_internal/cli/status_codes.py diff --git a/pipenv/patched/notpip/_internal/commands/__init__.py b/pipenv/patched/notpip/_internal/commands/__init__.py index 140c4609..a403c6f9 100644 --- a/pipenv/patched/notpip/_internal/commands/__init__.py +++ b/pipenv/patched/notpip/_internal/commands/__init__.py @@ -21,7 +21,7 @@ from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: from typing import List, Type # noqa: F401 - from pipenv.patched.notpip._internal.basecommand import Command # noqa: F401 + from pipenv.patched.notpip._internal.cli.base_command import Command # noqa: F401 commands_order = [ InstallCommand, diff --git a/pipenv/patched/notpip/_internal/commands/check.py b/pipenv/patched/notpip/_internal/commands/check.py index cd5ffb5f..adf4f5e7 100644 --- a/pipenv/patched/notpip/_internal/commands/check.py +++ b/pipenv/patched/notpip/_internal/commands/check.py @@ -1,6 +1,6 @@ import logging -from pipenv.patched.notpip._internal.basecommand import Command +from pipenv.patched.notpip._internal.cli.base_command import Command from pipenv.patched.notpip._internal.operations.check import ( check_package_set, create_package_set_from_installed, ) diff --git a/pipenv/patched/notpip/_internal/commands/completion.py b/pipenv/patched/notpip/_internal/commands/completion.py index f4c31c1b..cb8a11a7 100644 --- a/pipenv/patched/notpip/_internal/commands/completion.py +++ b/pipenv/patched/notpip/_internal/commands/completion.py @@ -3,7 +3,7 @@ from __future__ import absolute_import import sys import textwrap -from pipenv.patched.notpip._internal.basecommand import Command +from pipenv.patched.notpip._internal.cli.base_command import Command from pipenv.patched.notpip._internal.utils.misc import get_prog BASE_COMPLETION = """ diff --git a/pipenv/patched/notpip/_internal/commands/configuration.py b/pipenv/patched/notpip/_internal/commands/configuration.py index 090109c5..6c1dbdfd 100644 --- a/pipenv/patched/notpip/_internal/commands/configuration.py +++ b/pipenv/patched/notpip/_internal/commands/configuration.py @@ -2,11 +2,11 @@ import logging import os import subprocess -from pipenv.patched.notpip._internal.basecommand import Command +from pipenv.patched.notpip._internal.cli.base_command import Command +from pipenv.patched.notpip._internal.cli.status_codes import ERROR, SUCCESS from pipenv.patched.notpip._internal.configuration import Configuration, kinds from pipenv.patched.notpip._internal.exceptions import PipError from pipenv.patched.notpip._internal.locations import venv_config_file -from pipenv.patched.notpip._internal.status_codes import ERROR, SUCCESS from pipenv.patched.notpip._internal.utils.misc import get_prog logger = logging.getLogger(__name__) diff --git a/pipenv/patched/notpip/_internal/commands/download.py b/pipenv/patched/notpip/_internal/commands/download.py index 63d91b04..e5d87121 100644 --- a/pipenv/patched/notpip/_internal/commands/download.py +++ b/pipenv/patched/notpip/_internal/commands/download.py @@ -3,10 +3,8 @@ from __future__ import absolute_import import logging import os -from pipenv.patched.notpip._internal import cmdoptions -from pipenv.patched.notpip._internal.basecommand import RequirementCommand -from pipenv.patched.notpip._internal.exceptions import CommandError -from pipenv.patched.notpip._internal.index import FormatControl +from pipenv.patched.notpip._internal.cli import cmdoptions +from pipenv.patched.notpip._internal.cli.base_command import RequirementCommand from pipenv.patched.notpip._internal.operations.prepare import RequirementPreparer from pipenv.patched.notpip._internal.req import RequirementSet from pipenv.patched.notpip._internal.req.req_tracker import RequirementTracker @@ -69,52 +67,10 @@ class DownloadCommand(RequirementCommand): help=("Download packages into ."), ) - cmd_opts.add_option( - '--platform', - dest='platform', - metavar='platform', - default=None, - help=("Only download wheels compatible with . " - "Defaults to the platform of the running system."), - ) - - cmd_opts.add_option( - '--python-version', - dest='python_version', - metavar='python_version', - default=None, - help=("Only download wheels compatible with Python " - "interpreter version . If not specified, then the " - "current system interpreter minor version is used. A major " - "version (e.g. '2') can be specified to match all " - "minor revs of that major version. A minor version " - "(e.g. '34') can also be specified."), - ) - - cmd_opts.add_option( - '--implementation', - dest='implementation', - metavar='implementation', - default=None, - help=("Only download wheels compatible with Python " - "implementation , e.g. 'pp', 'jy', 'cp', " - " or 'ip'. If not specified, then the current " - "interpreter implementation is used. Use 'py' to force " - "implementation-agnostic wheels."), - ) - - cmd_opts.add_option( - '--abi', - dest='abi', - metavar='abi', - default=None, - help=("Only download wheels compatible with Python " - "abi , e.g. 'pypy_41'. If not specified, then the " - "current interpreter abi tag is used. Generally " - "you will need to specify --implementation, " - "--platform, and --python-version when using " - "this option."), - ) + cmd_opts.add_option(cmdoptions.platform()) + cmd_opts.add_option(cmdoptions.python_version()) + cmd_opts.add_option(cmdoptions.implementation()) + cmd_opts.add_option(cmdoptions.abi()) index_opts = cmdoptions.make_option_group( cmdoptions.index_group, @@ -135,25 +91,7 @@ class DownloadCommand(RequirementCommand): else: python_versions = None - dist_restriction_set = any([ - options.python_version, - options.platform, - options.abi, - options.implementation, - ]) - binary_only = FormatControl(set(), {':all:'}) - no_sdist_dependencies = ( - options.format_control != binary_only and - not options.ignore_dependencies - ) - if dist_restriction_set and no_sdist_dependencies: - raise CommandError( - "When restricting platform and interpreter constraints using " - "--python-version, --platform, --abi, or --implementation, " - "either --no-deps must be set, or --only-binary=:all: must be " - "set and --no-binary must not be set (or must be set to " - ":none:)." - ) + cmdoptions.check_dist_restriction(options) options.src_dir = os.path.abspath(options.src_dir) options.download_dir = normalize_path(options.download_dir) diff --git a/pipenv/patched/notpip/_internal/commands/freeze.py b/pipenv/patched/notpip/_internal/commands/freeze.py index 6959b6ab..343227ba 100644 --- a/pipenv/patched/notpip/_internal/commands/freeze.py +++ b/pipenv/patched/notpip/_internal/commands/freeze.py @@ -2,11 +2,11 @@ from __future__ import absolute_import import sys -from pipenv.patched.notpip._internal import index -from pipenv.patched.notpip._internal.basecommand import Command from pipenv.patched.notpip._internal.cache import WheelCache -from pipenv.patched.notpip._internal.compat import stdlib_pkgs +from pipenv.patched.notpip._internal.cli.base_command import Command +from pipenv.patched.notpip._internal.models.format_control import FormatControl from pipenv.patched.notpip._internal.operations.freeze import freeze +from pipenv.patched.notpip._internal.utils.compat import stdlib_pkgs DEV_PKGS = {'pip', 'setuptools', 'distribute', 'wheel'} @@ -71,7 +71,7 @@ class FreezeCommand(Command): self.parser.insert_option_group(0, self.cmd_opts) def run(self, options, args): - format_control = index.FormatControl(set(), set()) + format_control = FormatControl(set(), set()) wheel_cache = WheelCache(options.cache_dir, format_control) skip = set(stdlib_pkgs) if not options.freeze_all: diff --git a/pipenv/patched/notpip/_internal/commands/hash.py b/pipenv/patched/notpip/_internal/commands/hash.py index a86574da..183f11ae 100644 --- a/pipenv/patched/notpip/_internal/commands/hash.py +++ b/pipenv/patched/notpip/_internal/commands/hash.py @@ -4,8 +4,8 @@ import hashlib import logging import sys -from pipenv.patched.notpip._internal.basecommand import Command -from pipenv.patched.notpip._internal.status_codes import ERROR +from pipenv.patched.notpip._internal.cli.base_command import Command +from pipenv.patched.notpip._internal.cli.status_codes import ERROR from pipenv.patched.notpip._internal.utils.hashes import FAVORITE_HASH, STRONG_HASHES from pipenv.patched.notpip._internal.utils.misc import read_chunks diff --git a/pipenv/patched/notpip/_internal/commands/help.py b/pipenv/patched/notpip/_internal/commands/help.py index aaf0f87e..f2c61965 100644 --- a/pipenv/patched/notpip/_internal/commands/help.py +++ b/pipenv/patched/notpip/_internal/commands/help.py @@ -1,6 +1,7 @@ from __future__ import absolute_import -from pipenv.patched.notpip._internal.basecommand import SUCCESS, Command +from pipenv.patched.notpip._internal.cli.base_command import Command +from pipenv.patched.notpip._internal.cli.status_codes import SUCCESS from pipenv.patched.notpip._internal.exceptions import CommandError diff --git a/pipenv/patched/notpip/_internal/commands/install.py b/pipenv/patched/notpip/_internal/commands/install.py index ebdf07d7..ddcb4759 100644 --- a/pipenv/patched/notpip/_internal/commands/install.py +++ b/pipenv/patched/notpip/_internal/commands/install.py @@ -9,9 +9,10 @@ from optparse import SUPPRESS_HELP from pipenv.patched.notpip._vendor import pkg_resources -from pipenv.patched.notpip._internal import cmdoptions -from pipenv.patched.notpip._internal.basecommand import RequirementCommand from pipenv.patched.notpip._internal.cache import WheelCache +from pipenv.patched.notpip._internal.cli import cmdoptions +from pipenv.patched.notpip._internal.cli.base_command import RequirementCommand +from pipenv.patched.notpip._internal.cli.status_codes import ERROR from pipenv.patched.notpip._internal.exceptions import ( CommandError, InstallationError, PreviousBuildDirError, ) @@ -21,7 +22,6 @@ from pipenv.patched.notpip._internal.operations.prepare import RequirementPrepar from pipenv.patched.notpip._internal.req import RequirementSet, install_given_reqs from pipenv.patched.notpip._internal.req.req_tracker import RequirementTracker from pipenv.patched.notpip._internal.resolve import Resolver -from pipenv.patched.notpip._internal.status_codes import ERROR from pipenv.patched.notpip._internal.utils.filesystem import check_path_owner from pipenv.patched.notpip._internal.utils.misc import ( ensure_dir, get_installed_version, @@ -83,6 +83,11 @@ class InstallCommand(RequirementCommand): '. Use --upgrade to replace existing packages in ' 'with new versions.' ) + cmd_opts.add_option(cmdoptions.platform()) + cmd_opts.add_option(cmdoptions.python_version()) + cmd_opts.add_option(cmdoptions.implementation()) + cmd_opts.add_option(cmdoptions.abi()) + cmd_opts.add_option( '--user', dest='use_user_site', @@ -204,7 +209,6 @@ class InstallCommand(RequirementCommand): def run(self, options, args): cmdoptions.check_install_build_global(options) - upgrade_strategy = "to-satisfy-only" if options.upgrade: upgrade_strategy = options.upgrade_strategy @@ -212,6 +216,13 @@ class InstallCommand(RequirementCommand): if options.build_dir: options.build_dir = os.path.abspath(options.build_dir) + cmdoptions.check_dist_restriction(options, check_target=True) + + if options.python_version: + python_versions = [options.python_version] + else: + python_versions = None + options.src_dir = os.path.abspath(options.src_dir) install_options = options.install_options or [] if options.use_user_site: @@ -246,7 +257,14 @@ class InstallCommand(RequirementCommand): global_options = options.global_options or [] with self._build_session(options) as session: - finder = self._build_package_finder(options, session) + finder = self._build_package_finder( + options=options, + session=session, + platform=options.platform, + python_versions=python_versions, + abi=options.abi, + implementation=options.implementation, + ) build_delete = (not (options.no_clean or options.build_dir)) wheel_cache = WheelCache(options.cache_dir, options.format_control) @@ -266,6 +284,7 @@ class InstallCommand(RequirementCommand): ) as directory: requirement_set = RequirementSet( require_hashes=options.require_hashes, + check_supported_wheels=not options.target_dir, ) try: diff --git a/pipenv/patched/notpip/_internal/commands/list.py b/pipenv/patched/notpip/_internal/commands/list.py index 99aee99f..577c0b5f 100644 --- a/pipenv/patched/notpip/_internal/commands/list.py +++ b/pipenv/patched/notpip/_internal/commands/list.py @@ -6,8 +6,8 @@ import logging from pipenv.patched.notpip._vendor import six from pipenv.patched.notpip._vendor.six.moves import zip_longest -from pipenv.patched.notpip._internal.basecommand import Command -from pipenv.patched.notpip._internal.cmdoptions import index_group, make_option_group +from pipenv.patched.notpip._internal.cli import cmdoptions +from pipenv.patched.notpip._internal.cli.base_command import Command from pipenv.patched.notpip._internal.exceptions import CommandError from pipenv.patched.notpip._internal.index import PackageFinder from pipenv.patched.notpip._internal.utils.misc import ( @@ -102,7 +102,9 @@ class ListCommand(Command): help='Include editable package from output.', default=True, ) - index_opts = make_option_group(index_group, self.parser) + index_opts = cmdoptions.make_option_group( + cmdoptions.index_group, self.parser + ) self.parser.insert_option_group(0, index_opts) self.parser.insert_option_group(0, cmd_opts) diff --git a/pipenv/patched/notpip/_internal/commands/search.py b/pipenv/patched/notpip/_internal/commands/search.py index ac111c14..986208f9 100644 --- a/pipenv/patched/notpip/_internal/commands/search.py +++ b/pipenv/patched/notpip/_internal/commands/search.py @@ -11,12 +11,12 @@ from pipenv.patched.notpip._vendor.packaging.version import parse as parse_versi # why we ignore the type on this import from pipenv.patched.notpip._vendor.six.moves import xmlrpc_client # type: ignore -from pipenv.patched.notpip._internal.basecommand import SUCCESS, Command -from pipenv.patched.notpip._internal.compat import get_terminal_size +from pipenv.patched.notpip._internal.cli.base_command import Command +from pipenv.patched.notpip._internal.cli.status_codes import NO_MATCHES_FOUND, SUCCESS from pipenv.patched.notpip._internal.download import PipXmlrpcTransport from pipenv.patched.notpip._internal.exceptions import CommandError from pipenv.patched.notpip._internal.models.index import PyPI -from pipenv.patched.notpip._internal.status_codes import NO_MATCHES_FOUND +from pipenv.patched.notpip._internal.utils.compat import get_terminal_size from pipenv.patched.notpip._internal.utils.logging import indent_log logger = logging.getLogger(__name__) diff --git a/pipenv/patched/notpip/_internal/commands/show.py b/pipenv/patched/notpip/_internal/commands/show.py index 8de6b6b8..3fd24482 100644 --- a/pipenv/patched/notpip/_internal/commands/show.py +++ b/pipenv/patched/notpip/_internal/commands/show.py @@ -7,8 +7,8 @@ from email.parser import FeedParser # type: ignore from pipenv.patched.notpip._vendor import pkg_resources from pipenv.patched.notpip._vendor.packaging.utils import canonicalize_name -from pipenv.patched.notpip._internal.basecommand import Command -from pipenv.patched.notpip._internal.status_codes import ERROR, SUCCESS +from pipenv.patched.notpip._internal.cli.base_command import Command +from pipenv.patched.notpip._internal.cli.status_codes import ERROR, SUCCESS logger = logging.getLogger(__name__) diff --git a/pipenv/patched/notpip/_internal/commands/uninstall.py b/pipenv/patched/notpip/_internal/commands/uninstall.py index 45a0eba5..cf6a511c 100644 --- a/pipenv/patched/notpip/_internal/commands/uninstall.py +++ b/pipenv/patched/notpip/_internal/commands/uninstall.py @@ -2,9 +2,10 @@ from __future__ import absolute_import from pipenv.patched.notpip._vendor.packaging.utils import canonicalize_name -from pipenv.patched.notpip._internal.basecommand import Command +from pipenv.patched.notpip._internal.cli.base_command import Command from pipenv.patched.notpip._internal.exceptions import InstallationError -from pipenv.patched.notpip._internal.req import InstallRequirement, parse_requirements +from pipenv.patched.notpip._internal.req import parse_requirements +from pipenv.patched.notpip._internal.req.constructors import install_req_from_line from pipenv.patched.notpip._internal.utils.misc import protect_pip_from_modification_on_windows @@ -47,7 +48,7 @@ class UninstallCommand(Command): with self._build_session(options) as session: reqs_to_uninstall = {} for name in args: - req = InstallRequirement.from_line( + req = install_req_from_line( name, isolated=options.isolated_mode, ) if req.name: diff --git a/pipenv/patched/notpip/_internal/commands/wheel.py b/pipenv/patched/notpip/_internal/commands/wheel.py index c04d58ed..08d695ab 100644 --- a/pipenv/patched/notpip/_internal/commands/wheel.py +++ b/pipenv/patched/notpip/_internal/commands/wheel.py @@ -4,9 +4,9 @@ from __future__ import absolute_import import logging import os -from pipenv.patched.notpip._internal import cmdoptions -from pipenv.patched.notpip._internal.basecommand import RequirementCommand from pipenv.patched.notpip._internal.cache import WheelCache +from pipenv.patched.notpip._internal.cli import cmdoptions +from pipenv.patched.notpip._internal.cli.base_command import RequirementCommand from pipenv.patched.notpip._internal.exceptions import CommandError, PreviousBuildDirError from pipenv.patched.notpip._internal.operations.prepare import RequirementPreparer from pipenv.patched.notpip._internal.req import RequirementSet diff --git a/pipenv/patched/notpip/_internal/configuration.py b/pipenv/patched/notpip/_internal/configuration.py index 3df185f7..3c02c955 100644 --- a/pipenv/patched/notpip/_internal/configuration.py +++ b/pipenv/patched/notpip/_internal/configuration.py @@ -18,7 +18,9 @@ import os from pipenv.patched.notpip._vendor import six from pipenv.patched.notpip._vendor.six.moves import configparser -from pipenv.patched.notpip._internal.exceptions import ConfigurationError +from pipenv.patched.notpip._internal.exceptions import ( + ConfigurationError, ConfigurationFileCouldNotBeLoaded, +) from pipenv.patched.notpip._internal.locations import ( legacy_config_file, new_config_file, running_under_virtualenv, site_config_files, venv_config_file, @@ -289,11 +291,16 @@ class Configuration(object): try: parser.read(fname) except UnicodeDecodeError: - raise ConfigurationError(( - "ERROR: " - "Configuration file contains invalid %s characters.\n" - "Please fix your configuration, located at %s\n" - ) % (locale.getpreferredencoding(False), fname)) + # See https://github.com/pypa/pip/issues/4963 + raise ConfigurationFileCouldNotBeLoaded( + reason="contains invalid {} characters".format( + locale.getpreferredencoding(False) + ), + fname=fname, + ) + except configparser.Error as error: + # See https://github.com/pypa/pip/issues/4893 + raise ConfigurationFileCouldNotBeLoaded(error=error) return parser def _load_environment_vars(self): diff --git a/pipenv/patched/notpip/_internal/download.py b/pipenv/patched/notpip/_internal/download.py index 06a45644..8f4c38f5 100644 --- a/pipenv/patched/notpip/_internal/download.py +++ b/pipenv/patched/notpip/_internal/download.py @@ -323,7 +323,7 @@ class InsecureHTTPAdapter(HTTPAdapter): conn.ca_certs = None -class PipSession(Session): +class PipSession(requests.Session): timeout = None @@ -753,7 +753,7 @@ def _copy_dist_from_dir(link_path, location): # build an sdist setup_py = 'setup.py' - sdist_args = [os.environ.get('PIP_PYTHON_PATH', sys.executable)] + sdist_args = [sys.executable] sdist_args.append('-c') sdist_args.append(SETUPTOOLS_SHIM % setup_py) sdist_args.append('sdist') diff --git a/pipenv/patched/notpip/_internal/exceptions.py b/pipenv/patched/notpip/_internal/exceptions.py index 43595977..2eadcf28 100644 --- a/pipenv/patched/notpip/_internal/exceptions.py +++ b/pipenv/patched/notpip/_internal/exceptions.py @@ -247,3 +247,22 @@ class HashMismatch(HashError): class UnsupportedPythonVersion(InstallationError): """Unsupported python version according to Requires-Python package metadata.""" + + +class ConfigurationFileCouldNotBeLoaded(ConfigurationError): + """When there are errors while loading a configuration file + """ + + def __init__(self, reason="could not be loaded", fname=None, error=None): + super(ConfigurationFileCouldNotBeLoaded, self).__init__(error) + self.reason = reason + self.fname = fname + self.error = error + + def __str__(self): + if self.fname is not None: + message_part = " in {}.".format(self.fname) + else: + assert self.error is not None + message_part = ".\n{}\n".format(self.error.message) + return "Configuration file {}{}".format(self.reason, message_part) diff --git a/pipenv/patched/notpip/_internal/index.py b/pipenv/patched/notpip/_internal/index.py index 426880e9..b4b02373 100644 --- a/pipenv/patched/notpip/_internal/index.py +++ b/pipenv/patched/notpip/_internal/index.py @@ -20,24 +20,27 @@ from pipenv.patched.notpip._vendor.requests.exceptions import SSLError from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse from pipenv.patched.notpip._vendor.six.moves.urllib import request as urllib_request -from pipenv.patched.notpip._internal.compat import ipaddress from pipenv.patched.notpip._internal.download import HAS_TLS, is_url, path_to_url, url_to_path from pipenv.patched.notpip._internal.exceptions import ( BestVersionAlreadyInstalled, DistributionNotFound, InvalidWheelFilename, UnsupportedWheel, ) +from pipenv.patched.notpip._internal.models.candidate import InstallationCandidate +from pipenv.patched.notpip._internal.models.format_control import FormatControl from pipenv.patched.notpip._internal.models.index import PyPI +from pipenv.patched.notpip._internal.models.link import Link from pipenv.patched.notpip._internal.pep425tags import get_supported +from pipenv.patched.notpip._internal.utils.compat import ipaddress from pipenv.patched.notpip._internal.utils.deprecation import deprecated from pipenv.patched.notpip._internal.utils.logging import indent_log from pipenv.patched.notpip._internal.utils.misc import ( - ARCHIVE_EXTENSIONS, SUPPORTED_EXTENSIONS, cached_property, normalize_path, - remove_auth_from_url, splitext, + ARCHIVE_EXTENSIONS, SUPPORTED_EXTENSIONS, normalize_path, + remove_auth_from_url, ) from pipenv.patched.notpip._internal.utils.packaging import check_requires_python from pipenv.patched.notpip._internal.wheel import Wheel, wheel_ext -__all__ = ['FormatControl', 'fmt_ctl_handle_mutual_exclude', 'PackageFinder'] +__all__ = ['FormatControl', 'PackageFinder'] SECURE_ORIGINS = [ @@ -56,46 +59,120 @@ SECURE_ORIGINS = [ logger = logging.getLogger(__name__) -class InstallationCandidate(object): +def _get_content_type(url, session): + """Get the Content-Type of the given url, using a HEAD request""" + scheme, netloc, path, query, fragment = urllib_parse.urlsplit(url) + if scheme not in {'http', 'https'}: + # FIXME: some warning or something? + # assertion error? + return '' - def __init__(self, project, version, location, requires_python=None): - self.project = project - self.version = parse_version(version) - self.location = location - self._key = (self.project, self.version, self.location) - self.requires_python = requires_python + resp = session.head(url, allow_redirects=True) + resp.raise_for_status() - def __repr__(self): - return "".format( - self.project, self.version, self.location, + return resp.headers.get("Content-Type", "") + + +def _handle_get_page_fail(link, reason, url, meth=None): + if meth is None: + meth = logger.debug + meth("Could not fetch URL %s: %s - skipping", link, reason) + + +def _get_html_page(link, session=None): + if session is None: + raise TypeError( + "_get_html_page() missing 1 required keyword argument: 'session'" ) - def __hash__(self): - return hash(self._key) + url = link.url + url = url.split('#', 1)[0] - def __lt__(self, other): - return self._compare(other, lambda s, o: s < o) + # Check for VCS schemes that do not support lookup as web pages. + from pipenv.patched.notpip._internal.vcs import VcsSupport + for scheme in VcsSupport.schemes: + if url.lower().startswith(scheme) and url[len(scheme)] in '+:': + logger.debug('Cannot look at %s URL %s', scheme, link) + return None - def __le__(self, other): - return self._compare(other, lambda s, o: s <= o) + try: + filename = link.filename + for bad_ext in ARCHIVE_EXTENSIONS: + if filename.endswith(bad_ext): + content_type = _get_content_type(url, session=session) + if content_type.lower().startswith('text/html'): + break + else: + logger.debug( + 'Skipping page %s because of Content-Type: %s', + link, + content_type, + ) + return - def __eq__(self, other): - return self._compare(other, lambda s, o: s == o) + logger.debug('Getting page %s', url) - def __ge__(self, other): - return self._compare(other, lambda s, o: s >= o) + # Tack index.html onto file:// URLs that point to directories + (scheme, netloc, path, params, query, fragment) = \ + urllib_parse.urlparse(url) + if (scheme == 'file' and + os.path.isdir(urllib_request.url2pathname(path))): + # add trailing slash if not present so urljoin doesn't trim + # final segment + if not url.endswith('/'): + url += '/' + url = urllib_parse.urljoin(url, 'index.html') + logger.debug(' file: URL is directory, getting %s', url) - def __gt__(self, other): - return self._compare(other, lambda s, o: s > o) + resp = session.get( + url, + headers={ + "Accept": "text/html", + # We don't want to blindly returned cached data for + # /simple/, because authors generally expecting that + # twine upload && pip install will function, but if + # they've done a pip install in the last ~10 minutes + # it won't. Thus by setting this to zero we will not + # blindly use any cached data, however the benefit of + # using max-age=0 instead of no-cache, is that we will + # still support conditional requests, so we will still + # minimize traffic sent in cases where the page hasn't + # changed at all, we will just always incur the round + # trip for the conditional GET now instead of only + # once per 10 minutes. + # For more information, please see pypa/pip#5670. + "Cache-Control": "max-age=0", + }, + ) + resp.raise_for_status() - def __ne__(self, other): - return self._compare(other, lambda s, o: s != o) + # The check for archives above only works if the url ends with + # something that looks like an archive. However that is not a + # requirement of an url. Unless we issue a HEAD request on every + # url we cannot know ahead of time for sure if something is HTML + # or not. However we can check after we've downloaded it. + content_type = resp.headers.get('Content-Type', 'unknown') + if not content_type.lower().startswith("text/html"): + logger.debug( + 'Skipping page %s because of Content-Type: %s', + link, + content_type, + ) + return - def _compare(self, other, method): - if not isinstance(other, InstallationCandidate): - return NotImplemented - - return method(self._key, other._key) + inst = HTMLPage(resp.content, resp.url, resp.headers) + except requests.HTTPError as exc: + _handle_get_page_fail(link, exc, url) + except SSLError as exc: + reason = "There was a problem confirming the ssl certificate: " + reason += str(exc) + _handle_get_page_fail(link, reason, url, meth=logger.info) + except requests.ConnectionError as exc: + _handle_get_page_fail(link, "connection error: %s" % exc, url) + except requests.Timeout: + _handle_get_page_fail(link, "timed out", url) + else: + return inst class PackageFinder(object): @@ -210,15 +287,15 @@ class PackageFinder(object): return "\n".join(lines) def add_dependency_links(self, links): - # # FIXME: this shouldn't be global list this, it should only - # # apply to requirements of the package that specifies the - # # dependency_links value - # # FIXME: also, we should track comes_from (i.e., use Link) + # FIXME: this shouldn't be global list this, it should only + # apply to requirements of the package that specifies the + # dependency_links value + # FIXME: also, we should track comes_from (i.e., use Link) if self.process_dependency_links: deprecated( "Dependency Links processing has been deprecated and will be " "removed in a future release.", - replacement=None, + replacement="PEP 508 URL dependencies", gone_in="18.2", issue=4187, ) @@ -242,6 +319,7 @@ class PackageFinder(object): return extras + @staticmethod def _sort_locations(locations, expand_dir=False): """ @@ -467,7 +545,7 @@ class PackageFinder(object): logger.debug('* %s', location) canonical_name = canonicalize_name(project_name) - formats = fmt_ctl_formats(self.format_control, canonical_name) + formats = self.format_control.get_allowed_formats(canonical_name) search = Search(project_name, canonical_name, formats) find_links_versions = self._package_versions( # We trust every directly linked archive in find_links @@ -483,7 +561,7 @@ class PackageFinder(object): continue with indent_log(): page_versions.extend( - self._package_versions(page.links, search) + self._package_versions(page.iter_links(), search) ) dependency_versions = self._package_versions( @@ -626,7 +704,9 @@ class PackageFinder(object): try: page = self._get_page(location) - except requests.HTTPError as e: + except requests.HTTPError: + continue + if page is None: continue yield page @@ -743,7 +823,7 @@ class PackageFinder(object): return InstallationCandidate(search.supplied, version, link, link.requires_python) def _get_page(self, link): - return HTMLPage.get_page(link, session=self.session) + return _get_html_page(link, session=self.session) def egg_info_matches( @@ -763,7 +843,7 @@ def egg_info_matches( return None if search_name is None: full_match = match.group(0) - return full_match[full_match.index('-'):] + return full_match.split('-', 1)[-1] name = match.group(0).lower() # To match the "safe" name that pkg_resources creates: name = name.replace('_', '-') @@ -775,377 +855,71 @@ def egg_info_matches( return None +def _determine_base_url(document, page_url): + """Determine the HTML document's base URL. + + This looks for a ```` tag in the HTML document. If present, its href + attribute denotes the base URL of anchor tags in the document. If there is + no such tag (or if it does not have a valid href attribute), the HTML + file's URL is used as the base URL. + + :param document: An HTML document representation. The current + implementation expects the result of ``html5lib.parse()``. + :param page_url: The URL of the HTML document. + """ + for base in document.findall(".//base"): + href = base.get("href") + if href is not None: + return href + return page_url + + +def _get_encoding_from_headers(headers): + """Determine if we have any encoding information in our headers. + """ + if headers and "Content-Type" in headers: + content_type, params = cgi.parse_header(headers["Content-Type"]) + if "charset" in params: + return params['charset'] + return None + + +_CLEAN_LINK_RE = re.compile(r'[^a-z0-9$&+,/:;=?@.#%_\\|-]', re.I) + + +def _clean_link(url): + """Makes sure a link is fully encoded. That is, if a ' ' shows up in + the link, it will be rewritten to %20 (while not over-quoting + % or other characters).""" + return _CLEAN_LINK_RE.sub(lambda match: '%%%2x' % ord(match.group(0)), url) + + class HTMLPage(object): """Represents one page, along with its URL""" def __init__(self, content, url, headers=None): - # Determine if we have any encoding information in our headers - encoding = None - if headers and "Content-Type" in headers: - content_type, params = cgi.parse_header(headers["Content-Type"]) - - if "charset" in params: - encoding = params['charset'] - self.content = content - self.parsed = html5lib.parse( - self.content, - transport_encoding=encoding, - namespaceHTMLElements=False, - ) self.url = url self.headers = headers def __str__(self): return self.url - @classmethod - def get_page(cls, link, skip_archives=True, session=None): - if session is None: - raise TypeError( - "get_page() missing 1 required keyword argument: 'session'" - ) - - url = link.url - url = url.split('#', 1)[0] - - # Check for VCS schemes that do not support lookup as web pages. - from pipenv.patched.notpip._internal.vcs import VcsSupport - for scheme in VcsSupport.schemes: - if url.lower().startswith(scheme) and url[len(scheme)] in '+:': - logger.debug('Cannot look at %s URL %s', scheme, link) - return None - - try: - if skip_archives: - filename = link.filename - for bad_ext in ARCHIVE_EXTENSIONS: - if filename.endswith(bad_ext): - content_type = cls._get_content_type( - url, session=session, - ) - if content_type.lower().startswith('text/html'): - break - else: - logger.debug( - 'Skipping page %s because of Content-Type: %s', - link, - content_type, - ) - return - - logger.debug('Getting page %s', url) - - # Tack index.html onto file:// URLs that point to directories - (scheme, netloc, path, params, query, fragment) = \ - urllib_parse.urlparse(url) - if (scheme == 'file' and - os.path.isdir(urllib_request.url2pathname(path))): - # add trailing slash if not present so urljoin doesn't trim - # final segment - if not url.endswith('/'): - url += '/' - url = urllib_parse.urljoin(url, 'index.html') - logger.debug(' file: URL is directory, getting %s', url) - - resp = session.get( - url, - headers={ - "Accept": "text/html", - "Cache-Control": "max-age=600", - }, - ) - resp.raise_for_status() - - # The check for archives above only works if the url ends with - # something that looks like an archive. However that is not a - # requirement of an url. Unless we issue a HEAD request on every - # url we cannot know ahead of time for sure if something is HTML - # or not. However we can check after we've downloaded it. - content_type = resp.headers.get('Content-Type', 'unknown') - if not content_type.lower().startswith("text/html"): - logger.debug( - 'Skipping page %s because of Content-Type: %s', - link, - content_type, - ) - return - - inst = cls(resp.content, resp.url, resp.headers) - except requests.HTTPError as exc: - cls._handle_fail(link, exc, url) - except SSLError as exc: - reason = "There was a problem confirming the ssl certificate: " - reason += str(exc) - cls._handle_fail(link, reason, url, meth=logger.info) - except requests.ConnectionError as exc: - cls._handle_fail(link, "connection error: %s" % exc, url) - except requests.Timeout: - cls._handle_fail(link, "timed out", url) - else: - return inst - - @staticmethod - def _handle_fail(link, reason, url, meth=None): - if meth is None: - meth = logger.debug - - meth("Could not fetch URL %s: %s - skipping", link, reason) - - @staticmethod - def _get_content_type(url, session): - """Get the Content-Type of the given url, using a HEAD request""" - scheme, netloc, path, query, fragment = urllib_parse.urlsplit(url) - if scheme not in {'http', 'https'}: - # FIXME: some warning or something? - # assertion error? - return '' - - resp = session.head(url, allow_redirects=True) - resp.raise_for_status() - - return resp.headers.get("Content-Type", "") - - @cached_property - def base_url(self): - bases = [ - x for x in self.parsed.findall(".//base") - if x.get("href") is not None - ] - if bases and bases[0].get("href"): - return bases[0].get("href") - else: - return self.url - - @property - def links(self): + def iter_links(self): """Yields all links in the page""" - for anchor in self.parsed.findall(".//a"): + document = html5lib.parse( + self.content, + transport_encoding=_get_encoding_from_headers(self.headers), + namespaceHTMLElements=False, + ) + base_url = _determine_base_url(document, self.url) + for anchor in document.findall(".//a"): if anchor.get("href"): href = anchor.get("href") - url = self.clean_link( - urllib_parse.urljoin(self.base_url, href) - ) + url = _clean_link(urllib_parse.urljoin(base_url, href)) pyrequire = anchor.get('data-requires-python') pyrequire = unescape(pyrequire) if pyrequire else None - yield Link(url, self, requires_python=pyrequire) - - _clean_re = re.compile(r'[^a-z0-9$&+,/:;=?@.#%_\\|-]', re.I) - - def clean_link(self, url): - """Makes sure a link is fully encoded. That is, if a ' ' shows up in - the link, it will be rewritten to %20 (while not over-quoting - % or other characters).""" - return self._clean_re.sub( - lambda match: '%%%2x' % ord(match.group(0)), url) - - -class Link(object): - - def __init__(self, url, comes_from=None, requires_python=None): - """ - Object representing a parsed link from https://pypi.org/simple/* - - url: - url of the resource pointed to (href of the link) - comes_from: - instance of HTMLPage where the link was found, or string. - requires_python: - String containing the `Requires-Python` metadata field, specified - in PEP 345. This may be specified by a data-requires-python - attribute in the HTML link tag, as described in PEP 503. - """ - - # url can be a UNC windows share - if url.startswith('\\\\'): - url = path_to_url(url) - - self.url = url - self.comes_from = comes_from - self.requires_python = requires_python if requires_python else None - - def __str__(self): - if self.requires_python: - rp = ' (requires-python:%s)' % self.requires_python - else: - rp = '' - if self.comes_from: - return '%s (from %s)%s' % (self.url, self.comes_from, rp) - else: - return str(self.url) - - def __repr__(self): - return '' % self - - def __eq__(self, other): - if not isinstance(other, Link): - return NotImplemented - return self.url == other.url - - def __ne__(self, other): - if not isinstance(other, Link): - return NotImplemented - return self.url != other.url - - def __lt__(self, other): - if not isinstance(other, Link): - return NotImplemented - return self.url < other.url - - def __le__(self, other): - if not isinstance(other, Link): - return NotImplemented - return self.url <= other.url - - def __gt__(self, other): - if not isinstance(other, Link): - return NotImplemented - return self.url > other.url - - def __ge__(self, other): - if not isinstance(other, Link): - return NotImplemented - return self.url >= other.url - - def __hash__(self): - return hash(self.url) - - @property - def filename(self): - _, netloc, path, _, _ = urllib_parse.urlsplit(self.url) - name = posixpath.basename(path.rstrip('/')) or netloc - name = urllib_parse.unquote(name) - assert name, ('URL %r produced no filename' % self.url) - return name - - @property - def scheme(self): - return urllib_parse.urlsplit(self.url)[0] - - @property - def netloc(self): - return urllib_parse.urlsplit(self.url)[1] - - @property - def path(self): - return urllib_parse.unquote(urllib_parse.urlsplit(self.url)[2]) - - def splitext(self): - return splitext(posixpath.basename(self.path.rstrip('/'))) - - @property - def ext(self): - return self.splitext()[1] - - @property - def url_without_fragment(self): - scheme, netloc, path, query, fragment = urllib_parse.urlsplit(self.url) - return urllib_parse.urlunsplit((scheme, netloc, path, query, None)) - - _egg_fragment_re = re.compile(r'[#&]egg=([^&]*)') - - @property - def egg_fragment(self): - match = self._egg_fragment_re.search(self.url) - if not match: - return None - return match.group(1) - - _subdirectory_fragment_re = re.compile(r'[#&]subdirectory=([^&]*)') - - @property - def subdirectory_fragment(self): - match = self._subdirectory_fragment_re.search(self.url) - if not match: - return None - return match.group(1) - - _hash_re = re.compile( - r'(sha1|sha224|sha384|sha256|sha512|md5)=([a-f0-9]+)' - ) - - @property - def hash(self): - match = self._hash_re.search(self.url) - if match: - return match.group(2) - return None - - @property - def hash_name(self): - match = self._hash_re.search(self.url) - if match: - return match.group(1) - return None - - @property - def show_url(self): - return posixpath.basename(self.url.split('#', 1)[0].split('?', 1)[0]) - - @property - def is_wheel(self): - return self.ext == wheel_ext - - @property - def is_artifact(self): - """ - Determines if this points to an actual artifact (e.g. a tarball) or if - it points to an "abstract" thing like a path or a VCS location. - """ - from pipenv.patched.notpip._internal.vcs import vcs - - if self.scheme in vcs.all_schemes: - return False - - return True - - -FormatControl = namedtuple('FormatControl', 'no_binary only_binary') -"""This object has two fields, no_binary and only_binary. - -If a field is falsy, it isn't set. If it is {':all:'}, it should match all -packages except those listed in the other field. Only one field can be set -to {':all:'} at a time. The rest of the time exact package name matches -are listed, with any given package only showing up in one field at a time. -""" - - -def fmt_ctl_handle_mutual_exclude(value, target, other): - new = value.split(',') - while ':all:' in new: - other.clear() - target.clear() - target.add(':all:') - del new[:new.index(':all:') + 1] - if ':none:' not in new: - # Without a none, we want to discard everything as :all: covers it - return - for name in new: - if name == ':none:': - target.clear() - continue - name = canonicalize_name(name) - other.discard(name) - target.add(name) - - -def fmt_ctl_formats(fmt_ctl, canonical_name): - result = {"binary", "source"} - if canonical_name in fmt_ctl.only_binary: - result.discard('source') - elif canonical_name in fmt_ctl.no_binary: - result.discard('binary') - elif ':all:' in fmt_ctl.only_binary: - result.discard('source') - elif ':all:' in fmt_ctl.no_binary: - result.discard('binary') - return frozenset(result) - - -def fmt_ctl_no_binary(fmt_ctl): - fmt_ctl_handle_mutual_exclude( - ':all:', fmt_ctl.no_binary, fmt_ctl.only_binary, - ) + yield Link(url, self.url, requires_python=pyrequire) Search = namedtuple('Search', 'supplied canonical formats') diff --git a/pipenv/patched/notpip/_internal/locations.py b/pipenv/patched/notpip/_internal/locations.py index 29c6db79..3c7d5bd8 100644 --- a/pipenv/patched/notpip/_internal/locations.py +++ b/pipenv/patched/notpip/_internal/locations.py @@ -10,8 +10,8 @@ import sysconfig from distutils import sysconfig as distutils_sysconfig from distutils.command.install import SCHEME_KEYS # type: ignore -from pipenv.patched.notpip._internal.compat import WINDOWS, expanduser from pipenv.patched.notpip._internal.utils import appdirs +from pipenv.patched.notpip._internal.utils.compat import WINDOWS, expanduser # Application Directories USER_CACHE_DIR = appdirs.user_cache_dir("pip") diff --git a/pipenv/patched/notpip/_internal/models/candidate.py b/pipenv/patched/notpip/_internal/models/candidate.py new file mode 100644 index 00000000..9627589e --- /dev/null +++ b/pipenv/patched/notpip/_internal/models/candidate.py @@ -0,0 +1,24 @@ +from pipenv.patched.notpip._vendor.packaging.version import parse as parse_version + +from pipenv.patched.notpip._internal.utils.models import KeyBasedCompareMixin + + +class InstallationCandidate(KeyBasedCompareMixin): + """Represents a potential "candidate" for installation. + """ + + def __init__(self, project, version, location, requires_python=None): + self.project = project + self.version = parse_version(version) + self.location = location + self.requires_python = requires_python + + super(InstallationCandidate, self).__init__( + key=(self.project, self.version, self.location), + defining_class=InstallationCandidate + ) + + def __repr__(self): + return "".format( + self.project, self.version, self.location, + ) diff --git a/pipenv/patched/notpip/_internal/models/format_control.py b/pipenv/patched/notpip/_internal/models/format_control.py new file mode 100644 index 00000000..caad3cba --- /dev/null +++ b/pipenv/patched/notpip/_internal/models/format_control.py @@ -0,0 +1,62 @@ +from pipenv.patched.notpip._vendor.packaging.utils import canonicalize_name + + +class FormatControl(object): + """A helper class for controlling formats from which packages are installed. + If a field is falsy, it isn't set. If it is {':all:'}, it should match all + packages except those listed in the other field. Only one field can be set + to {':all:'} at a time. The rest of the time exact package name matches + are listed, with any given package only showing up in one field at a time. + """ + def __init__(self, no_binary=None, only_binary=None): + self.no_binary = set() if no_binary is None else no_binary + self.only_binary = set() if only_binary is None else only_binary + + def __eq__(self, other): + return self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not self.__eq__(other) + + def __repr__(self): + return "{}({}, {})".format( + self.__class__.__name__, + self.no_binary, + self.only_binary + ) + + @staticmethod + def handle_mutual_excludes(value, target, other): + new = value.split(',') + while ':all:' in new: + other.clear() + target.clear() + target.add(':all:') + del new[:new.index(':all:') + 1] + # Without a none, we want to discard everything as :all: covers it + if ':none:' not in new: + return + for name in new: + if name == ':none:': + target.clear() + continue + name = canonicalize_name(name) + other.discard(name) + target.add(name) + + def get_allowed_formats(self, canonical_name): + result = {"binary", "source"} + if canonical_name in self.only_binary: + result.discard('source') + elif canonical_name in self.no_binary: + result.discard('binary') + elif ':all:' in self.only_binary: + result.discard('source') + elif ':all:' in self.no_binary: + result.discard('binary') + return frozenset(result) + + def disallow_binaries(self): + self.handle_mutual_excludes( + ':all:', self.no_binary, self.only_binary, + ) diff --git a/pipenv/patched/notpip/_internal/models/index.py b/pipenv/patched/notpip/_internal/models/index.py index f9e84894..0983fc9c 100644 --- a/pipenv/patched/notpip/_internal/models/index.py +++ b/pipenv/patched/notpip/_internal/models/index.py @@ -1,15 +1,29 @@ from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse -class Index(object): - def __init__(self, url): +class PackageIndex(object): + """Represents a Package Index and provides easier access to endpoints + """ + + def __init__(self, url, file_storage_domain): + super(PackageIndex, self).__init__() self.url = url self.netloc = urllib_parse.urlsplit(url).netloc - self.simple_url = self.url_to_path('simple') - self.pypi_url = self.url_to_path('pypi') + self.simple_url = self._url_for_path('simple') + self.pypi_url = self._url_for_path('pypi') - def url_to_path(self, path): + # This is part of a temporary hack used to block installs of PyPI + # packages which depend on external urls only necessary until PyPI can + # block such packages themselves + self.file_storage_domain = file_storage_domain + + def _url_for_path(self, path): return urllib_parse.urljoin(self.url, path) -PyPI = Index('https://pypi.org/') +PyPI = PackageIndex( + 'https://pypi.org/', file_storage_domain='files.pythonhosted.org' +) +TestPyPI = PackageIndex( + 'https://test.pypi.org/', file_storage_domain='test-files.pythonhosted.org' +) diff --git a/pipenv/patched/notpip/_internal/models/link.py b/pipenv/patched/notpip/_internal/models/link.py new file mode 100644 index 00000000..686af1d0 --- /dev/null +++ b/pipenv/patched/notpip/_internal/models/link.py @@ -0,0 +1,141 @@ +import posixpath +import re + +from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse + +from pipenv.patched.notpip._internal.download import path_to_url +from pipenv.patched.notpip._internal.utils.misc import splitext +from pipenv.patched.notpip._internal.utils.models import KeyBasedCompareMixin +from pipenv.patched.notpip._internal.wheel import wheel_ext + + +class Link(KeyBasedCompareMixin): + """Represents a parsed link from a Package Index's simple URL + """ + + def __init__(self, url, comes_from=None, requires_python=None): + """ + url: + url of the resource pointed to (href of the link) + comes_from: + instance of HTMLPage where the link was found, or string. + requires_python: + String containing the `Requires-Python` metadata field, specified + in PEP 345. This may be specified by a data-requires-python + attribute in the HTML link tag, as described in PEP 503. + """ + + # url can be a UNC windows share + if url.startswith('\\\\'): + url = path_to_url(url) + + self.url = url + self.comes_from = comes_from + self.requires_python = requires_python if requires_python else None + + super(Link, self).__init__( + key=(self.url), + defining_class=Link + ) + + def __str__(self): + if self.requires_python: + rp = ' (requires-python:%s)' % self.requires_python + else: + rp = '' + if self.comes_from: + return '%s (from %s)%s' % (self.url, self.comes_from, rp) + else: + return str(self.url) + + def __repr__(self): + return '' % self + + @property + def filename(self): + _, netloc, path, _, _ = urllib_parse.urlsplit(self.url) + name = posixpath.basename(path.rstrip('/')) or netloc + name = urllib_parse.unquote(name) + assert name, ('URL %r produced no filename' % self.url) + return name + + @property + def scheme(self): + return urllib_parse.urlsplit(self.url)[0] + + @property + def netloc(self): + return urllib_parse.urlsplit(self.url)[1] + + @property + def path(self): + return urllib_parse.unquote(urllib_parse.urlsplit(self.url)[2]) + + def splitext(self): + return splitext(posixpath.basename(self.path.rstrip('/'))) + + @property + def ext(self): + return self.splitext()[1] + + @property + def url_without_fragment(self): + scheme, netloc, path, query, fragment = urllib_parse.urlsplit(self.url) + return urllib_parse.urlunsplit((scheme, netloc, path, query, None)) + + _egg_fragment_re = re.compile(r'[#&]egg=([^&]*)') + + @property + def egg_fragment(self): + match = self._egg_fragment_re.search(self.url) + if not match: + return None + return match.group(1) + + _subdirectory_fragment_re = re.compile(r'[#&]subdirectory=([^&]*)') + + @property + def subdirectory_fragment(self): + match = self._subdirectory_fragment_re.search(self.url) + if not match: + return None + return match.group(1) + + _hash_re = re.compile( + r'(sha1|sha224|sha384|sha256|sha512|md5)=([a-f0-9]+)' + ) + + @property + def hash(self): + match = self._hash_re.search(self.url) + if match: + return match.group(2) + return None + + @property + def hash_name(self): + match = self._hash_re.search(self.url) + if match: + return match.group(1) + return None + + @property + def show_url(self): + return posixpath.basename(self.url.split('#', 1)[0].split('?', 1)[0]) + + @property + def is_wheel(self): + return self.ext == wheel_ext + + @property + def is_artifact(self): + """ + Determines if this points to an actual artifact (e.g. a tarball) or if + it points to an "abstract" thing like a path or a VCS location. + """ + from pipenv.patched.notpip._internal.vcs import vcs + + if self.scheme in vcs.all_schemes: + return False + + return True diff --git a/pipenv/patched/notpip/_internal/operations/freeze.py b/pipenv/patched/notpip/_internal/operations/freeze.py index 532989bc..b18b98e4 100644 --- a/pipenv/patched/notpip/_internal/operations/freeze.py +++ b/pipenv/patched/notpip/_internal/operations/freeze.py @@ -10,11 +10,13 @@ from pipenv.patched.notpip._vendor.packaging.utils import canonicalize_name from pipenv.patched.notpip._vendor.pkg_resources import RequirementParseError from pipenv.patched.notpip._internal.exceptions import InstallationError -from pipenv.patched.notpip._internal.req import InstallRequirement +from pipenv.patched.notpip._internal.req.constructors import ( + install_req_from_editable, install_req_from_line, +) from pipenv.patched.notpip._internal.req.req_file import COMMENT_RE from pipenv.patched.notpip._internal.utils.deprecation import deprecated from pipenv.patched.notpip._internal.utils.misc import ( - dist_is_editable, get_installed_distributions, + dist_is_editable, get_installed_distributions, make_vcs_requirement_url, ) logger = logging.getLogger(__name__) @@ -99,13 +101,13 @@ def freeze( line = line[2:].strip() else: line = line[len('--editable'):].strip().lstrip('=') - line_req = InstallRequirement.from_editable( + line_req = install_req_from_editable( line, isolated=isolated, wheel_cache=wheel_cache, ) else: - line_req = InstallRequirement.from_line( + line_req = install_req_from_line( COMMENT_RE.sub('', line).strip(), isolated=isolated, wheel_cache=wheel_cache, @@ -166,7 +168,13 @@ class FrozenRequirement(object): _date_re = re.compile(r'-(20\d\d\d\d\d\d)$') @classmethod - def from_dist(cls, dist, dependency_links): + def _init_args_from_dist(cls, dist, dependency_links): + """ + Compute and return arguments (req, editable, comments) to pass to + FrozenRequirement.__init__(). + + This method is for use in FrozenRequirement.from_dist(). + """ location = os.path.normcase(os.path.abspath(dist.location)) comments = [] from pipenv.patched.notpip._internal.vcs import vcs, get_src_requirement @@ -231,12 +239,15 @@ class FrozenRequirement(object): else: rev = '{%s}' % date_match.group(1) editable = True - req = '%s@%s#egg=%s' % ( - svn_location, - rev, - cls.egg_name(dist) - ) - return cls(dist.project_name, req, editable, comments) + egg_name = cls.egg_name(dist) + req = make_vcs_requirement_url(svn_location, rev, egg_name) + + return (req, editable, comments) + + @classmethod + def from_dist(cls, dist, dependency_links): + args = cls._init_args_from_dist(dist, dependency_links) + return cls(dist.project_name, *args) @staticmethod def egg_name(dist): diff --git a/pipenv/patched/notpip/_internal/operations/prepare.py b/pipenv/patched/notpip/_internal/operations/prepare.py index 9ebc3ebd..d61270b4 100644 --- a/pipenv/patched/notpip/_internal/operations/prepare.py +++ b/pipenv/patched/notpip/_internal/operations/prepare.py @@ -7,7 +7,6 @@ import os from pipenv.patched.notpip._vendor import pkg_resources, requests from pipenv.patched.notpip._internal.build_env import BuildEnvironment -from pipenv.patched.notpip._internal.compat import expanduser from pipenv.patched.notpip._internal.download import ( is_dir_url, is_file_url, is_vcs_url, unpack_url, url_to_path, ) @@ -15,6 +14,7 @@ from pipenv.patched.notpip._internal.exceptions import ( DirectoryUrlHashUnsupported, HashUnpinned, InstallationError, PreviousBuildDirError, VcsHashUnsupported, ) +from pipenv.patched.notpip._internal.utils.compat import expanduser from pipenv.patched.notpip._internal.utils.hashes import MissingHashes from pipenv.patched.notpip._internal.utils.logging import indent_log from pipenv.patched.notpip._internal.utils.misc import display_path, normalize_path, rmtree @@ -65,7 +65,7 @@ class DistAbstraction(object): """Return a setuptools Dist object.""" raise NotImplementedError(self.dist) - def prep_for_dist(self, finder): + def prep_for_dist(self, finder, build_isolation): """Ensure that we can get a Dist for this requirement.""" raise NotImplementedError(self.dist) @@ -93,36 +93,36 @@ class IsSDist(DistAbstraction): return dist def prep_for_dist(self, finder, build_isolation): - # Before calling "setup.py egg_info", we need to set-up the build - # environment. - build_requirements = self.req.get_pep_518_info() - should_isolate = build_isolation and build_requirements is not None + # Prepare for building. We need to: + # 1. Load pyproject.toml (if it exists) + # 2. Set up the build environment + + self.req.load_pyproject_toml() + should_isolate = self.req.use_pep517 and build_isolation if should_isolate: - # Haven't implemented PEP 517 yet, so spew a warning about it if - # build-requirements don't include setuptools and wheel. - missing_requirements = {'setuptools', 'wheel'} - { - pkg_resources.Requirement(r).key for r in build_requirements - } - if missing_requirements: + # Isolate in a BuildEnvironment and install the build-time + # requirements. + self.req.build_env = BuildEnvironment() + self.req.build_env.install_requirements( + finder, self.req.pyproject_requires, + "Installing build dependencies" + ) + missing = [] + if self.req.requirements_to_check: + check = self.req.requirements_to_check + missing = self.req.build_env.missing_requirements(check) + if missing: logger.warning( "Missing build requirements in pyproject.toml for %s.", self.req, ) logger.warning( - "This version of pip does not implement PEP 517 so it " - "cannot build a wheel without %s.", - " and ".join(map(repr, sorted(missing_requirements))) + "The project does not specify a build backend, and pip " + "cannot fall back to setuptools without %s.", + " and ".join(map(repr, sorted(missing))) ) - # Isolate in a BuildEnvironment and install the build-time - # requirements. - self.req.build_env = BuildEnvironment() - self.req.build_env.install_requirements( - finder, build_requirements, - "Installing build dependencies" - ) - try: self.req.run_egg_info() except (OSError, TypeError): @@ -136,7 +136,7 @@ class Installed(DistAbstraction): def dist(self, finder): return self.req.satisfied_by - def prep_for_dist(self, finder): + def prep_for_dist(self, finder, build_isolation): pass diff --git a/pipenv/patched/notpip/_internal/pep425tags.py b/pipenv/patched/notpip/_internal/pep425tags.py index 4205f6e0..182c1c88 100644 --- a/pipenv/patched/notpip/_internal/pep425tags.py +++ b/pipenv/patched/notpip/_internal/pep425tags.py @@ -10,7 +10,12 @@ import sysconfig import warnings from collections import OrderedDict -import pipenv.patched.notpip._internal.utils.glibc +try: + import pipenv.patched.notpip._internal.utils.glibc +except ImportError: + import pipenv.patched.notpip.utils.glibc + +from pipenv.patched.notpip._internal.utils.compat import get_extension_suffixes logger = logging.getLogger(__name__) @@ -252,10 +257,9 @@ def get_supported(versions=None, noarch=False, platform=None, abis[0:0] = [abi] abi3s = set() - import imp - for suffix in imp.get_suffixes(): - if suffix[0].startswith('.abi'): - abi3s.add(suffix[0].split('.', 2)[1]) + for suffix in get_extension_suffixes(): + if suffix.startswith('.abi'): + abi3s.add(suffix.split('.', 2)[1]) abis.extend(sorted(list(abi3s))) diff --git a/pipenv/patched/notpip/_internal/pyproject.py b/pipenv/patched/notpip/_internal/pyproject.py new file mode 100644 index 00000000..a47e0f05 --- /dev/null +++ b/pipenv/patched/notpip/_internal/pyproject.py @@ -0,0 +1,144 @@ +from __future__ import absolute_import + +import io +import os + +from pipenv.patched.notpip._vendor import pytoml, six + +from pipenv.patched.notpip._internal.exceptions import InstallationError + + +def _is_list_of_str(obj): + return ( + isinstance(obj, list) and + all(isinstance(item, six.string_types) for item in obj) + ) + + +def load_pyproject_toml(use_pep517, pyproject_toml, setup_py, req_name): + """Load the pyproject.toml file. + + Parameters: + use_pep517 - Has the user requested PEP 517 processing? None + means the user hasn't explicitly specified. + pyproject_toml - Location of the project's pyproject.toml file + setup_py - Location of the project's setup.py file + req_name - The name of the requirement we're processing (for + error reporting) + + Returns: + None if we should use the legacy code path, otherwise a tuple + ( + requirements from pyproject.toml, + name of PEP 517 backend, + requirements we should check are installed after setting + up the build environment + ) + """ + has_pyproject = os.path.isfile(pyproject_toml) + has_setup = os.path.isfile(setup_py) + + if has_pyproject: + with io.open(pyproject_toml, encoding="utf-8") as f: + pp_toml = pytoml.load(f) + build_system = pp_toml.get("build-system") + else: + build_system = None + + # The following cases must use PEP 517 + # We check for use_pep517 equalling False because that + # means the user explicitly requested --no-use-pep517 + if has_pyproject and not has_setup: + if use_pep517 is False: + raise InstallationError( + "Disabling PEP 517 processing is invalid: " + "project does not have a setup.py" + ) + use_pep517 = True + elif build_system and "build-backend" in build_system: + if use_pep517 is False: + raise InstallationError( + "Disabling PEP 517 processing is invalid: " + "project specifies a build backend of {} " + "in pyproject.toml".format( + build_system["build-backend"] + ) + ) + use_pep517 = True + + # If we haven't worked out whether to use PEP 517 yet, + # and the user hasn't explicitly stated a preference, + # we do so if the project has a pyproject.toml file. + elif use_pep517 is None: + use_pep517 = has_pyproject + + # At this point, we know whether we're going to use PEP 517. + assert use_pep517 is not None + + # If we're using the legacy code path, there is nothing further + # for us to do here. + if not use_pep517: + return None + + if build_system is None: + # Either the user has a pyproject.toml with no build-system + # section, or the user has no pyproject.toml, but has opted in + # explicitly via --use-pep517. + # In the absence of any explicit backend specification, we + # assume the setuptools backend, and require wheel and a version + # of setuptools that supports that backend. + build_system = { + "requires": ["setuptools>=38.2.5", "wheel"], + "build-backend": "setuptools.build_meta", + } + + # If we're using PEP 517, we have build system information (either + # from pyproject.toml, or defaulted by the code above). + # Note that at this point, we do not know if the user has actually + # specified a backend, though. + assert build_system is not None + + # Ensure that the build-system section in pyproject.toml conforms + # to PEP 518. + error_template = ( + "{package} has a pyproject.toml file that does not comply " + "with PEP 518: {reason}" + ) + + # Specifying the build-system table but not the requires key is invalid + if "requires" not in build_system: + raise InstallationError( + error_template.format(package=req_name, reason=( + "it has a 'build-system' table but not " + "'build-system.requires' which is mandatory in the table" + )) + ) + + # Error out if requires is not a list of strings + requires = build_system["requires"] + if not _is_list_of_str(requires): + raise InstallationError(error_template.format( + package=req_name, + reason="'build-system.requires' is not a list of strings.", + )) + + backend = build_system.get("build-backend") + check = [] + if backend is None: + # If the user didn't specify a backend, we assume they want to use + # the setuptools backend. But we can't be sure they have included + # a version of setuptools which supplies the backend, or wheel + # (which is neede by the backend) in their requirements. So we + # make a note to check that those requirements are present once + # we have set up the environment. + # TODO: Review this - it's quite a lot of work to check for a very + # specific case. The problem is, that case is potentially quite + # common - projects that adopted PEP 518 early for the ability to + # specify requirements to execute setup.py, but never considered + # needing to mention the build tools themselves. The original PEP + # 518 code had a similar check (but implemented in a different + # way). + backend = "setuptools.build_meta" + check = ["setuptools>=38.2.5", "wheel"] + + return (requires, backend, check) diff --git a/pipenv/patched/notpip/_internal/req/constructors.py b/pipenv/patched/notpip/_internal/req/constructors.py new file mode 100644 index 00000000..9fe28d88 --- /dev/null +++ b/pipenv/patched/notpip/_internal/req/constructors.py @@ -0,0 +1,298 @@ +"""Backing implementation for InstallRequirement's various constructors + +The idea here is that these formed a major chunk of InstallRequirement's size +so, moving them and support code dedicated to them outside of that class +helps creates for better understandability for the rest of the code. + +These are meant to be used elsewhere within pip to create instances of +InstallRequirement. +""" + +import logging +import os +import re +import traceback + +from pipenv.patched.notpip._vendor.packaging.markers import Marker +from pipenv.patched.notpip._vendor.packaging.requirements import InvalidRequirement, Requirement +from pipenv.patched.notpip._vendor.packaging.specifiers import Specifier +from pipenv.patched.notpip._vendor.pkg_resources import RequirementParseError, parse_requirements + +from pipenv.patched.notpip._internal.download import ( + is_archive_file, is_url, path_to_url, url_to_path, +) +from pipenv.patched.notpip._internal.exceptions import InstallationError +from pipenv.patched.notpip._internal.models.index import PyPI, TestPyPI +from pipenv.patched.notpip._internal.models.link import Link +from pipenv.patched.notpip._internal.req.req_install import InstallRequirement +from pipenv.patched.notpip._internal.utils.misc import is_installable_dir +from pipenv.patched.notpip._internal.vcs import vcs +from pipenv.patched.notpip._internal.wheel import Wheel + +__all__ = [ + "install_req_from_editable", "install_req_from_line", + "parse_editable" +] + +logger = logging.getLogger(__name__) +operators = Specifier._operators.keys() + + +def _strip_extras(path): + m = re.match(r'^(.+)(\[[^\]]+\])$', path) + extras = None + if m: + path_no_extras = m.group(1) + extras = m.group(2) + else: + path_no_extras = path + + return path_no_extras, extras + + +def parse_editable(editable_req): + """Parses an editable requirement into: + - a requirement name + - an URL + - extras + - editable options + Accepted requirements: + svn+http://blahblah@rev#egg=Foobar[baz]&subdirectory=version_subdir + .[some_extra] + """ + + url = editable_req + + # If a file path is specified with extras, strip off the extras. + url_no_extras, extras = _strip_extras(url) + + if os.path.isdir(url_no_extras): + if not os.path.exists(os.path.join(url_no_extras, 'setup.py')): + raise InstallationError( + "Directory %r is not installable. File 'setup.py' not found." % + url_no_extras + ) + # Treating it as code that has already been checked out + url_no_extras = path_to_url(url_no_extras) + + if url_no_extras.lower().startswith('file:'): + package_name = Link(url_no_extras).egg_fragment + if extras: + return ( + package_name, + url_no_extras, + Requirement("placeholder" + extras.lower()).extras, + ) + else: + return package_name, url_no_extras, None + + for version_control in vcs: + if url.lower().startswith('%s:' % version_control): + url = '%s+%s' % (version_control, url) + break + + if '+' not in url: + raise InstallationError( + '%s should either be a path to a local project or a VCS url ' + 'beginning with svn+, git+, hg+, or bzr+' % + editable_req + ) + + vc_type = url.split('+', 1)[0].lower() + + if not vcs.get_backend(vc_type): + error_message = 'For --editable=%s only ' % editable_req + \ + ', '.join([backend.name + '+URL' for backend in vcs.backends]) + \ + ' is currently supported' + raise InstallationError(error_message) + + package_name = Link(url).egg_fragment + if not package_name: + raise InstallationError( + "Could not detect requirement name for '%s', please specify one " + "with #egg=your_package_name" % editable_req + ) + return package_name, url, None + + +def deduce_helpful_msg(req): + """Returns helpful msg in case requirements file does not exist, + or cannot be parsed. + + :params req: Requirements file path + """ + msg = "" + if os.path.exists(req): + msg = " It does exist." + # Try to parse and check if it is a requirements file. + try: + with open(req, 'r') as fp: + # parse first line only + next(parse_requirements(fp.read())) + msg += " The argument you provided " + \ + "(%s) appears to be a" % (req) + \ + " requirements file. If that is the" + \ + " case, use the '-r' flag to install" + \ + " the packages specified within it." + except RequirementParseError: + logger.debug("Cannot parse '%s' as requirements \ + file" % (req), exc_info=1) + else: + msg += " File '%s' does not exist." % (req) + return msg + + +# ---- The actual constructors follow ---- + + +def install_req_from_editable( + editable_req, comes_from=None, isolated=False, options=None, + wheel_cache=None, constraint=False +): + name, url, extras_override = parse_editable(editable_req) + if url.startswith('file:'): + source_dir = url_to_path(url) + else: + source_dir = None + + if name is not None: + try: + req = Requirement(name) + except InvalidRequirement: + raise InstallationError("Invalid requirement: '%s'" % name) + else: + req = None + return InstallRequirement( + req, comes_from, source_dir=source_dir, + editable=True, + link=Link(url), + constraint=constraint, + isolated=isolated, + options=options if options else {}, + wheel_cache=wheel_cache, + extras=extras_override or (), + ) + + +def install_req_from_line( + name, comes_from=None, isolated=False, options=None, wheel_cache=None, + constraint=False +): + """Creates an InstallRequirement from a name, which might be a + requirement, directory containing 'setup.py', filename, or URL. + """ + if is_url(name): + marker_sep = '; ' + else: + marker_sep = ';' + if marker_sep in name: + name, markers = name.split(marker_sep, 1) + markers = markers.strip() + if not markers: + markers = None + else: + markers = Marker(markers) + else: + markers = None + name = name.strip() + req = None + path = os.path.normpath(os.path.abspath(name)) + link = None + extras = None + + if is_url(name): + link = Link(name) + else: + p, extras = _strip_extras(path) + looks_like_dir = os.path.isdir(p) and ( + os.path.sep in name or + (os.path.altsep is not None and os.path.altsep in name) or + name.startswith('.') + ) + if looks_like_dir: + if not is_installable_dir(p): + raise InstallationError( + "Directory %r is not installable. Neither 'setup.py' " + "nor 'pyproject.toml' found." % name + ) + link = Link(path_to_url(p)) + elif is_archive_file(p): + if not os.path.isfile(p): + logger.warning( + 'Requirement %r looks like a filename, but the ' + 'file does not exist', + name + ) + link = Link(path_to_url(p)) + + # it's a local file, dir, or url + if link: + # Handle relative file URLs + if link.scheme == 'file' and re.search(r'\.\./', link.url): + link = Link( + path_to_url(os.path.normpath(os.path.abspath(link.path)))) + # wheel file + if link.is_wheel: + wheel = Wheel(link.filename) # can raise InvalidWheelFilename + req = "%s==%s" % (wheel.name, wheel.version) + else: + # set the req to the egg fragment. when it's not there, this + # will become an 'unnamed' requirement + req = link.egg_fragment + + # a requirement specifier + else: + req = name + + if extras: + extras = Requirement("placeholder" + extras.lower()).extras + else: + extras = () + if req is not None: + try: + req = Requirement(req) + except InvalidRequirement: + if os.path.sep in req: + add_msg = "It looks like a path." + add_msg += deduce_helpful_msg(req) + elif '=' in req and not any(op in req for op in operators): + add_msg = "= is not a valid operator. Did you mean == ?" + else: + add_msg = traceback.format_exc() + raise InstallationError( + "Invalid requirement: '%s'\n%s" % (req, add_msg) + ) + + return InstallRequirement( + req, comes_from, link=link, markers=markers, + isolated=isolated, + options=options if options else {}, + wheel_cache=wheel_cache, + constraint=constraint, + extras=extras, + ) + + +def install_req_from_req( + req, comes_from=None, isolated=False, wheel_cache=None +): + try: + req = Requirement(req) + except InvalidRequirement: + raise InstallationError("Invalid requirement: '%s'" % req) + + domains_not_allowed = [ + PyPI.file_storage_domain, + TestPyPI.file_storage_domain, + ] + if req.url and comes_from.link.netloc in domains_not_allowed: + # Explicitly disallow pypi packages that depend on external urls + raise InstallationError( + "Packages installed from PyPI cannot depend on packages " + "which are not also hosted on PyPI.\n" + "%s depends on %s " % (comes_from.name, req) + ) + + return InstallRequirement( + req, comes_from, isolated=isolated, wheel_cache=wheel_cache + ) diff --git a/pipenv/patched/notpip/_internal/req/req_file.py b/pipenv/patched/notpip/_internal/req/req_file.py index 66a58022..5f23cd3a 100644 --- a/pipenv/patched/notpip/_internal/req/req_file.py +++ b/pipenv/patched/notpip/_internal/req/req_file.py @@ -13,10 +13,12 @@ import sys from pipenv.patched.notpip._vendor.six.moves import filterfalse from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse -from pipenv.patched.notpip._internal import cmdoptions +from pipenv.patched.notpip._internal.cli import cmdoptions from pipenv.patched.notpip._internal.download import get_file_content from pipenv.patched.notpip._internal.exceptions import RequirementsFileParseError -from pipenv.patched.notpip._internal.req.req_install import InstallRequirement +from pipenv.patched.notpip._internal.req.constructors import ( + install_req_from_editable, install_req_from_line, +) __all__ = ['parse_requirements'] @@ -151,7 +153,7 @@ def process_line(line, filename, line_number, finder=None, comes_from=None, for dest in SUPPORTED_OPTIONS_REQ_DEST: if dest in opts.__dict__ and opts.__dict__[dest]: req_options[dest] = opts.__dict__[dest] - yield InstallRequirement.from_line( + yield install_req_from_line( args_str, line_comes_from, constraint=constraint, isolated=isolated, options=req_options, wheel_cache=wheel_cache ) @@ -159,7 +161,7 @@ def process_line(line, filename, line_number, finder=None, comes_from=None, # yield an editable requirement elif opts.editables: isolated = options.isolated_mode if options else False - yield InstallRequirement.from_editable( + yield install_req_from_editable( opts.editables[0], comes_from=line_comes_from, constraint=constraint, isolated=isolated, wheel_cache=wheel_cache ) diff --git a/pipenv/patched/notpip/_internal/req/req_install.py b/pipenv/patched/notpip/_internal/req/req_install.py index a9c642c0..3f32892c 100644 --- a/pipenv/patched/notpip/_internal/req/req_install.py +++ b/pipenv/patched/notpip/_internal/req/req_install.py @@ -1,66 +1,46 @@ from __future__ import absolute_import -import io import logging import os -import re import shutil import sys import sysconfig -import traceback import zipfile from distutils.util import change_root -from email.parser import FeedParser # type: ignore -from pipenv.patched.notpip._vendor import pkg_resources, pytoml, six -from pipenv.patched.notpip._vendor.packaging import specifiers -from pipenv.patched.notpip._vendor.packaging.markers import Marker -from pipenv.patched.notpip._vendor.packaging.requirements import InvalidRequirement, Requirement +from pipenv.patched.notpip._vendor import pkg_resources, six +from pipenv.patched.notpip._vendor.packaging.requirements import Requirement from pipenv.patched.notpip._vendor.packaging.utils import canonicalize_name from pipenv.patched.notpip._vendor.packaging.version import Version from pipenv.patched.notpip._vendor.packaging.version import parse as parse_version -from pipenv.patched.notpip._vendor.pkg_resources import RequirementParseError, parse_requirements +from pipenv.patched.notpip._vendor.pep517.wrappers import Pep517HookCaller from pipenv.patched.notpip._internal import wheel from pipenv.patched.notpip._internal.build_env import NoOpBuildEnvironment -from pipenv.patched.notpip._internal.compat import native_str -from pipenv.patched.notpip._internal.download import ( - is_archive_file, is_url, path_to_url, url_to_path, -) from pipenv.patched.notpip._internal.exceptions import InstallationError from pipenv.patched.notpip._internal.locations import ( PIP_DELETE_MARKER_FILENAME, running_under_virtualenv, ) +from pipenv.patched.notpip._internal.models.link import Link +from pipenv.patched.notpip._internal.pyproject import load_pyproject_toml from pipenv.patched.notpip._internal.req.req_uninstall import UninstallPathSet +from pipenv.patched.notpip._internal.utils.compat import native_str from pipenv.patched.notpip._internal.utils.hashes import Hashes from pipenv.patched.notpip._internal.utils.logging import indent_log from pipenv.patched.notpip._internal.utils.misc import ( _make_build_dir, ask_path_exists, backup_dir, call_subprocess, display_path, dist_in_site_packages, dist_in_usersite, ensure_dir, - get_installed_version, is_installable_dir, read_text_file, rmtree, + get_installed_version, rmtree, ) +from pipenv.patched.notpip._internal.utils.packaging import get_metadata from pipenv.patched.notpip._internal.utils.setuptools_build import SETUPTOOLS_SHIM from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory from pipenv.patched.notpip._internal.utils.ui import open_spinner from pipenv.patched.notpip._internal.vcs import vcs -from pipenv.patched.notpip._internal.wheel import Wheel, move_wheel_files +from pipenv.patched.notpip._internal.wheel import move_wheel_files logger = logging.getLogger(__name__) -operators = specifiers.Specifier._operators.keys() - - -def _strip_extras(path): - m = re.match(r'^(.+)(\[[^\]]+\])$', path) - extras = None - if m: - path_no_extras = m.group(1) - extras = m.group(2) - else: - path_no_extras = path - - return path_no_extras, extras - class InstallRequirement(object): """ @@ -87,7 +67,6 @@ class InstallRequirement(object): if link is not None: self.link = self.original_link = link else: - from pipenv.patched.notpip._internal.index import Link self.link = self.original_link = req and req.url and Link(req.url) if extras: @@ -128,155 +107,32 @@ class InstallRequirement(object): self.isolated = isolated self.build_env = NoOpBuildEnvironment() - # Constructors - # TODO: Move these out of this class into custom methods. - @classmethod - def from_editable(cls, editable_req, comes_from=None, isolated=False, - options=None, wheel_cache=None, constraint=False): - from pipenv.patched.notpip._internal.index import Link + # The static build requirements (from pyproject.toml) + self.pyproject_requires = None - name, url, extras_override = parse_editable(editable_req) - if url.startswith('file:'): - source_dir = url_to_path(url) - else: - source_dir = None + # Build requirements that we will check are available + # TODO: We don't do this for --no-build-isolation. Should we? + self.requirements_to_check = [] - if name is not None: - try: - req = Requirement(name) - except InvalidRequirement: - raise InstallationError("Invalid requirement: '%s'" % name) - else: - req = None - return cls( - req, comes_from, source_dir=source_dir, - editable=True, - link=Link(url), - constraint=constraint, - isolated=isolated, - options=options if options else {}, - wheel_cache=wheel_cache, - extras=extras_override or (), - ) + # The PEP 517 backend we should use to build the project + self.pep517_backend = None - @classmethod - def from_req(cls, req, comes_from=None, isolated=False, wheel_cache=None): - try: - req = Requirement(req) - except InvalidRequirement: - raise InstallationError("Invalid requirement: '%s'" % req) - if req.url: - raise InstallationError( - "Direct url requirement (like %s) are not allowed for " - "dependencies" % req - ) - return cls(req, comes_from, isolated=isolated, wheel_cache=wheel_cache) - - @classmethod - def from_line( - cls, name, comes_from=None, isolated=False, options=None, - wheel_cache=None, constraint=False): - """Creates an InstallRequirement from a name, which might be a - requirement, directory containing 'setup.py', filename, or URL. - """ - from pipenv.patched.notpip._internal.index import Link - - if is_url(name): - marker_sep = '; ' - else: - marker_sep = ';' - if marker_sep in name: - name, markers = name.split(marker_sep, 1) - markers = markers.strip() - if not markers: - markers = None - else: - markers = Marker(markers) - else: - markers = None - name = name.strip() - req = None - path = os.path.normpath(os.path.abspath(name)) - link = None - extras = None - - if is_url(name): - link = Link(name) - else: - p, extras = _strip_extras(path) - looks_like_dir = os.path.isdir(p) and ( - os.path.sep in name or - (os.path.altsep is not None and os.path.altsep in name) or - name.startswith('.') - ) - if looks_like_dir: - if not is_installable_dir(p): - raise InstallationError( - "Directory %r is not installable. File 'setup.py' " - "not found." % name - ) - link = Link(path_to_url(p)) - elif is_archive_file(p): - if not os.path.isfile(p): - logger.warning( - 'Requirement %r looks like a filename, but the ' - 'file does not exist', - name - ) - link = Link(path_to_url(p)) - - # it's a local file, dir, or url - if link: - # Handle relative file URLs - if link.scheme == 'file' and re.search(r'\.\./', link.url): - link = Link( - path_to_url(os.path.normpath(os.path.abspath(link.path)))) - # wheel file - if link.is_wheel: - wheel = Wheel(link.filename) # can raise InvalidWheelFilename - req = "%s==%s" % (wheel.name, wheel.version) - else: - # set the req to the egg fragment. when it's not there, this - # will become an 'unnamed' requirement - req = link.egg_fragment - - # a requirement specifier - else: - req = name - - if extras: - extras = Requirement("placeholder" + extras.lower()).extras - else: - extras = () - if req is not None: - try: - req = Requirement(req) - except InvalidRequirement: - if os.path.sep in req: - add_msg = "It looks like a path." - add_msg += deduce_helpful_msg(req) - elif '=' in req and not any(op in req for op in operators): - add_msg = "= is not a valid operator. Did you mean == ?" - else: - add_msg = traceback.format_exc() - raise InstallationError( - "Invalid requirement: '%s'\n%s" % (req, add_msg)) - return cls( - req, comes_from, link=link, markers=markers, - isolated=isolated, - options=options if options else {}, - wheel_cache=wheel_cache, - constraint=constraint, - extras=extras, - ) + # Are we using PEP 517 for this requirement? + # After pyproject.toml has been loaded, the only valid values are True + # and False. Before loading, None is valid (meaning "use the default"). + # Setting an explicit value before loading pyproject.toml is supported, + # but after loading this flag should be treated as read only. + self.use_pep517 = None def __str__(self): if self.req: s = str(self.req) if self.link: s += ' from %s' % self.link.url + elif self.link: + s = self.link.url else: - s = self.link.url if self.link else None + s = '' if self.satisfied_by is not None: s += ' in %s' % display_path(self.satisfied_by.location) if self.comes_from: @@ -429,7 +285,7 @@ class InstallRequirement(object): package is not available until we run egg_info, so the build_location will return a temporary directory and store the _ideal_build_dir. - This is only called by self.egg_info_path to fix the temporary build + This is only called by self.run_egg_info to fix the temporary build directory. """ if self.source_dir is not None: @@ -557,48 +413,29 @@ class InstallRequirement(object): return pp_toml - def get_pep_518_info(self): - """Get PEP 518 build-time requirements. + def load_pyproject_toml(self): + """Load the pyproject.toml file. - Returns the list of the packages required to build the project, - specified as per PEP 518 within the package. If `pyproject.toml` is not - present, returns None to signify not using the same. + After calling this routine, all of the attributes related to PEP 517 + processing for this requirement have been set. In particular, the + use_pep517 attribute can be used to determine whether we should + follow the PEP 517 or legacy (setup.py) code path. """ - # If pyproject.toml does not exist, don't do anything. - if not os.path.isfile(self.pyproject_toml): - return None - - error_template = ( - "{package} has a pyproject.toml file that does not comply " - "with PEP 518: {reason}" + pep517_data = load_pyproject_toml( + self.use_pep517, + self.pyproject_toml, + self.setup_py, + str(self) ) - with io.open(self.pyproject_toml, encoding="utf-8") as f: - pp_toml = pytoml.load(f) - - # If there is no build-system table, just use setuptools and wheel. - if "build-system" not in pp_toml: - return ["setuptools", "wheel"] - - # Specifying the build-system table but not the requires key is invalid - build_system = pp_toml["build-system"] - if "requires" not in build_system: - raise InstallationError( - error_template.format(package=self, reason=( - "it has a 'build-system' table but not " - "'build-system.requires' which is mandatory in the table" - )) - ) - - # Error out if it's not a list of strings - requires = build_system["requires"] - if not _is_list_of_str(requires): - raise InstallationError(error_template.format( - package=self, - reason="'build-system.requires' is not a list of strings.", - )) - - return requires + if pep517_data is None: + self.use_pep517 = False + else: + self.use_pep517 = True + requires, backend, check = pep517_data + self.requirements_to_check = check + self.pyproject_requires = requires + self.pep517_backend = Pep517HookCaller(self.setup_py_dir, backend) def run_egg_info(self): assert self.source_dir @@ -615,7 +452,8 @@ class InstallRequirement(object): with indent_log(): script = SETUPTOOLS_SHIM % self.setup_py - base_cmd = [os.environ.get('PIP_PYTHON_PATH', sys.executable), '-c', script] + sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) + base_cmd = [sys_executable, '-c', script] if self.isolated: base_cmd += ["--no-user-cfg"] egg_info_cmd = base_cmd + ['egg_info'] @@ -636,20 +474,20 @@ class InstallRequirement(object): command_desc='python setup.py egg_info') if not self.req: - if isinstance(parse_version(self.pkg_info()["Version"]), Version): + if isinstance(parse_version(self.metadata["Version"]), Version): op = "==" else: op = "===" self.req = Requirement( "".join([ - self.pkg_info()["Name"], + self.metadata["Name"], op, - self.pkg_info()["Version"], + self.metadata["Version"], ]) ) self._correct_build_location() else: - metadata_name = canonicalize_name(self.pkg_info()["Name"]) + metadata_name = canonicalize_name(self.metadata["Name"]) if canonicalize_name(self.req.name) != metadata_name: logger.warning( 'Running setup.py (path:%s) egg_info for package %s ' @@ -659,19 +497,8 @@ class InstallRequirement(object): ) self.req = Requirement(metadata_name) - def egg_info_data(self, filename): - if self.satisfied_by is not None: - if not self.satisfied_by.has_metadata(filename): - return None - return self.satisfied_by.get_metadata(filename) - assert self.source_dir - filename = self.egg_info_path(filename) - if not os.path.exists(filename): - return None - data = read_text_file(filename) - return data - - def egg_info_path(self, filename): + @property + def egg_info_path(self): if self._egg_info_path is None: if self.editable: base = self.source_dir @@ -709,8 +536,7 @@ class InstallRequirement(object): if not filenames: raise InstallationError( - "Files/directories (from %s) not found in %s" - % (filename, base) + "Files/directories not found in %s" % base ) # if we have more than one match, we pick the toplevel one. This # can easily be the case if there is a dist folder which contains @@ -721,24 +547,18 @@ class InstallRequirement(object): (os.path.altsep and x.count(os.path.altsep) or 0) ) self._egg_info_path = os.path.join(base, filenames[0]) - return os.path.join(self._egg_info_path, filename) + return self._egg_info_path - def pkg_info(self): - p = FeedParser() - data = self.egg_info_data('PKG-INFO') - if not data: - logger.warning( - 'No PKG-INFO file found in %s', - display_path(self.egg_info_path('PKG-INFO')), - ) - p.feed(data or '') - return p.close() + @property + def metadata(self): + if not hasattr(self, '_metadata'): + self._metadata = get_metadata(self.get_dist()) - _requirements_section_re = re.compile(r'\[(.*?)\]') + return self._metadata def get_dist(self): """Return a pkg_resources.Distribution built from self.egg_info_path""" - egg_info = self.egg_info_path('').rstrip(os.path.sep) + egg_info = self.egg_info_path.rstrip(os.path.sep) base_dir = os.path.dirname(egg_info) metadata = pkg_resources.PathMetadata(base_dir, egg_info) dist_name = os.path.splitext(os.path.basename(egg_info))[0] @@ -750,7 +570,7 @@ class InstallRequirement(object): def assert_source_matches_version(self): assert self.source_dir - version = self.pkg_info()['version'] + version = self.metadata['version'] if self.req.specifier and version not in self.req.specifier: logger.warning( 'Requested %s, but installing version %s', @@ -794,10 +614,11 @@ class InstallRequirement(object): with indent_log(): # FIXME: should we do --install-headers here too? + sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) with self.build_env: call_subprocess( [ - os.environ.get('PIP_PYTHON_PATH', sys.executable), + sys_executable, '-c', SETUPTOOLS_SHIM % self.setup_py ] + @@ -877,7 +698,7 @@ class InstallRequirement(object): def archive(self, build_dir): assert self.source_dir create_archive = True - archive_name = '%s-%s.zip' % (self.name, self.pkg_info()["version"]) + archive_name = '%s-%s.zip' % (self.name, self.metadata["version"]) archive_path = os.path.join(build_dir, archive_name) if os.path.exists(archive_path): response = ask_path_exists( @@ -1015,7 +836,8 @@ class InstallRequirement(object): def get_install_args(self, global_options, record_filename, root, prefix, pycompile): - install_args = [os.environ.get('PIP_PYTHON_PATH', sys.executable), "-u"] + sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) + install_args = [sys_executable, "-u"] install_args.append('-c') install_args.append(SETUPTOOLS_SHIM % self.setup_py) install_args += list(global_options) + \ @@ -1039,104 +861,3 @@ class InstallRequirement(object): py_ver_str, self.name)] return install_args - - -def parse_editable(editable_req): - """Parses an editable requirement into: - - a requirement name - - an URL - - extras - - editable options - Accepted requirements: - svn+http://blahblah@rev#egg=Foobar[baz]&subdirectory=version_subdir - .[some_extra] - """ - - from pipenv.patched.notpip._internal.index import Link - - url = editable_req - - # If a file path is specified with extras, strip off the extras. - url_no_extras, extras = _strip_extras(url) - - if os.path.isdir(url_no_extras): - if not os.path.exists(os.path.join(url_no_extras, 'setup.py')): - raise InstallationError( - "Directory %r is not installable. File 'setup.py' not found." % - url_no_extras - ) - # Treating it as code that has already been checked out - url_no_extras = path_to_url(url_no_extras) - - if url_no_extras.lower().startswith('file:'): - package_name = Link(url_no_extras).egg_fragment - if extras: - return ( - package_name, - url_no_extras, - Requirement("placeholder" + extras.lower()).extras, - ) - else: - return package_name, url_no_extras, None - - for version_control in vcs: - if url.lower().startswith('%s:' % version_control): - url = '%s+%s' % (version_control, url) - break - - if '+' not in url: - raise InstallationError( - '%s should either be a path to a local project or a VCS url ' - 'beginning with svn+, git+, hg+, or bzr+' % - editable_req - ) - - vc_type = url.split('+', 1)[0].lower() - - if not vcs.get_backend(vc_type): - error_message = 'For --editable=%s only ' % editable_req + \ - ', '.join([backend.name + '+URL' for backend in vcs.backends]) + \ - ' is currently supported' - raise InstallationError(error_message) - - package_name = Link(url).egg_fragment - if not package_name: - raise InstallationError( - "Could not detect requirement name for '%s', please specify one " - "with #egg=your_package_name" % editable_req - ) - return package_name, url, None - - -def deduce_helpful_msg(req): - """Returns helpful msg in case requirements file does not exist, - or cannot be parsed. - - :params req: Requirements file path - """ - msg = "" - if os.path.exists(req): - msg = " It does exist." - # Try to parse and check if it is a requirements file. - try: - with open(req, 'r') as fp: - # parse first line only - next(parse_requirements(fp.read())) - msg += " The argument you provided " + \ - "(%s) appears to be a" % (req) + \ - " requirements file. If that is the" + \ - " case, use the '-r' flag to install" + \ - " the packages specified within it." - except RequirementParseError: - logger.debug("Cannot parse '%s' as requirements \ - file" % (req), exc_info=1) - else: - msg += " File '%s' does not exist." % (req) - return msg - - -def _is_list_of_str(obj): - return ( - isinstance(obj, list) and - all(isinstance(item, six.string_types) for item in obj) - ) diff --git a/pipenv/patched/notpip/_internal/req/req_set.py b/pipenv/patched/notpip/_internal/req/req_set.py index 2c54c85a..a65851ff 100644 --- a/pipenv/patched/notpip/_internal/req/req_set.py +++ b/pipenv/patched/notpip/_internal/req/req_set.py @@ -12,19 +12,22 @@ logger = logging.getLogger(__name__) class RequirementSet(object): - def __init__(self, require_hashes=False, ignore_compatibility=True): + def __init__(self, require_hashes=False, check_supported_wheels=True, ignore_compatibility=True): """Create a RequirementSet. """ self.requirements = OrderedDict() self.require_hashes = require_hashes + self.check_supported_wheels = check_supported_wheels + if ignore_compatibility: + self.check_supported_wheels = False + self.ignore_compatibility = True if (check_supported_wheels is False or ignore_compatibility is True) else False # Mapping of alias: real_name self.requirement_aliases = {} self.unnamed_requirements = [] self.successfully_downloaded = [] self.reqs_to_cleanup = [] - self.ignore_compatibility = ignore_compatibility def __str__(self): reqs = [req for req in self.requirements.values() @@ -56,17 +59,22 @@ class RequirementSet(object): requirement is applicable and has just been added. """ name = install_req.name + + # If the markers do not match, ignore this requirement. if not install_req.match_markers(extras_requested): - logger.info("Ignoring %s: markers '%s' don't match your " - "environment", install_req.name, - install_req.markers) + logger.info( + "Ignoring %s: markers '%s' don't match your environment", + name, install_req.markers, + ) return [], None - # This check has to come after we filter requirements with the - # environment markers. + # If the wheel is not supported, raise an error. + # Should check this after filtering out based on environment markers to + # allow specifying different wheels based on the environment/OS, in a + # single requirements file. if install_req.link and install_req.link.is_wheel: wheel = Wheel(install_req.link.filename) - if not wheel.supported() and not self.ignore_compatibility: + if self.check_supported_wheels and not wheel.supported(): raise InstallationError( "%s is not a supported wheel on this platform." % wheel.filename @@ -78,59 +86,73 @@ class RequirementSet(object): "a non direct req should have a parent" ) + # Unnamed requirements are scanned again and the requirement won't be + # added as a dependency until after scanning. if not name: # url or path requirement w/o an egg fragment self.unnamed_requirements.append(install_req) return [install_req], None - else: - try: - existing_req = self.get_requirement(name) - except KeyError: - existing_req = None - if (parent_req_name is None and existing_req and not - existing_req.constraint and - existing_req.extras == install_req.extras and not - existing_req.req.specifier == install_req.req.specifier): - raise InstallationError( - 'Double requirement given: %s (already in %s, name=%r)' - % (install_req, existing_req, name)) - if not existing_req: - # Add requirement - self.requirements[name] = install_req - # FIXME: what about other normalizations? E.g., _ vs. -? - if name.lower() != name: - self.requirement_aliases[name.lower()] = name - result = [install_req] - else: - # Assume there's no need to scan, and that we've already - # encountered this for scanning. - result = [] - if not install_req.constraint and existing_req.constraint: - if (install_req.link and not (existing_req.link and - install_req.link.path == existing_req.link.path)): - self.reqs_to_cleanup.append(install_req) - raise InstallationError( - "Could not satisfy constraints for '%s': " - "installation from path or url cannot be " - "constrained to a version" % name, - ) - # If we're now installing a constraint, mark the existing - # object for real installation. - existing_req.constraint = False - existing_req.extras = tuple( - sorted(set(existing_req.extras).union( - set(install_req.extras)))) - logger.debug("Setting %s extras to: %s", - existing_req, existing_req.extras) - # And now we need to scan this. - result = [existing_req] - # Canonicalise to the already-added object for the backref - # check below. - install_req = existing_req - # We return install_req here to allow for the caller to add it to - # the dependency information for the parent package. - return result, install_req + try: + existing_req = self.get_requirement(name) + except KeyError: + existing_req = None + + has_conflicting_requirement = ( + parent_req_name is None and + existing_req and + not existing_req.constraint and + existing_req.extras == install_req.extras and + existing_req.req.specifier != install_req.req.specifier + ) + if has_conflicting_requirement: + raise InstallationError( + "Double requirement given: %s (already in %s, name=%r)" + % (install_req, existing_req, name) + ) + + # When no existing requirement exists, add the requirement as a + # dependency and it will be scanned again after. + if not existing_req: + self.requirements[name] = install_req + # FIXME: what about other normalizations? E.g., _ vs. -? + if name.lower() != name: + self.requirement_aliases[name.lower()] = name + # We'd want to rescan this requirements later + return [install_req], install_req + + # Assume there's no need to scan, and that we've already + # encountered this for scanning. + if install_req.constraint or not existing_req.constraint: + return [], existing_req + + does_not_satisfy_constraint = ( + install_req.link and + not ( + existing_req.link and + install_req.link.path == existing_req.link.path + ) + ) + if does_not_satisfy_constraint: + self.reqs_to_cleanup.append(install_req) + raise InstallationError( + "Could not satisfy constraints for '%s': " + "installation from path or url cannot be " + "constrained to a version" % name, + ) + # If we're now installing a constraint, mark the existing + # object for real installation. + existing_req.constraint = False + existing_req.extras = tuple(sorted( + set(existing_req.extras) | set(install_req.extras) + )) + logger.debug( + "Setting %s extras to: %s", + existing_req, existing_req.extras, + ) + # Return the existing requirement for addition to the parent and + # scanning again. + return [existing_req], existing_req def has_requirement(self, project_name): name = project_name.lower() @@ -152,7 +174,7 @@ class RequirementSet(object): return self.requirements[name] if name in self.requirement_aliases: return self.requirements[self.requirement_aliases[name]] - # raise KeyError("No project with the name %r" % project_name) + pass def cleanup_files(self): """Clean up files, remove builds.""" diff --git a/pipenv/patched/notpip/_internal/req/req_uninstall.py b/pipenv/patched/notpip/_internal/req/req_uninstall.py index 3ccd3265..4cd15d84 100644 --- a/pipenv/patched/notpip/_internal/req/req_uninstall.py +++ b/pipenv/patched/notpip/_internal/req/req_uninstall.py @@ -9,9 +9,9 @@ import sysconfig from pipenv.patched.notpip._vendor import pkg_resources -from pipenv.patched.notpip._internal.compat import WINDOWS, cache_from_source, uses_pycache from pipenv.patched.notpip._internal.exceptions import UninstallationError from pipenv.patched.notpip._internal.locations import bin_py, bin_user +from pipenv.patched.notpip._internal.utils.compat import WINDOWS, cache_from_source, uses_pycache from pipenv.patched.notpip._internal.utils.logging import indent_log from pipenv.patched.notpip._internal.utils.misc import ( FakeFile, ask, dist_in_usersite, dist_is_local, egg_link_path, is_local, @@ -120,6 +120,8 @@ def compress_for_output_listing(paths): folders.add(os.path.dirname(path)) files.add(path) + _normcased_files = set(map(os.path.normcase, files)) + folders = compact(folders) # This walks the tree using os.walk to not miss extra folders @@ -130,8 +132,9 @@ def compress_for_output_listing(paths): if fname.endswith(".pyc"): continue - file_ = os.path.normcase(os.path.join(dirpath, fname)) - if os.path.isfile(file_) and file_ not in files: + file_ = os.path.join(dirpath, fname) + if (os.path.isfile(file_) and + os.path.normcase(file_) not in _normcased_files): # We are skipping this file. Add it to the set. will_skip.add(file_) diff --git a/pipenv/patched/notpip/_internal/resolve.py b/pipenv/patched/notpip/_internal/resolve.py index 461a2bb8..b0d096f9 100644 --- a/pipenv/patched/notpip/_internal/resolve.py +++ b/pipenv/patched/notpip/_internal/resolve.py @@ -18,7 +18,7 @@ from pipenv.patched.notpip._internal.exceptions import ( BestVersionAlreadyInstalled, DistributionNotFound, HashError, HashErrors, UnsupportedPythonVersion, ) -from pipenv.patched.notpip._internal.req.req_install import InstallRequirement +from pipenv.patched.notpip._internal.req.constructors import install_req_from_req from pipenv.patched.notpip._internal.utils.logging import indent_log from pipenv.patched.notpip._internal.utils.misc import dist_in_usersite, ensure_dir from pipenv.patched.notpip._internal.utils.packaging import check_dist_requires_python @@ -249,9 +249,6 @@ class Resolver(object): # Tell user what we are doing for this requirement: # obtain (editable), skipping, processing (local url), collecting # (remote url or package name) - if ignore_requires_python or self.ignore_requires_python: - self.ignore_compatibility = True - if req_to_install.constraint or req_to_install.prepared: return [] @@ -267,7 +264,7 @@ class Resolver(object): try: check_dist_requires_python(dist) except UnsupportedPythonVersion as err: - if self.ignore_compatibility: + if self.ignore_requires_python or self.ignore_compatibility: logger.warning(err.args[0]) else: raise @@ -281,7 +278,7 @@ class Resolver(object): more_reqs = [] def add_req(subreq, extras_requested): - sub_install_req = InstallRequirement.from_req( + sub_install_req = install_req_from_req( str(subreq), req_to_install, isolated=self.isolated, @@ -303,10 +300,10 @@ class Resolver(object): # We add req_to_install before its dependencies, so that we # can refer to it when adding dependencies. if not requirement_set.has_requirement(req_to_install.name): + # 'unnamed' requirements will get added here available_requested = sorted( set(dist.extras) & set(req_to_install.extras) ) - # 'unnamed' requirements will get added here req_to_install.is_direct = True requirement_set.add_requirement( req_to_install, parent_req_name=None, @@ -338,7 +335,7 @@ class Resolver(object): for available in available_requested: if hasattr(dist, '_DistInfoDistribution__dep_map'): for req in dist._DistInfoDistribution__dep_map[available]: - req = InstallRequirement.from_req( + req = install_req_from_req( str(req), req_to_install, isolated=self.isolated, diff --git a/pipenv/patched/notpip/_internal/utils/appdirs.py b/pipenv/patched/notpip/_internal/utils/appdirs.py index 291de7a9..e8e14526 100644 --- a/pipenv/patched/notpip/_internal/utils/appdirs.py +++ b/pipenv/patched/notpip/_internal/utils/appdirs.py @@ -9,7 +9,7 @@ import sys from pipenv.patched.notpip._vendor.six import PY2, text_type -from pipenv.patched.notpip._internal.compat import WINDOWS, expanduser +from pipenv.patched.notpip._internal.utils.compat import WINDOWS, expanduser def user_cache_dir(appname): diff --git a/pipenv/patched/notpip/_internal/compat.py b/pipenv/patched/notpip/_internal/utils/compat.py similarity index 96% rename from pipenv/patched/notpip/_internal/compat.py rename to pipenv/patched/notpip/_internal/utils/compat.py index 6e51e32a..483bfdc8 100644 --- a/pipenv/patched/notpip/_internal/compat.py +++ b/pipenv/patched/notpip/_internal/utils/compat.py @@ -25,6 +25,7 @@ except ImportError: __all__ = [ "ipaddress", "uses_pycache", "console_to_str", "native_str", "get_path_uid", "stdlib_pkgs", "WINDOWS", "samefile", "get_terminal_size", + "get_extension_suffixes", ] @@ -160,6 +161,18 @@ def get_path_uid(path): return file_uid +if sys.version_info >= (3, 4): + from importlib.machinery import EXTENSION_SUFFIXES + + def get_extension_suffixes(): + return EXTENSION_SUFFIXES +else: + from imp import get_suffixes + + def get_extension_suffixes(): + return [suffix[0] for suffix in get_suffixes()] + + def expanduser(path): """ Expand ~ and ~user constructions. diff --git a/pipenv/patched/notpip/_internal/utils/filesystem.py b/pipenv/patched/notpip/_internal/utils/filesystem.py index 91976486..e8d6a2bb 100644 --- a/pipenv/patched/notpip/_internal/utils/filesystem.py +++ b/pipenv/patched/notpip/_internal/utils/filesystem.py @@ -1,7 +1,7 @@ import os import os.path -from pipenv.patched.notpip._internal.compat import get_path_uid +from pipenv.patched.notpip._internal.utils.compat import get_path_uid def check_path_owner(path): diff --git a/pipenv/patched/notpip/_internal/utils/logging.py b/pipenv/patched/notpip/_internal/utils/logging.py index 257a6234..576c4fa0 100644 --- a/pipenv/patched/notpip/_internal/utils/logging.py +++ b/pipenv/patched/notpip/_internal/utils/logging.py @@ -5,7 +5,7 @@ import logging import logging.handlers import os -from pipenv.patched.notpip._internal.compat import WINDOWS +from pipenv.patched.notpip._internal.utils.compat import WINDOWS from pipenv.patched.notpip._internal.utils.misc import ensure_dir try: diff --git a/pipenv/patched/notpip/_internal/utils/misc.py b/pipenv/patched/notpip/_internal/utils/misc.py index e254f3d4..45e5204c 100644 --- a/pipenv/patched/notpip/_internal/utils/misc.py +++ b/pipenv/patched/notpip/_internal/utils/misc.py @@ -26,14 +26,14 @@ from pipenv.patched.notpip._vendor.six import PY2 from pipenv.patched.notpip._vendor.six.moves import input from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse -from pipenv.patched.notpip._internal.compat import ( - WINDOWS, console_to_str, expanduser, stdlib_pkgs, -) from pipenv.patched.notpip._internal.exceptions import CommandError, InstallationError from pipenv.patched.notpip._internal.locations import ( running_under_virtualenv, site_packages, user_site, virtualenv_no_global, write_delete_marker_file, ) +from pipenv.patched.notpip._internal.utils.compat import ( + WINDOWS, console_to_str, expanduser, stdlib_pkgs, +) if PY2: from io import BytesIO as StringIO @@ -96,7 +96,7 @@ def get_prog(): try: prog = os.path.basename(sys.argv[0]) if prog in ('__main__.py', '-c'): - return "%s -m pip" % os.environ.get('PIP_PYTHON_PATH', sys.executable) + return "%s -m pip" % sys.executable else: return prog except (AttributeError, TypeError, IndexError): @@ -187,12 +187,16 @@ def format_size(bytes): def is_installable_dir(path): - """Return True if `path` is a directory containing a setup.py file.""" + """Is path is a directory containing setup.py or pyproject.toml? + """ if not os.path.isdir(path): return False setup_py = os.path.join(path, 'setup.py') if os.path.isfile(setup_py): return True + pyproject_toml = os.path.join(path, 'pyproject.toml') + if os.path.isfile(pyproject_toml): + return True return False @@ -852,6 +856,44 @@ def enum(*sequential, **named): return type('Enum', (), enums) +def make_vcs_requirement_url(repo_url, rev, egg_project_name, subdir=None): + """ + Return the URL for a VCS requirement. + + Args: + repo_url: the remote VCS url, with any needed VCS prefix (e.g. "git+"). + """ + req = '{}@{}#egg={}'.format(repo_url, rev, egg_project_name) + if subdir: + req += '&subdirectory={}'.format(subdir) + + return req + + +def split_auth_from_netloc(netloc): + """ + Parse out and remove the auth information from a netloc. + + Returns: (netloc, (username, password)). + """ + if '@' not in netloc: + return netloc, (None, None) + + # Split from the right because that's how urllib.parse.urlsplit() + # behaves if more than one @ is present (which can be checked using + # the password attribute of urlsplit()'s return value). + auth, netloc = netloc.rsplit('@', 1) + if ':' in auth: + # Split from the left because that's how urllib.parse.urlsplit() + # behaves if more than one : is present (which again can be checked + # using the password attribute of the return value) + user_pass = tuple(auth.split(':', 1)) + else: + user_pass = auth, None + + return netloc, user_pass + + def remove_auth_from_url(url): # Return a copy of url with 'username:password@' removed. # username/pass params are passed to subversion through flags @@ -859,12 +901,11 @@ def remove_auth_from_url(url): # parsed url purl = urllib_parse.urlsplit(url) - stripped_netloc = \ - purl.netloc.split('@')[-1] + netloc, user_pass = split_auth_from_netloc(purl.netloc) # stripped url url_pieces = ( - purl.scheme, stripped_netloc, purl.path, purl.query, purl.fragment + purl.scheme, netloc, purl.path, purl.query, purl.fragment ) surl = urllib_parse.urlunsplit(url_pieces) return surl diff --git a/pipenv/patched/notpip/_internal/utils/models.py b/pipenv/patched/notpip/_internal/utils/models.py new file mode 100644 index 00000000..d5cb80a7 --- /dev/null +++ b/pipenv/patched/notpip/_internal/utils/models.py @@ -0,0 +1,40 @@ +"""Utilities for defining models +""" + +import operator + + +class KeyBasedCompareMixin(object): + """Provides comparision capabilities that is based on a key + """ + + def __init__(self, key, defining_class): + self._compare_key = key + self._defining_class = defining_class + + def __hash__(self): + return hash(self._compare_key) + + def __lt__(self, other): + return self._compare(other, operator.__lt__) + + def __le__(self, other): + return self._compare(other, operator.__le__) + + def __gt__(self, other): + return self._compare(other, operator.__gt__) + + def __ge__(self, other): + return self._compare(other, operator.__ge__) + + def __eq__(self, other): + return self._compare(other, operator.__eq__) + + def __ne__(self, other): + return self._compare(other, operator.__ne__) + + def _compare(self, other, method): + if not isinstance(other, self._defining_class): + return NotImplemented + + return method(self._compare_key, other._compare_key) diff --git a/pipenv/patched/notpip/_internal/utils/outdated.py b/pipenv/patched/notpip/_internal/utils/outdated.py index 6133e6fd..f8b1fe04 100644 --- a/pipenv/patched/notpip/_internal/utils/outdated.py +++ b/pipenv/patched/notpip/_internal/utils/outdated.py @@ -9,8 +9,8 @@ import sys from pipenv.patched.notpip._vendor import lockfile, pkg_resources from pipenv.patched.notpip._vendor.packaging import version as packaging_version -from pipenv.patched.notpip._internal.compat import WINDOWS from pipenv.patched.notpip._internal.index import PackageFinder +from pipenv.patched.notpip._internal.utils.compat import WINDOWS from pipenv.patched.notpip._internal.utils.filesystem import check_path_owner from pipenv.patched.notpip._internal.utils.misc import ensure_dir, get_installed_version @@ -22,16 +22,25 @@ logger = logging.getLogger(__name__) class SelfCheckState(object): def __init__(self, cache_dir): - self.statefile_path = os.path.join(cache_dir, "selfcheck.json") + self.state = {} + self.statefile_path = None - # Load the existing state - try: - with open(self.statefile_path) as statefile: - self.state = json.load(statefile)[sys.prefix] - except (IOError, ValueError, KeyError): - self.state = {} + # Try to load the existing state + if cache_dir: + self.statefile_path = os.path.join(cache_dir, "selfcheck.json") + try: + with open(self.statefile_path) as statefile: + self.state = json.load(statefile)[sys.prefix] + except (IOError, ValueError, KeyError): + # Explicitly suppressing exceptions, since we don't want to + # error out if the cache file is invalid. + pass def save(self, pypi_version, current_time): + # If we do not have a path to cache in, don't bother saving. + if not self.statefile_path: + return + # Check to make sure that we own the directory if not check_path_owner(os.path.dirname(self.statefile_path)): return diff --git a/pipenv/patched/notpip/_internal/utils/packaging.py b/pipenv/patched/notpip/_internal/utils/packaging.py index 13547743..d1e8ecaa 100644 --- a/pipenv/patched/notpip/_internal/utils/packaging.py +++ b/pipenv/patched/notpip/_internal/utils/packaging.py @@ -8,6 +8,7 @@ from pipenv.patched.notpip._vendor import pkg_resources from pipenv.patched.notpip._vendor.packaging import specifiers, version from pipenv.patched.notpip._internal import exceptions +from pipenv.patched.notpip._internal.utils.misc import display_path logger = logging.getLogger(__name__) @@ -35,22 +36,31 @@ def check_requires_python(requires_python): def get_metadata(dist): if (isinstance(dist, pkg_resources.DistInfoDistribution) and dist.has_metadata('METADATA')): - return dist.get_metadata('METADATA') + metadata = dist.get_metadata('METADATA') elif dist.has_metadata('PKG-INFO'): - return dist.get_metadata('PKG-INFO') + metadata = dist.get_metadata('PKG-INFO') + else: + logger.warning("No metadata found in %s", display_path(dist.location)) + metadata = '' + + feed_parser = FeedParser() + feed_parser.feed(metadata) + return feed_parser.close() def check_dist_requires_python(dist, absorb=True): - metadata = get_metadata(dist) - feed_parser = FeedParser() - feed_parser.feed(metadata) - pkg_info_dict = feed_parser.close() + pkg_info_dict = get_metadata(dist) requires_python = pkg_info_dict.get('Requires-Python') - if not absorb: + if absorb: return requires_python try: if not check_requires_python(requires_python): - return requires_python + raise exceptions.UnsupportedPythonVersion( + "%s requires Python '%s' but the running Python is %s" % ( + dist.project_name, + requires_python, + '.'.join(map(str, sys.version_info[:3])),) + ) except specifiers.InvalidSpecifier as e: logger.warning( "Package %s has an invalid Requires-Python entry %s - %s", diff --git a/pipenv/patched/notpip/_internal/utils/temp_dir.py b/pipenv/patched/notpip/_internal/utils/temp_dir.py index ba472b0d..893dc975 100644 --- a/pipenv/patched/notpip/_internal/utils/temp_dir.py +++ b/pipenv/patched/notpip/_internal/utils/temp_dir.py @@ -3,8 +3,10 @@ from __future__ import absolute_import import logging import os.path import tempfile +import warnings from pipenv.patched.notpip._internal.utils.misc import rmtree +from pipenv.vendor.vistir.compat import finalize, ResourceWarning logger = logging.getLogger(__name__) @@ -45,6 +47,20 @@ class TempDirectory(object): self.path = path self.delete = delete self.kind = kind + self._finalizer = None + if path: + self._register_finalizer() + + def _register_finalizer(self): + if self.delete and self.path: + self._finalizer = finalize( + self, + self._cleanup, + self.path, + warn_message=None + ) + else: + self._finalizer = None def __repr__(self): return "<{} {!r}>".format(self.__class__.__name__, self.path) @@ -72,11 +88,27 @@ class TempDirectory(object): self.path = os.path.realpath( tempfile.mkdtemp(prefix="pip-{}-".format(self.kind)) ) + self._register_finalizer() logger.debug("Created temporary directory: {}".format(self.path)) + @classmethod + def _cleanup(cls, name, warn_message=None): + try: + rmtree(name) + except OSError: + pass + else: + if warn_message: + warnings.warn(warn_message, ResourceWarning) + def cleanup(self): """Remove the temporary directory created and reset state """ - if self.path is not None and os.path.exists(self.path): - rmtree(self.path) - self.path = None + if getattr(self._finalizer, "detach", None) and self._finalizer.detach(): + if os.path.exists(self.path): + try: + rmtree(self.path) + except OSError: + pass + else: + self.path = None diff --git a/pipenv/patched/notpip/_internal/utils/ui.py b/pipenv/patched/notpip/_internal/utils/ui.py index b96863cc..6eebd17d 100644 --- a/pipenv/patched/notpip/_internal/utils/ui.py +++ b/pipenv/patched/notpip/_internal/utils/ui.py @@ -15,7 +15,7 @@ from pipenv.patched.notpip._vendor.progress.bar import ( from pipenv.patched.notpip._vendor.progress.helpers import HIDE_CURSOR, SHOW_CURSOR, WritelnMixin from pipenv.patched.notpip._vendor.progress.spinner import Spinner -from pipenv.patched.notpip._internal.compat import WINDOWS +from pipenv.patched.notpip._internal.utils.compat import WINDOWS from pipenv.patched.notpip._internal.utils.logging import get_indentation from pipenv.patched.notpip._internal.utils.misc import format_size from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING diff --git a/pipenv/patched/notpip/_internal/vcs/__init__.py b/pipenv/patched/notpip/_internal/vcs/__init__.py index 146f2829..5aeac633 100644 --- a/pipenv/patched/notpip/_internal/vcs/__init__.py +++ b/pipenv/patched/notpip/_internal/vcs/__init__.py @@ -17,7 +17,7 @@ from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: from typing import Dict, Optional, Tuple # noqa: F401 - from pipenv.patched.notpip._internal.basecommand import Command # noqa: F401 + from pipenv.patched.notpip._internal.cli.base_command import Command # noqa: F401 __all__ = ['vcs', 'get_src_requirement'] @@ -200,12 +200,6 @@ class VersionControl(object): drive, tail = os.path.splitdrive(repo) return repo.startswith(os.path.sep) or drive - # See issue #1083 for why this method was introduced: - # https://github.com/pypa/pip/issues/1083 - def translate_egg_surname(self, surname): - # For example, Django has branches of the form "stable/1.7.x". - return surname.replace('/', '_') - def export(self, location): """ Export the repository at the url to the destination location @@ -213,51 +207,65 @@ class VersionControl(object): """ raise NotImplementedError - def get_url_rev(self, url): + def get_netloc_and_auth(self, netloc, scheme): """ - Returns the correct repository URL and revision by parsing the given - repository URL + Parse the repository URL's netloc, and return the new netloc to use + along with auth information. + + Args: + netloc: the original repository URL netloc. + scheme: the repository URL's scheme without the vcs prefix. + + This is mainly for the Subversion class to override, so that auth + information can be provided via the --username and --password options + instead of through the URL. For other subclasses like Git without + such an option, auth information must stay in the URL. + + Returns: (netloc, (username, password)). + """ + return netloc, (None, None) + + def get_url_rev_and_auth(self, url): + """ + Parse the repository URL to use, and return the URL, revision, + and auth info to use. + + Returns: (url, rev, (username, password)). """ - error_message = ( - "Sorry, '%s' is a malformed VCS url. " - "The format is +://, " - "e.g. svn+http://myrepo/svn/MyApp#egg=MyApp" - ) - assert '+' in url, error_message % url - url = url.split('+', 1)[1] scheme, netloc, path, query, frag = urllib_parse.urlsplit(url) + if '+' not in scheme: + raise ValueError( + "Sorry, {!r} is a malformed VCS url. " + "The format is +://, " + "e.g. svn+http://myrepo/svn/MyApp#egg=MyApp".format(url) + ) + # Remove the vcs prefix. + scheme = scheme.split('+', 1)[1] + netloc, user_pass = self.get_netloc_and_auth(netloc, scheme) rev = None if '@' in path: path, rev = path.rsplit('@', 1) url = urllib_parse.urlunsplit((scheme, netloc, path, query, '')) - return url, rev + return url, rev, user_pass - def get_url_rev_args(self, url): + def make_rev_args(self, username, password): """ - Return the URL and RevOptions "extra arguments" to use in obtain(), - as a tuple (url, extra_args). + Return the RevOptions "extra arguments" to use in obtain(). """ - return url, [] + return [] def get_url_rev_options(self, url): """ Return the URL and RevOptions object to use in obtain() and in some cases export(), as a tuple (url, rev_options). """ - url, rev = self.get_url_rev(url) - url, extra_args = self.get_url_rev_args(url) + url, rev, user_pass = self.get_url_rev_and_auth(url) + username, password = user_pass + extra_args = self.make_rev_args(username, password) rev_options = self.make_rev_options(rev, extra_args=extra_args) return url, rev_options - def get_info(self, location): - """ - Returns (url, revision), where both are strings - """ - assert not location.rstrip('/').endswith(self.dirname), \ - 'Bad directory: %s' % location - return self.get_url(location), self.get_revision(location) - def normalize_url(self, url): """ Normalize a URL for comparison by unquoting it and removing any @@ -291,7 +299,7 @@ class VersionControl(object): """ raise NotImplementedError - def update(self, dest, rev_options): + def update(self, dest, url, rev_options): """ Update an already-existing repo to the given ``rev_options``. @@ -341,7 +349,7 @@ class VersionControl(object): self.repo_name, rev_display, ) - self.update(dest, rev_options) + self.update(dest, url, rev_options) else: logger.info('Skipping because already up-to-date.') return @@ -421,8 +429,6 @@ class VersionControl(object): def get_url(self, location): """ Return the url used at location - - This is used in get_info() and obtain(). """ raise NotImplementedError diff --git a/pipenv/patched/notpip/_internal/vcs/bazaar.py b/pipenv/patched/notpip/_internal/vcs/bazaar.py index b2664cd8..890448ed 100644 --- a/pipenv/patched/notpip/_internal/vcs/bazaar.py +++ b/pipenv/patched/notpip/_internal/vcs/bazaar.py @@ -6,7 +6,9 @@ import os from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse from pipenv.patched.notpip._internal.download import path_to_url -from pipenv.patched.notpip._internal.utils.misc import display_path, rmtree +from pipenv.patched.notpip._internal.utils.misc import ( + display_path, make_vcs_requirement_url, rmtree, +) from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory from pipenv.patched.notpip._internal.vcs import VersionControl, vcs @@ -62,16 +64,16 @@ class Bazaar(VersionControl): def switch(self, dest, url, rev_options): self.run_command(['switch', url], cwd=dest) - def update(self, dest, rev_options): + def update(self, dest, url, rev_options): cmd_args = ['pull', '-q'] + rev_options.to_args() self.run_command(cmd_args, cwd=dest) - def get_url_rev(self, url): + def get_url_rev_and_auth(self, url): # hotfix the URL scheme after removing bzr+ from bzr+ssh:// readd it - url, rev = super(Bazaar, self).get_url_rev(url) + url, rev, user_pass = super(Bazaar, self).get_url_rev_and_auth(url) if url.startswith('ssh://'): url = 'bzr+' + url - return url, rev + return url, rev, user_pass def get_url(self, location): urls = self.run_command(['info'], show_stdout=False, cwd=location) @@ -98,9 +100,9 @@ class Bazaar(VersionControl): return None if not repo.lower().startswith('bzr:'): repo = 'bzr+' + repo - egg_project_name = dist.egg_name().split('-', 1)[0] current_rev = self.get_revision(location) - return '%s@%s#egg=%s' % (repo, current_rev, egg_project_name) + egg_project_name = dist.egg_name().split('-', 1)[0] + return make_vcs_requirement_url(repo, current_rev, egg_project_name) def is_commit_id_equal(self, dest, name): """Always assume the versions don't match""" diff --git a/pipenv/patched/notpip/_internal/vcs/git.py b/pipenv/patched/notpip/_internal/vcs/git.py index ef2dd908..3db56144 100644 --- a/pipenv/patched/notpip/_internal/vcs/git.py +++ b/pipenv/patched/notpip/_internal/vcs/git.py @@ -8,9 +8,9 @@ from pipenv.patched.notpip._vendor.packaging.version import parse as parse_versi from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse from pipenv.patched.notpip._vendor.six.moves.urllib import request as urllib_request -from pipenv.patched.notpip._internal.compat import samefile from pipenv.patched.notpip._internal.exceptions import BadCommand -from pipenv.patched.notpip._internal.utils.misc import display_path +from pipenv.patched.notpip._internal.utils.compat import samefile +from pipenv.patched.notpip._internal.utils.misc import display_path, make_vcs_requirement_url from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory from pipenv.patched.notpip._internal.vcs import VersionControl, vcs @@ -77,6 +77,20 @@ class Git(VersionControl): version = '.'.join(version.split('.')[:3]) return parse_version(version) + def get_branch(self, location): + """ + Return the current branch, or None if HEAD isn't at a branch + (e.g. detached HEAD). + """ + args = ['rev-parse', '--abbrev-ref', 'HEAD'] + output = self.run_command(args, show_stdout=False, cwd=location) + branch = output.strip() + + if branch == 'HEAD': + return None + + return branch + def export(self, location): """Export the Git repository at the url to the destination location""" if not location.endswith('/'): @@ -91,8 +105,8 @@ class Git(VersionControl): def get_revision_sha(self, dest, rev): """ - Return a commit hash for the given revision if it names a remote - branch or tag. Otherwise, return None. + Return (sha_or_none, is_branch), where sha_or_none is a commit hash + if the revision names a remote branch or tag, otherwise None. Args: dest: the repository directory. @@ -115,22 +129,30 @@ class Git(VersionControl): branch_ref = 'refs/remotes/origin/{}'.format(rev) tag_ref = 'refs/tags/{}'.format(rev) - return refs.get(branch_ref) or refs.get(tag_ref) + sha = refs.get(branch_ref) + if sha is not None: + return (sha, True) - def check_rev_options(self, dest, rev_options): - """Check the revision options before checkout. + sha = refs.get(tag_ref) - Returns a new RevOptions object for the SHA1 of the branch or tag - if found. + return (sha, False) + + def resolve_revision(self, dest, url, rev_options): + """ + Resolve a revision to a new RevOptions object with the SHA1 of the + branch, tag, or ref if found. Args: rev_options: a RevOptions object. """ rev = rev_options.arg_rev - sha = self.get_revision_sha(dest, rev) + sha, is_branch = self.get_revision_sha(dest, rev) if sha is not None: - return rev_options.make_new(sha) + rev_options = rev_options.make_new(sha) + rev_options.branch_name = rev if is_branch else None + + return rev_options # Do not show a warning for the common case of something that has # the form of a Git commit hash. @@ -139,6 +161,19 @@ class Git(VersionControl): "Did not find branch or tag '%s', assuming revision or ref.", rev, ) + + if not rev.startswith('refs/'): + return rev_options + + # If it looks like a ref, we have to fetch it explicitly. + self.run_command( + ['fetch', '-q', url] + rev_options.to_args(), + cwd=dest, + ) + # Change the revision to the SHA of the ref we fetched + sha = self.get_revision(dest, rev='FETCH_HEAD') + rev_options = rev_options.make_new(sha) + return rev_options def is_commit_id_equal(self, dest, name): @@ -164,20 +199,22 @@ class Git(VersionControl): if rev_options.rev: # Then a specific revision was requested. - rev_options = self.check_rev_options(dest, rev_options) - # Only do a checkout if the current commit id doesn't match - # the requested revision. - if not self.is_commit_id_equal(dest, rev_options.rev): - rev = rev_options.rev - # Only fetch the revision if it's a ref - if rev.startswith('refs/'): - self.run_command( - ['fetch', '-q', url] + rev_options.to_args(), - cwd=dest, - ) - # Change the revision to the SHA of the ref we fetched - rev = 'FETCH_HEAD' - self.run_command(['checkout', '-q', rev], cwd=dest) + rev_options = self.resolve_revision(dest, url, rev_options) + branch_name = getattr(rev_options, 'branch_name', None) + if branch_name is None: + # Only do a checkout if the current commit id doesn't match + # the requested revision. + if not self.is_commit_id_equal(dest, rev_options.rev): + cmd_args = ['checkout', '-q'] + rev_options.to_args() + self.run_command(cmd_args, cwd=dest) + elif self.get_branch(dest) != branch_name: + # Then a specific branch was requested, and that branch + # is not yet checked out. + track_branch = 'origin/{}'.format(branch_name) + cmd_args = [ + 'checkout', '-b', branch_name, '--track', track_branch, + ] + self.run_command(cmd_args, cwd=dest) #: repo may contain submodules self.update_submodules(dest) @@ -189,7 +226,7 @@ class Git(VersionControl): self.update_submodules(dest) - def update(self, dest, rev_options): + def update(self, dest, url, rev_options): # First fetch changes from the default remote if self.get_git_version() >= parse_version('1.9.0'): # fetch tags in addition to everything else @@ -197,7 +234,7 @@ class Git(VersionControl): else: self.run_command(['fetch', '-q'], cwd=dest) # Then reset to wanted revision (maybe even origin/master) - rev_options = self.check_rev_options(dest, rev_options) + rev_options = self.resolve_revision(dest, url, rev_options) cmd_args = ['reset', '--hard', '-q'] + rev_options.to_args() self.run_command(cmd_args, cwd=dest) #: update submodules @@ -218,9 +255,11 @@ class Git(VersionControl): url = found_remote.split(' ')[1] return url.strip() - def get_revision(self, location): + def get_revision(self, location, rev=None): + if rev is None: + rev = 'HEAD' current_rev = self.run_command( - ['rev-parse', 'HEAD'], show_stdout=False, cwd=location, + ['rev-parse', rev], show_stdout=False, cwd=location, ) return current_rev.strip() @@ -255,17 +294,15 @@ class Git(VersionControl): repo = self.get_url(location) if not repo.lower().startswith('git:'): repo = 'git+' + repo - egg_project_name = dist.egg_name().split('-', 1)[0] - if not repo: - return None current_rev = self.get_revision(location) - req = '%s@%s#egg=%s' % (repo, current_rev, egg_project_name) - subdirectory = self._get_subdirectory(location) - if subdirectory: - req += '&subdirectory=' + subdirectory + egg_project_name = dist.egg_name().split('-', 1)[0] + subdir = self._get_subdirectory(location) + req = make_vcs_requirement_url(repo, current_rev, egg_project_name, + subdir=subdir) + return req - def get_url_rev(self, url): + def get_url_rev_and_auth(self, url): """ Prefixes stub URLs like 'user@hostname:user/repo.git' with 'ssh://'. That's required because although they use SSH they sometimes don't @@ -275,12 +312,12 @@ class Git(VersionControl): if '://' not in url: assert 'file:' not in url url = url.replace('git+', 'git+ssh://') - url, rev = super(Git, self).get_url_rev(url) + url, rev, user_pass = super(Git, self).get_url_rev_and_auth(url) url = url.replace('ssh://', '') else: - url, rev = super(Git, self).get_url_rev(url) + url, rev, user_pass = super(Git, self).get_url_rev_and_auth(url) - return url, rev + return url, rev, user_pass def update_submodules(self, location): if not os.path.exists(os.path.join(location, '.gitmodules')): diff --git a/pipenv/patched/notpip/_internal/vcs/mercurial.py b/pipenv/patched/notpip/_internal/vcs/mercurial.py index a143e765..d76d47f3 100644 --- a/pipenv/patched/notpip/_internal/vcs/mercurial.py +++ b/pipenv/patched/notpip/_internal/vcs/mercurial.py @@ -6,7 +6,7 @@ import os from pipenv.patched.notpip._vendor.six.moves import configparser from pipenv.patched.notpip._internal.download import path_to_url -from pipenv.patched.notpip._internal.utils.misc import display_path +from pipenv.patched.notpip._internal.utils.misc import display_path, make_vcs_requirement_url from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory from pipenv.patched.notpip._internal.vcs import VersionControl, vcs @@ -59,7 +59,7 @@ class Mercurial(VersionControl): cmd_args = ['update', '-q'] + rev_options.to_args() self.run_command(cmd_args, cwd=dest) - def update(self, dest, rev_options): + def update(self, dest, url, rev_options): self.run_command(['pull', '-q'], cwd=dest) cmd_args = ['update', '-q'] + rev_options.to_args() self.run_command(cmd_args, cwd=dest) @@ -88,11 +88,10 @@ class Mercurial(VersionControl): repo = self.get_url(location) if not repo.lower().startswith('hg:'): repo = 'hg+' + repo - egg_project_name = dist.egg_name().split('-', 1)[0] - if not repo: - return None current_rev_hash = self.get_revision_hash(location) - return '%s@%s#egg=%s' % (repo, current_rev_hash, egg_project_name) + egg_project_name = dist.egg_name().split('-', 1)[0] + return make_vcs_requirement_url(repo, current_rev_hash, + egg_project_name) def is_commit_id_equal(self, dest, name): """Always assume the versions don't match""" diff --git a/pipenv/patched/notpip/_internal/vcs/subversion.py b/pipenv/patched/notpip/_internal/vcs/subversion.py index 5adbdaa3..f3c3db4d 100644 --- a/pipenv/patched/notpip/_internal/vcs/subversion.py +++ b/pipenv/patched/notpip/_internal/vcs/subversion.py @@ -4,17 +4,15 @@ import logging import os import re -from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse - -from pipenv.patched.notpip._internal.index import Link +from pipenv.patched.notpip._internal.models.link import Link from pipenv.patched.notpip._internal.utils.logging import indent_log -from pipenv.patched.notpip._internal.utils.misc import display_path, remove_auth_from_url, rmtree +from pipenv.patched.notpip._internal.utils.misc import ( + display_path, make_vcs_requirement_url, rmtree, split_auth_from_netloc, +) from pipenv.patched.notpip._internal.vcs import VersionControl, vcs _svn_xml_url_re = re.compile('url="([^"]+)"') _svn_rev_re = re.compile(r'committed-rev="(\d+)"') -_svn_url_re = re.compile(r'URL: (.+)') -_svn_revision_re = re.compile(r'Revision: (.+)') _svn_info_xml_rev_re = re.compile(r'\s*revision="(\d+)"') _svn_info_xml_url_re = re.compile(r'(.*)') @@ -31,34 +29,6 @@ class Subversion(VersionControl): def get_base_rev_args(self, rev): return ['-r', rev] - def get_info(self, location): - """Returns (url, revision), where both are strings""" - assert not location.rstrip('/').endswith(self.dirname), \ - 'Bad directory: %s' % location - output = self.run_command( - ['info', location], - show_stdout=False, - extra_environ={'LANG': 'C'}, - ) - match = _svn_url_re.search(output) - if not match: - logger.warning( - 'Cannot determine URL of svn checkout %s', - display_path(location), - ) - logger.debug('Output that cannot be parsed: \n%s', output) - return None, None - url = match.group(1).strip() - match = _svn_revision_re.search(output) - if not match: - logger.warning( - 'Cannot determine revision of svn checkout %s', - display_path(location), - ) - logger.debug('Output that cannot be parsed: \n%s', output) - return url, None - return url, match.group(1) - def export(self, location): """Export the svn repository at the url to the destination location""" url, rev_options = self.get_url_rev_options(self.url) @@ -87,7 +57,7 @@ class Subversion(VersionControl): cmd_args = ['switch'] + rev_options.to_args() + [url, dest] self.run_command(cmd_args) - def update(self, dest, rev_options): + def update(self, dest, url, rev_options): cmd_args = ['update'] + rev_options.to_args() + [dest] self.run_command(cmd_args) @@ -132,18 +102,34 @@ class Subversion(VersionControl): revision = max(revision, localrev) return revision - def get_url_rev(self, url): + def get_netloc_and_auth(self, netloc, scheme): + """ + This override allows the auth information to be passed to svn via the + --username and --password options instead of via the URL. + """ + if scheme == 'ssh': + # The --username and --password options can't be used for + # svn+ssh URLs, so keep the auth information in the URL. + return super(Subversion, self).get_netloc_and_auth( + netloc, scheme) + + return split_auth_from_netloc(netloc) + + def get_url_rev_and_auth(self, url): # hotfix the URL scheme after removing svn+ from svn+ssh:// readd it - url, rev = super(Subversion, self).get_url_rev(url) + url, rev, user_pass = super(Subversion, self).get_url_rev_and_auth(url) if url.startswith('ssh://'): url = 'svn+' + url - return url, rev + return url, rev, user_pass - def get_url_rev_args(self, url): - extra_args = get_rev_options_args(url) - url = remove_auth_from_url(url) + def make_rev_args(self, username, password): + extra_args = [] + if username: + extra_args += ['--username', username] + if password: + extra_args += ['--password', password] - return url, extra_args + return extra_args def get_url(self, location): # In cases where the source is in a subdirectory, not alongside @@ -213,42 +199,15 @@ class Subversion(VersionControl): repo = self.get_url(location) if repo is None: return None + repo = 'svn+' + repo + rev = self.get_revision(location) # FIXME: why not project name? egg_project_name = dist.egg_name().split('-', 1)[0] - rev = self.get_revision(location) - return 'svn+%s@%s#egg=%s' % (repo, rev, egg_project_name) + return make_vcs_requirement_url(repo, rev, egg_project_name) def is_commit_id_equal(self, dest, name): """Always assume the versions don't match""" return False -def get_rev_options_args(url): - """ - Return the extra arguments to pass to RevOptions. - """ - r = urllib_parse.urlsplit(url) - if hasattr(r, 'username'): - # >= Python-2.5 - username, password = r.username, r.password - else: - netloc = r[1] - if '@' in netloc: - auth = netloc.split('@')[0] - if ':' in auth: - username, password = auth.split(':', 1) - else: - username, password = auth, None - else: - username, password = None, None - - extra_args = [] - if username: - extra_args += ['--username', username] - if password: - extra_args += ['--password', password] - - return extra_args - - vcs.register(Subversion) diff --git a/pipenv/patched/notpip/_internal/wheel.py b/pipenv/patched/notpip/_internal/wheel.py index 14ec0014..6df5a3a3 100644 --- a/pipenv/patched/notpip/_internal/wheel.py +++ b/pipenv/patched/notpip/_internal/wheel.py @@ -167,7 +167,8 @@ def message_about_scripts_not_on_PATH(scripts): ] # If an executable sits with sys.executable, we don't warn for it. # This covers the case of venv invocations without activating the venv. - not_warn_dirs.append(os.path.normcase(os.path.dirname(sys.executable))) + executable_loc = os.environ.get("PIP_PYTHON_PATH", sys.executable) + not_warn_dirs.append(os.path.normcase(os.path.dirname(executable_loc))) warn_for = { parent_dir: scripts for parent_dir, scripts in grouped_by_dir.items() if os.path.normcase(parent_dir) not in not_warn_dirs @@ -475,7 +476,7 @@ if __name__ == '__main__': if warn_script_location: msg = message_about_scripts_not_on_PATH(generated_console_scripts) if msg is not None: - logger.warn(msg) + logger.warning(msg) if len(gui) > 0: generated.extend( @@ -500,16 +501,19 @@ if __name__ == '__main__': with open_for_csv(temp_record, 'w+') as record_out: reader = csv.reader(record_in) writer = csv.writer(record_out) + outrows = [] for row in reader: row[0] = installed.pop(row[0], row[0]) if row[0] in changed: row[1], row[2] = rehash(row[0]) - writer.writerow(row) + outrows.append(tuple(row)) for f in generated: digest, length = rehash(f) - writer.writerow((normpath(f, lib_dir), digest, length)) + outrows.append((normpath(f, lib_dir), digest, length)) for f in installed: - writer.writerow((installed[f], '', '')) + outrows.append((installed[f], '', '')) + for row in sorted(outrows): + writer.writerow(row) shutil.move(temp_record, record) @@ -664,8 +668,9 @@ class WheelBuilder(object): # isolating. Currently, it breaks Python in virtualenvs, because it # relies on site.py to find parts of the standard library outside the # virtualenv. + executable_loc = os.environ.get('PIP_PYTHON_PATH', sys.executable) return [ - os.environ.get('PIP_PYTHON_PATH', sys.executable), '-u', '-c', + executable_loc, '-u', '-c', SETUPTOOLS_SHIM % req.setup_py ] + list(self.global_options) @@ -710,6 +715,7 @@ class WheelBuilder(object): :return: True if all the wheels built correctly. """ from pipenv.patched.notpip._internal import index + from pipenv.patched.notpip._internal.models.link import Link building_is_possible = self._wheel_dir or ( autobuilding and self.wheel_cache.cache_dir @@ -717,6 +723,7 @@ class WheelBuilder(object): assert building_is_possible buildset = [] + format_control = self.finder.format_control for req in requirements: if req.constraint: continue @@ -740,8 +747,7 @@ class WheelBuilder(object): if index.egg_info_matches(base, None, link) is None: # E.g. local directory. Build wheel just for this run. ephem_cache = True - if "binary" not in index.fmt_ctl_formats( - self.finder.format_control, + if "binary" not in format_control.get_allowed_formats( canonicalize_name(req.name)): logger.info( "Skipping bdist_wheel for %s, due to binaries " @@ -802,7 +808,7 @@ class WheelBuilder(object): self.preparer.build_dir ) # Update the link for this. - req.link = index.Link(path_to_url(wheel_file)) + req.link = Link(path_to_url(wheel_file)) assert req.link.is_wheel # extract the wheel into the dir unpack_url( diff --git a/pipenv/patched/notpip/_vendor/certifi/__init__.py b/pipenv/patched/notpip/_vendor/certifi/__init__.py index 0c4963ef..aa329fbb 100644 --- a/pipenv/patched/notpip/_vendor/certifi/__init__.py +++ b/pipenv/patched/notpip/_vendor/certifi/__init__.py @@ -1,3 +1,3 @@ from .core import where, old_where -__version__ = "2018.04.16" +__version__ = "2018.08.24" diff --git a/pipenv/patched/notpip/_vendor/certifi/__main__.py b/pipenv/patched/notpip/_vendor/certifi/__main__.py index 5f1da0dd..983ed0f9 100644 --- a/pipenv/patched/notpip/_vendor/certifi/__main__.py +++ b/pipenv/patched/notpip/_vendor/certifi/__main__.py @@ -1,2 +1,2 @@ -from certifi import where +from pipenv.patched.notpip._vendor.certifi import where print(where()) diff --git a/pipenv/patched/notpip/_vendor/certifi/cacert.pem b/pipenv/patched/notpip/_vendor/certifi/cacert.pem index 2713f541..85de024e 100644 --- a/pipenv/patched/notpip/_vendor/certifi/cacert.pem +++ b/pipenv/patched/notpip/_vendor/certifi/cacert.pem @@ -3692,169 +3692,6 @@ lSTAGiecMjvAwNW6qef4BENThe5SId6d9SWDPp5YSy/XZxMOIQIwBeF1Ad5o7Sof TUwJCA3sS61kFyjndc5FZXIhF8siQQ6ME5g4mlRtm8rifOoCWCKR -----END CERTIFICATE----- -# Issuer: CN=Certplus Root CA G1 O=Certplus -# Subject: CN=Certplus Root CA G1 O=Certplus -# Label: "Certplus Root CA G1" -# Serial: 1491911565779898356709731176965615564637713 -# MD5 Fingerprint: 7f:09:9c:f7:d9:b9:5c:69:69:56:d5:37:3e:14:0d:42 -# SHA1 Fingerprint: 22:fd:d0:b7:fd:a2:4e:0d:ac:49:2c:a0:ac:a6:7b:6a:1f:e3:f7:66 -# SHA256 Fingerprint: 15:2a:40:2b:fc:df:2c:d5:48:05:4d:22:75:b3:9c:7f:ca:3e:c0:97:80:78:b0:f0:ea:76:e5:61:a6:c7:43:3e ------BEGIN CERTIFICATE----- -MIIFazCCA1OgAwIBAgISESBVg+QtPlRWhS2DN7cs3EYRMA0GCSqGSIb3DQEBDQUA -MD4xCzAJBgNVBAYTAkZSMREwDwYDVQQKDAhDZXJ0cGx1czEcMBoGA1UEAwwTQ2Vy -dHBsdXMgUm9vdCBDQSBHMTAeFw0xNDA1MjYwMDAwMDBaFw0zODAxMTUwMDAwMDBa -MD4xCzAJBgNVBAYTAkZSMREwDwYDVQQKDAhDZXJ0cGx1czEcMBoGA1UEAwwTQ2Vy -dHBsdXMgUm9vdCBDQSBHMTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIB -ANpQh7bauKk+nWT6VjOaVj0W5QOVsjQcmm1iBdTYj+eJZJ+622SLZOZ5KmHNr49a -iZFluVj8tANfkT8tEBXgfs+8/H9DZ6itXjYj2JizTfNDnjl8KvzsiNWI7nC9hRYt -6kuJPKNxQv4c/dMcLRC4hlTqQ7jbxofaqK6AJc96Jh2qkbBIb6613p7Y1/oA/caP -0FG7Yn2ksYyy/yARujVjBYZHYEMzkPZHogNPlk2dT8Hq6pyi/jQu3rfKG3akt62f -6ajUeD94/vI4CTYd0hYCyOwqaK/1jpTvLRN6HkJKHRUxrgwEV/xhc/MxVoYxgKDE -EW4wduOU8F8ExKyHcomYxZ3MVwia9Az8fXoFOvpHgDm2z4QTd28n6v+WZxcIbekN -1iNQMLAVdBM+5S//Ds3EC0pd8NgAM0lm66EYfFkuPSi5YXHLtaW6uOrc4nBvCGrc -h2c0798wct3zyT8j/zXhviEpIDCB5BmlIOklynMxdCm+4kLV87ImZsdo/Rmz5yCT -mehd4F6H50boJZwKKSTUzViGUkAksnsPmBIgJPaQbEfIDbsYIC7Z/fyL8inqh3SV -4EJQeIQEQWGw9CEjjy3LKCHyamz0GqbFFLQ3ZU+V/YDI+HLlJWvEYLF7bY5KinPO -WftwenMGE9nTdDckQQoRb5fc5+R+ob0V8rqHDz1oihYHAgMBAAGjYzBhMA4GA1Ud -DwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBSowcCbkahDFXxd -Bie0KlHYlwuBsTAfBgNVHSMEGDAWgBSowcCbkahDFXxdBie0KlHYlwuBsTANBgkq -hkiG9w0BAQ0FAAOCAgEAnFZvAX7RvUz1isbwJh/k4DgYzDLDKTudQSk0YcbX8ACh -66Ryj5QXvBMsdbRX7gp8CXrc1cqh0DQT+Hern+X+2B50ioUHj3/MeXrKls3N/U/7 -/SMNkPX0XtPGYX2eEeAC7gkE2Qfdpoq3DIMku4NQkv5gdRE+2J2winq14J2by5BS -S7CTKtQ+FjPlnsZlFT5kOwQ/2wyPX1wdaR+v8+khjPPvl/aatxm2hHSco1S1cE5j -2FddUyGbQJJD+tZ3VTNPZNX70Cxqjm0lpu+F6ALEUz65noe8zDUa3qHpimOHZR4R -Kttjd5cUvpoUmRGywO6wT/gUITJDT5+rosuoD6o7BlXGEilXCNQ314cnrUlZp5Gr -RHpejXDbl85IULFzk/bwg2D5zfHhMf1bfHEhYxQUqq/F3pN+aLHsIqKqkHWetUNy -6mSjhEv9DKgma3GX7lZjZuhCVPnHHd/Qj1vfyDBviP4NxDMcU6ij/UgQ8uQKTuEV -V/xuZDDCVRHc6qnNSlSsKWNEz0pAoNZoWRsz+e86i9sgktxChL8Bq4fA1SCC28a5 -g4VCXA9DO2pJNdWY9BW/+mGBDAkgGNLQFwzLSABQ6XaCjGTXOqAHVcweMcDvOrRl -++O/QmueD6i9a5jc2NvLi6Td11n0bt3+qsOR0C5CB8AMTVPNJLFMWx5R9N/pkvo= ------END CERTIFICATE----- - -# Issuer: CN=Certplus Root CA G2 O=Certplus -# Subject: CN=Certplus Root CA G2 O=Certplus -# Label: "Certplus Root CA G2" -# Serial: 1492087096131536844209563509228951875861589 -# MD5 Fingerprint: a7:ee:c4:78:2d:1b:ee:2d:b9:29:ce:d6:a7:96:32:31 -# SHA1 Fingerprint: 4f:65:8e:1f:e9:06:d8:28:02:e9:54:47:41:c9:54:25:5d:69:cc:1a -# SHA256 Fingerprint: 6c:c0:50:41:e6:44:5e:74:69:6c:4c:fb:c9:f8:0f:54:3b:7e:ab:bb:44:b4:ce:6f:78:7c:6a:99:71:c4:2f:17 ------BEGIN CERTIFICATE----- -MIICHDCCAaKgAwIBAgISESDZkc6uo+jF5//pAq/Pc7xVMAoGCCqGSM49BAMDMD4x -CzAJBgNVBAYTAkZSMREwDwYDVQQKDAhDZXJ0cGx1czEcMBoGA1UEAwwTQ2VydHBs -dXMgUm9vdCBDQSBHMjAeFw0xNDA1MjYwMDAwMDBaFw0zODAxMTUwMDAwMDBaMD4x -CzAJBgNVBAYTAkZSMREwDwYDVQQKDAhDZXJ0cGx1czEcMBoGA1UEAwwTQ2VydHBs -dXMgUm9vdCBDQSBHMjB2MBAGByqGSM49AgEGBSuBBAAiA2IABM0PW1aC3/BFGtat -93nwHcmsltaeTpwftEIRyoa/bfuFo8XlGVzX7qY/aWfYeOKmycTbLXku54uNAm8x -Ik0G42ByRZ0OQneezs/lf4WbGOT8zC5y0xaTTsqZY1yhBSpsBqNjMGEwDgYDVR0P -AQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFNqDYwJ5jtpMxjwj -FNiPwyCrKGBZMB8GA1UdIwQYMBaAFNqDYwJ5jtpMxjwjFNiPwyCrKGBZMAoGCCqG -SM49BAMDA2gAMGUCMHD+sAvZ94OX7PNVHdTcswYO/jOYnYs5kGuUIe22113WTNch -p+e/IQ8rzfcq3IUHnQIxAIYUFuXcsGXCwI4Un78kFmjlvPl5adytRSv3tjFzzAal -U5ORGpOucGpnutee5WEaXw== ------END CERTIFICATE----- - -# Issuer: CN=OpenTrust Root CA G1 O=OpenTrust -# Subject: CN=OpenTrust Root CA G1 O=OpenTrust -# Label: "OpenTrust Root CA G1" -# Serial: 1492036577811947013770400127034825178844775 -# MD5 Fingerprint: 76:00:cc:81:29:cd:55:5e:88:6a:7a:2e:f7:4d:39:da -# SHA1 Fingerprint: 79:91:e8:34:f7:e2:ee:dd:08:95:01:52:e9:55:2d:14:e9:58:d5:7e -# SHA256 Fingerprint: 56:c7:71:28:d9:8c:18:d9:1b:4c:fd:ff:bc:25:ee:91:03:d4:75:8e:a2:ab:ad:82:6a:90:f3:45:7d:46:0e:b4 ------BEGIN CERTIFICATE----- -MIIFbzCCA1egAwIBAgISESCzkFU5fX82bWTCp59rY45nMA0GCSqGSIb3DQEBCwUA -MEAxCzAJBgNVBAYTAkZSMRIwEAYDVQQKDAlPcGVuVHJ1c3QxHTAbBgNVBAMMFE9w -ZW5UcnVzdCBSb290IENBIEcxMB4XDTE0MDUyNjA4NDU1MFoXDTM4MDExNTAwMDAw -MFowQDELMAkGA1UEBhMCRlIxEjAQBgNVBAoMCU9wZW5UcnVzdDEdMBsGA1UEAwwU -T3BlblRydXN0IFJvb3QgQ0EgRzEwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK -AoICAQD4eUbalsUwXopxAy1wpLuwxQjczeY1wICkES3d5oeuXT2R0odsN7faYp6b -wiTXj/HbpqbfRm9RpnHLPhsxZ2L3EVs0J9V5ToybWL0iEA1cJwzdMOWo010hOHQX -/uMftk87ay3bfWAfjH1MBcLrARYVmBSO0ZB3Ij/swjm4eTrwSSTilZHcYTSSjFR0 -77F9jAHiOH3BX2pfJLKOYheteSCtqx234LSWSE9mQxAGFiQD4eCcjsZGT44ameGP -uY4zbGneWK2gDqdkVBFpRGZPTBKnjix9xNRbxQA0MMHZmf4yzgeEtE7NCv82TWLx -p2NX5Ntqp66/K7nJ5rInieV+mhxNaMbBGN4zK1FGSxyO9z0M+Yo0FMT7MzUj8czx -Kselu7Cizv5Ta01BG2Yospb6p64KTrk5M0ScdMGTHPjgniQlQ/GbI4Kq3ywgsNw2 -TgOzfALU5nsaqocTvz6hdLubDuHAk5/XpGbKuxs74zD0M1mKB3IDVedzagMxbm+W -G+Oin6+Sx+31QrclTDsTBM8clq8cIqPQqwWyTBIjUtz9GVsnnB47ev1CI9sjgBPw -vFEVVJSmdz7QdFG9URQIOTfLHzSpMJ1ShC5VkLG631UAC9hWLbFJSXKAqWLXwPYY -EQRVzXR7z2FwefR7LFxckvzluFqrTJOVoSfupb7PcSNCupt2LQIDAQABo2MwYTAO -BgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUl0YhVyE1 -2jZVx/PxN3DlCPaTKbYwHwYDVR0jBBgwFoAUl0YhVyE12jZVx/PxN3DlCPaTKbYw -DQYJKoZIhvcNAQELBQADggIBAB3dAmB84DWn5ph76kTOZ0BP8pNuZtQ5iSas000E -PLuHIT839HEl2ku6q5aCgZG27dmxpGWX4m9kWaSW7mDKHyP7Rbr/jyTwyqkxf3kf -gLMtMrpkZ2CvuVnN35pJ06iCsfmYlIrM4LvgBBuZYLFGZdwIorJGnkSI6pN+VxbS -FXJfLkur1J1juONI5f6ELlgKn0Md/rcYkoZDSw6cMoYsYPXpSOqV7XAp8dUv/TW0 -V8/bhUiZucJvbI/NeJWsZCj9VrDDb8O+WVLhX4SPgPL0DTatdrOjteFkdjpY3H1P -XlZs5VVZV6Xf8YpmMIzUUmI4d7S+KNfKNsSbBfD4Fdvb8e80nR14SohWZ25g/4/I -i+GOvUKpMwpZQhISKvqxnUOOBZuZ2mKtVzazHbYNeS2WuOvyDEsMpZTGMKcmGS3t -TAZQMPH9WD25SxdfGbRqhFS0OE85og2WaMMolP3tLR9Ka0OWLpABEPs4poEL0L91 -09S5zvE/bw4cHjdx5RiHdRk/ULlepEU0rbDK5uUTdg8xFKmOLZTW1YVNcxVPS/Ky -Pu1svf0OnWZzsD2097+o4BGkxK51CUpjAEggpsadCwmKtODmzj7HPiY46SvepghJ -AwSQiumPv+i2tCqjI40cHLI5kqiPAlxAOXXUc0ECd97N4EOH1uS6SsNsEn/+KuYj -1oxx ------END CERTIFICATE----- - -# Issuer: CN=OpenTrust Root CA G2 O=OpenTrust -# Subject: CN=OpenTrust Root CA G2 O=OpenTrust -# Label: "OpenTrust Root CA G2" -# Serial: 1492012448042702096986875987676935573415441 -# MD5 Fingerprint: 57:24:b6:59:24:6b:ae:c8:fe:1c:0c:20:f2:c0:4e:eb -# SHA1 Fingerprint: 79:5f:88:60:c5:ab:7c:3d:92:e6:cb:f4:8d:e1:45:cd:11:ef:60:0b -# SHA256 Fingerprint: 27:99:58:29:fe:6a:75:15:c1:bf:e8:48:f9:c4:76:1d:b1:6c:22:59:29:25:7b:f4:0d:08:94:f2:9e:a8:ba:f2 ------BEGIN CERTIFICATE----- -MIIFbzCCA1egAwIBAgISESChaRu/vbm9UpaPI+hIvyYRMA0GCSqGSIb3DQEBDQUA -MEAxCzAJBgNVBAYTAkZSMRIwEAYDVQQKDAlPcGVuVHJ1c3QxHTAbBgNVBAMMFE9w -ZW5UcnVzdCBSb290IENBIEcyMB4XDTE0MDUyNjAwMDAwMFoXDTM4MDExNTAwMDAw -MFowQDELMAkGA1UEBhMCRlIxEjAQBgNVBAoMCU9wZW5UcnVzdDEdMBsGA1UEAwwU -T3BlblRydXN0IFJvb3QgQ0EgRzIwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK -AoICAQDMtlelM5QQgTJT32F+D3Y5z1zCU3UdSXqWON2ic2rxb95eolq5cSG+Ntmh -/LzubKh8NBpxGuga2F8ORAbtp+Dz0mEL4DKiltE48MLaARf85KxP6O6JHnSrT78e -CbY2albz4e6WiWYkBuTNQjpK3eCasMSCRbP+yatcfD7J6xcvDH1urqWPyKwlCm/6 -1UWY0jUJ9gNDlP7ZvyCVeYCYitmJNbtRG6Q3ffyZO6v/v6wNj0OxmXsWEH4db0fE -FY8ElggGQgT4hNYdvJGmQr5J1WqIP7wtUdGejeBSzFfdNTVY27SPJIjki9/ca1TS -gSuyzpJLHB9G+h3Ykst2Z7UJmQnlrBcUVXDGPKBWCgOz3GIZ38i1MH/1PCZ1Eb3X -G7OHngevZXHloM8apwkQHZOJZlvoPGIytbU6bumFAYueQ4xncyhZW+vj3CzMpSZy -YhK05pyDRPZRpOLAeiRXyg6lPzq1O4vldu5w5pLeFlwoW5cZJ5L+epJUzpM5ChaH -vGOz9bGTXOBut9Dq+WIyiET7vycotjCVXRIouZW+j1MY5aIYFuJWpLIsEPUdN6b4 -t/bQWVyJ98LVtZR00dX+G7bw5tYee9I8y6jj9RjzIR9u701oBnstXW5DiabA+aC/ -gh7PU3+06yzbXfZqfUAkBXKJOAGTy3HCOV0GEfZvePg3DTmEJwIDAQABo2MwYTAO -BgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUajn6QiL3 -5okATV59M4PLuG53hq8wHwYDVR0jBBgwFoAUajn6QiL35okATV59M4PLuG53hq8w -DQYJKoZIhvcNAQENBQADggIBAJjLq0A85TMCl38th6aP1F5Kr7ge57tx+4BkJamz -Gj5oXScmp7oq4fBXgwpkTx4idBvpkF/wrM//T2h6OKQQbA2xx6R3gBi2oihEdqc0 -nXGEL8pZ0keImUEiyTCYYW49qKgFbdEfwFFEVn8nNQLdXpgKQuswv42hm1GqO+qT -RmTFAHneIWv2V6CG1wZy7HBGS4tz3aAhdT7cHcCP009zHIXZ/n9iyJVvttN7jLpT -wm+bREx50B1ws9efAvSyB7DH5fitIw6mVskpEndI2S9G/Tvw/HRwkqWOOAgfZDC2 -t0v7NqwQjqBSM2OdAzVWxWm9xiNaJ5T2pBL4LTM8oValX9YZ6e18CL13zSdkzJTa -TkZQh+D5wVOAHrut+0dSixv9ovneDiK3PTNZbNTe9ZUGMg1RGUFcPk8G97krgCf2 -o6p6fAbhQ8MTOWIaNr3gKC6UAuQpLmBVrkA9sHSSXvAgZJY/X0VdiLWK2gKgW0VU -3jg9CcCoSmVGFvyqv1ROTVu+OEO3KMqLM6oaJbolXCkvW0pujOotnCr2BXbgd5eA -iN1nE28daCSLT7d0geX0YJ96Vdc+N9oWaz53rK4YcJUIeSkDiv7BO7M/Gg+kO14f -WKGVyasvc0rQLW6aWQ9VGHgtPFGml4vmu7JwqkwR3v98KzfUetF3NI/n+UL3PIEM -S1IK ------END CERTIFICATE----- - -# Issuer: CN=OpenTrust Root CA G3 O=OpenTrust -# Subject: CN=OpenTrust Root CA G3 O=OpenTrust -# Label: "OpenTrust Root CA G3" -# Serial: 1492104908271485653071219941864171170455615 -# MD5 Fingerprint: 21:37:b4:17:16:92:7b:67:46:70:a9:96:d7:a8:13:24 -# SHA1 Fingerprint: 6e:26:64:f3:56:bf:34:55:bf:d1:93:3f:7c:01:de:d8:13:da:8a:a6 -# SHA256 Fingerprint: b7:c3:62:31:70:6e:81:07:8c:36:7c:b8:96:19:8f:1e:32:08:dd:92:69:49:dd:8f:57:09:a4:10:f7:5b:62:92 ------BEGIN CERTIFICATE----- -MIICITCCAaagAwIBAgISESDm+Ez8JLC+BUCs2oMbNGA/MAoGCCqGSM49BAMDMEAx -CzAJBgNVBAYTAkZSMRIwEAYDVQQKDAlPcGVuVHJ1c3QxHTAbBgNVBAMMFE9wZW5U -cnVzdCBSb290IENBIEczMB4XDTE0MDUyNjAwMDAwMFoXDTM4MDExNTAwMDAwMFow -QDELMAkGA1UEBhMCRlIxEjAQBgNVBAoMCU9wZW5UcnVzdDEdMBsGA1UEAwwUT3Bl -blRydXN0IFJvb3QgQ0EgRzMwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAARK7liuTcpm -3gY6oxH84Bjwbhy6LTAMidnW7ptzg6kjFYwvWYpa3RTqnVkrQ7cG7DK2uu5Bta1d -oYXM6h0UZqNnfkbilPPntlahFVmhTzeXuSIevRHr9LIfXsMUmuXZl5mjYzBhMA4G -A1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBRHd8MUi2I5 -DMlv4VBN0BBY3JWIbTAfBgNVHSMEGDAWgBRHd8MUi2I5DMlv4VBN0BBY3JWIbTAK -BggqhkjOPQQDAwNpADBmAjEAj6jcnboMBBf6Fek9LykBl7+BFjNAk2z8+e2AcG+q -j9uEwov1NcoG3GRvaBbhj5G5AjEA2Euly8LQCGzpGPta3U1fJAuwACEl74+nBCZx -4nxp5V2a+EEfOzmTk51V6s2N8fvB ------END CERTIFICATE----- - # Issuer: CN=ISRG Root X1 O=Internet Security Research Group # Subject: CN=ISRG Root X1 O=Internet Security Research Group # Label: "ISRG Root X1" @@ -4398,3 +4235,66 @@ MA8GA1UdEwEB/wQFMAMBAf8wHwYDVR0jBBgwFoAUW8pe5d7SgarNqC1kUbbZcpuX ytRrJPOwPYdGWBrssd9v+1a6cGvHOMzosYxPD/fxZ3YOg9AeUY8CMD32IygmTMZg h5Mmm7I1HrrW9zzRHM76JTymGoEVW/MSD2zuZYrJh6j5B+BimoxcSg== -----END CERTIFICATE----- + +# Issuer: CN=GlobalSign O=GlobalSign OU=GlobalSign Root CA - R6 +# Subject: CN=GlobalSign O=GlobalSign OU=GlobalSign Root CA - R6 +# Label: "GlobalSign Root CA - R6" +# Serial: 1417766617973444989252670301619537 +# MD5 Fingerprint: 4f:dd:07:e4:d4:22:64:39:1e:0c:37:42:ea:d1:c6:ae +# SHA1 Fingerprint: 80:94:64:0e:b5:a7:a1:ca:11:9c:1f:dd:d5:9f:81:02:63:a7:fb:d1 +# SHA256 Fingerprint: 2c:ab:ea:fe:37:d0:6c:a2:2a:ba:73:91:c0:03:3d:25:98:29:52:c4:53:64:73:49:76:3a:3a:b5:ad:6c:cf:69 +-----BEGIN CERTIFICATE----- +MIIFgzCCA2ugAwIBAgIORea7A4Mzw4VlSOb/RVEwDQYJKoZIhvcNAQEMBQAwTDEg +MB4GA1UECxMXR2xvYmFsU2lnbiBSb290IENBIC0gUjYxEzARBgNVBAoTCkdsb2Jh +bFNpZ24xEzARBgNVBAMTCkdsb2JhbFNpZ24wHhcNMTQxMjEwMDAwMDAwWhcNMzQx +MjEwMDAwMDAwWjBMMSAwHgYDVQQLExdHbG9iYWxTaWduIFJvb3QgQ0EgLSBSNjET +MBEGA1UEChMKR2xvYmFsU2lnbjETMBEGA1UEAxMKR2xvYmFsU2lnbjCCAiIwDQYJ +KoZIhvcNAQEBBQADggIPADCCAgoCggIBAJUH6HPKZvnsFMp7PPcNCPG0RQssgrRI +xutbPK6DuEGSMxSkb3/pKszGsIhrxbaJ0cay/xTOURQh7ErdG1rG1ofuTToVBu1k +ZguSgMpE3nOUTvOniX9PeGMIyBJQbUJmL025eShNUhqKGoC3GYEOfsSKvGRMIRxD +aNc9PIrFsmbVkJq3MQbFvuJtMgamHvm566qjuL++gmNQ0PAYid/kD3n16qIfKtJw +LnvnvJO7bVPiSHyMEAc4/2ayd2F+4OqMPKq0pPbzlUoSB239jLKJz9CgYXfIWHSw +1CM69106yqLbnQneXUQtkPGBzVeS+n68UARjNN9rkxi+azayOeSsJDa38O+2HBNX +k7besvjihbdzorg1qkXy4J02oW9UivFyVm4uiMVRQkQVlO6jxTiWm05OWgtH8wY2 +SXcwvHE35absIQh1/OZhFj931dmRl4QKbNQCTXTAFO39OfuD8l4UoQSwC+n+7o/h +bguyCLNhZglqsQY6ZZZZwPA1/cnaKI0aEYdwgQqomnUdnjqGBQCe24DWJfncBZ4n +WUx2OVvq+aWh2IMP0f/fMBH5hc8zSPXKbWQULHpYT9NLCEnFlWQaYw55PfWzjMpY +rZxCRXluDocZXFSxZba/jJvcE+kNb7gu3GduyYsRtYQUigAZcIN5kZeR1Bonvzce +MgfYFGM8KEyvAgMBAAGjYzBhMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTAD +AQH/MB0GA1UdDgQWBBSubAWjkxPioufi1xzWx/B/yGdToDAfBgNVHSMEGDAWgBSu +bAWjkxPioufi1xzWx/B/yGdToDANBgkqhkiG9w0BAQwFAAOCAgEAgyXt6NH9lVLN +nsAEoJFp5lzQhN7craJP6Ed41mWYqVuoPId8AorRbrcWc+ZfwFSY1XS+wc3iEZGt +Ixg93eFyRJa0lV7Ae46ZeBZDE1ZXs6KzO7V33EByrKPrmzU+sQghoefEQzd5Mr61 +55wsTLxDKZmOMNOsIeDjHfrYBzN2VAAiKrlNIC5waNrlU/yDXNOd8v9EDERm8tLj +vUYAGm0CuiVdjaExUd1URhxN25mW7xocBFymFe944Hn+Xds+qkxV/ZoVqW/hpvvf +cDDpw+5CRu3CkwWJ+n1jez/QcYF8AOiYrg54NMMl+68KnyBr3TsTjxKM4kEaSHpz +oHdpx7Zcf4LIHv5YGygrqGytXm3ABdJ7t+uA/iU3/gKbaKxCXcPu9czc8FB10jZp +nOZ7BN9uBmm23goJSFmH63sUYHpkqmlD75HHTOwY3WzvUy2MmeFe8nI+z1TIvWfs +pA9MRf/TuTAjB0yPEL+GltmZWrSZVxykzLsViVO6LAUP5MSeGbEYNNVMnbrt9x+v +JJUEeKgDu+6B5dpffItKoZB0JaezPkvILFa9x8jvOOJckvB595yEunQtYQEgfn7R +8k8HWV+LLUNS60YMlOH1Zkd5d9VUWx+tJDfLRVpOoERIyNiwmcUVhAn21klJwGW4 +5hpxbqCo8YLoRT5s1gLXCmeDBVrJpBA= +-----END CERTIFICATE----- + +# Issuer: CN=OISTE WISeKey Global Root GC CA O=WISeKey OU=OISTE Foundation Endorsed +# Subject: CN=OISTE WISeKey Global Root GC CA O=WISeKey OU=OISTE Foundation Endorsed +# Label: "OISTE WISeKey Global Root GC CA" +# Serial: 44084345621038548146064804565436152554 +# MD5 Fingerprint: a9:d6:b9:2d:2f:93:64:f8:a5:69:ca:91:e9:68:07:23 +# SHA1 Fingerprint: e0:11:84:5e:34:de:be:88:81:b9:9c:f6:16:26:d1:96:1f:c3:b9:31 +# SHA256 Fingerprint: 85:60:f9:1c:36:24:da:ba:95:70:b5:fe:a0:db:e3:6f:f1:1a:83:23:be:94:86:85:4f:b3:f3:4a:55:71:19:8d +-----BEGIN CERTIFICATE----- +MIICaTCCAe+gAwIBAgIQISpWDK7aDKtARb8roi066jAKBggqhkjOPQQDAzBtMQsw +CQYDVQQGEwJDSDEQMA4GA1UEChMHV0lTZUtleTEiMCAGA1UECxMZT0lTVEUgRm91 +bmRhdGlvbiBFbmRvcnNlZDEoMCYGA1UEAxMfT0lTVEUgV0lTZUtleSBHbG9iYWwg +Um9vdCBHQyBDQTAeFw0xNzA1MDkwOTQ4MzRaFw00MjA1MDkwOTU4MzNaMG0xCzAJ +BgNVBAYTAkNIMRAwDgYDVQQKEwdXSVNlS2V5MSIwIAYDVQQLExlPSVNURSBGb3Vu +ZGF0aW9uIEVuZG9yc2VkMSgwJgYDVQQDEx9PSVNURSBXSVNlS2V5IEdsb2JhbCBS +b290IEdDIENBMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAETOlQwMYPchi82PG6s4ni +eUqjFqdrVCTbUf/q9Akkwwsin8tqJ4KBDdLArzHkdIJuyiXZjHWd8dvQmqJLIX4W +p2OQ0jnUsYd4XxiWD1AbNTcPasbc2RNNpI6QN+a9WzGRo1QwUjAOBgNVHQ8BAf8E +BAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUSIcUrOPDnpBgOtfKie7T +rYy0UGYwEAYJKwYBBAGCNxUBBAMCAQAwCgYIKoZIzj0EAwMDaAAwZQIwJsdpW9zV +57LnyAyMjMPdeYwbY9XJUpROTYJKcx6ygISpJcBMWm1JKWB4E+J+SOtkAjEA2zQg +Mgj/mkkCtojeFK9dbJlxjRo/i9fgojaGHAeCOnZT/cKi7e97sIBPWA9LUzm9 +-----END CERTIFICATE----- diff --git a/pipenv/patched/notpip/_vendor/packaging/__about__.py b/pipenv/patched/notpip/_vendor/packaging/__about__.py index 4255c5b5..21fc6ce3 100644 --- a/pipenv/patched/notpip/_vendor/packaging/__about__.py +++ b/pipenv/patched/notpip/_vendor/packaging/__about__.py @@ -12,10 +12,10 @@ __title__ = "packaging" __summary__ = "Core utilities for Python packages" __uri__ = "https://github.com/pypa/packaging" -__version__ = "17.1" +__version__ = "18.0" __author__ = "Donald Stufft and individual contributors" __email__ = "donald@stufft.io" __license__ = "BSD or Apache License, Version 2.0" -__copyright__ = "Copyright 2014-2016 %s" % __author__ +__copyright__ = "Copyright 2014-2018 %s" % __author__ diff --git a/pipenv/patched/notpip/_vendor/packaging/requirements.py b/pipenv/patched/notpip/_vendor/packaging/requirements.py index 0da25914..5ec5d74a 100644 --- a/pipenv/patched/notpip/_vendor/packaging/requirements.py +++ b/pipenv/patched/notpip/_vendor/packaging/requirements.py @@ -92,16 +92,16 @@ class Requirement(object): try: req = REQUIREMENT.parseString(requirement_string) except ParseException as e: - raise InvalidRequirement( - "Invalid requirement, parse error at \"{0!r}\"".format( - requirement_string[e.loc:e.loc + 8])) + raise InvalidRequirement("Parse error at \"{0!r}\": {1}".format( + requirement_string[e.loc:e.loc + 8], e.msg + )) self.name = req.name if req.url: parsed_url = urlparse.urlparse(req.url) if not (parsed_url.scheme and parsed_url.netloc) or ( not parsed_url.scheme and not parsed_url.netloc): - raise InvalidRequirement("Invalid URL given") + raise InvalidRequirement("Invalid URL: {0}".format(req.url)) self.url = req.url else: self.url = None diff --git a/pipenv/patched/notpip/_vendor/packaging/specifiers.py b/pipenv/patched/notpip/_vendor/packaging/specifiers.py index 9b6353f0..4c798999 100644 --- a/pipenv/patched/notpip/_vendor/packaging/specifiers.py +++ b/pipenv/patched/notpip/_vendor/packaging/specifiers.py @@ -503,7 +503,7 @@ class Specifier(_IndividualSpecifier): return False # Ensure that we do not allow a local version of the version mentioned - # in the specifier, which is techincally greater than, to match. + # in the specifier, which is technically greater than, to match. if prospective.local is not None: if Version(prospective.base_version) == Version(spec.base_version): return False diff --git a/pipenv/patched/prettytoml/LICENSE b/pipenv/patched/notpip/_vendor/pep517/LICENSE similarity index 88% rename from pipenv/patched/prettytoml/LICENSE rename to pipenv/patched/notpip/_vendor/pep517/LICENSE index 116fa4e5..b0ae9dbc 100644 --- a/pipenv/patched/prettytoml/LICENSE +++ b/pipenv/patched/notpip/_vendor/pep517/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2015 Jumpscale +Copyright (c) 2017 Thomas Kluyver Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -9,14 +9,13 @@ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/pipenv/patched/notpip/_vendor/pep517/__init__.py b/pipenv/patched/notpip/_vendor/pep517/__init__.py new file mode 100644 index 00000000..8beedea4 --- /dev/null +++ b/pipenv/patched/notpip/_vendor/pep517/__init__.py @@ -0,0 +1,4 @@ +"""Wrappers to build Python packages using PEP 517 hooks +""" + +__version__ = '0.2' diff --git a/pipenv/patched/notpip/_vendor/pep517/_in_process.py b/pipenv/patched/notpip/_vendor/pep517/_in_process.py new file mode 100644 index 00000000..baa14d38 --- /dev/null +++ b/pipenv/patched/notpip/_vendor/pep517/_in_process.py @@ -0,0 +1,182 @@ +"""This is invoked in a subprocess to call the build backend hooks. + +It expects: +- Command line args: hook_name, control_dir +- Environment variable: PEP517_BUILD_BACKEND=entry.point:spec +- control_dir/input.json: + - {"kwargs": {...}} + +Results: +- control_dir/output.json + - {"return_val": ...} +""" +from glob import glob +from importlib import import_module +import os +from os.path import join as pjoin +import re +import shutil +import sys + +# This is run as a script, not a module, so it can't do a relative import +import compat + +def _build_backend(): + """Find and load the build backend""" + ep = os.environ['PEP517_BUILD_BACKEND'] + mod_path, _, obj_path = ep.partition(':') + obj = import_module(mod_path) + if obj_path: + for path_part in obj_path.split('.'): + obj = getattr(obj, path_part) + return obj + +def get_requires_for_build_wheel(config_settings): + """Invoke the optional get_requires_for_build_wheel hook + + Returns [] if the hook is not defined. + """ + backend = _build_backend() + try: + hook = backend.get_requires_for_build_wheel + except AttributeError: + return [] + else: + return hook(config_settings) + +def prepare_metadata_for_build_wheel(metadata_directory, config_settings): + """Invoke optional prepare_metadata_for_build_wheel + + Implements a fallback by building a wheel if the hook isn't defined. + """ + backend = _build_backend() + try: + hook = backend.prepare_metadata_for_build_wheel + except AttributeError: + return _get_wheel_metadata_from_wheel(backend, metadata_directory, + config_settings) + else: + return hook(metadata_directory, config_settings) + +WHEEL_BUILT_MARKER = 'PEP517_ALREADY_BUILT_WHEEL' + +def _dist_info_files(whl_zip): + """Identify the .dist-info folder inside a wheel ZipFile.""" + res = [] + for path in whl_zip.namelist(): + m = re.match(r'[^/\\]+-[^/\\]+\.dist-info/', path) + if m: + res.append(path) + if res: + return res + raise Exception("No .dist-info folder found in wheel") + +def _get_wheel_metadata_from_wheel(backend, metadata_directory, config_settings): + """Build a wheel and extract the metadata from it. + + Fallback for when the build backend does not define the 'get_wheel_metadata' + hook. + """ + from zipfile import ZipFile + whl_basename = backend.build_wheel(metadata_directory, config_settings) + with open(os.path.join(metadata_directory, WHEEL_BUILT_MARKER), 'wb'): + pass # Touch marker file + + whl_file = os.path.join(metadata_directory, whl_basename) + with ZipFile(whl_file) as zipf: + dist_info = _dist_info_files(zipf) + zipf.extractall(path=metadata_directory, members=dist_info) + return dist_info[0].split('/')[0] + +def _find_already_built_wheel(metadata_directory): + """Check for a wheel already built during the get_wheel_metadata hook. + """ + if not metadata_directory: + return None + metadata_parent = os.path.dirname(metadata_directory) + if not os.path.isfile(pjoin(metadata_parent, WHEEL_BUILT_MARKER)): + return None + + whl_files = glob(os.path.join(metadata_parent, '*.whl')) + if not whl_files: + print('Found wheel built marker, but no .whl files') + return None + if len(whl_files) > 1: + print('Found multiple .whl files; unspecified behaviour. ' + 'Will call build_wheel.') + return None + + # Exactly one .whl file + return whl_files[0] + +def build_wheel(wheel_directory, config_settings, metadata_directory=None): + """Invoke the mandatory build_wheel hook. + + If a wheel was already built in the prepare_metadata_for_build_wheel fallback, this + will copy it rather than rebuilding the wheel. + """ + prebuilt_whl = _find_already_built_wheel(metadata_directory) + if prebuilt_whl: + shutil.copy2(prebuilt_whl, wheel_directory) + return os.path.basename(prebuilt_whl) + + return _build_backend().build_wheel(wheel_directory, config_settings, + metadata_directory) + + +def get_requires_for_build_sdist(config_settings): + """Invoke the optional get_requires_for_build_wheel hook + + Returns [] if the hook is not defined. + """ + backend = _build_backend() + try: + hook = backend.get_requires_for_build_sdist + except AttributeError: + return [] + else: + return hook(config_settings) + +class _DummyException(Exception): + """Nothing should ever raise this exception""" + +class GotUnsupportedOperation(Exception): + """For internal use when backend raises UnsupportedOperation""" + +def build_sdist(sdist_directory, config_settings): + """Invoke the mandatory build_sdist hook.""" + backend = _build_backend() + try: + return backend.build_sdist(sdist_directory, config_settings) + except getattr(backend, 'UnsupportedOperation', _DummyException): + raise GotUnsupportedOperation + +HOOK_NAMES = { + 'get_requires_for_build_wheel', + 'prepare_metadata_for_build_wheel', + 'build_wheel', + 'get_requires_for_build_sdist', + 'build_sdist', +} + +def main(): + if len(sys.argv) < 3: + sys.exit("Needs args: hook_name, control_dir") + hook_name = sys.argv[1] + control_dir = sys.argv[2] + if hook_name not in HOOK_NAMES: + sys.exit("Unknown hook: %s" % hook_name) + hook = globals()[hook_name] + + hook_input = compat.read_json(pjoin(control_dir, 'input.json')) + + json_out = {'unsupported': False, 'return_val': None} + try: + json_out['return_val'] = hook(**hook_input['kwargs']) + except GotUnsupportedOperation: + json_out['unsupported'] = True + + compat.write_json(json_out, pjoin(control_dir, 'output.json'), indent=2) + +if __name__ == '__main__': + main() diff --git a/pipenv/patched/notpip/_vendor/pep517/check.py b/pipenv/patched/notpip/_vendor/pep517/check.py new file mode 100644 index 00000000..3dffd2e0 --- /dev/null +++ b/pipenv/patched/notpip/_vendor/pep517/check.py @@ -0,0 +1,194 @@ +"""Check a project and backend by attempting to build using PEP 517 hooks. +""" +import argparse +import logging +import os +from os.path import isfile, join as pjoin +from pipenv.patched.notpip._vendor.pytoml import TomlError, load as toml_load +import shutil +from subprocess import CalledProcessError +import sys +import tarfile +from tempfile import mkdtemp +import zipfile + +from .colorlog import enable_colourful_output +from .envbuild import BuildEnvironment +from .wrappers import Pep517HookCaller + +log = logging.getLogger(__name__) + +def check_build_sdist(hooks): + with BuildEnvironment() as env: + try: + env.pip_install(hooks.build_sys_requires) + log.info('Installed static build dependencies') + except CalledProcessError: + log.error('Failed to install static build dependencies') + return False + + try: + reqs = hooks.get_requires_for_build_sdist({}) + log.info('Got build requires: %s', reqs) + except: + log.error('Failure in get_requires_for_build_sdist', exc_info=True) + return False + + try: + env.pip_install(reqs) + log.info('Installed dynamic build dependencies') + except CalledProcessError: + log.error('Failed to install dynamic build dependencies') + return False + + td = mkdtemp() + log.info('Trying to build sdist in %s', td) + try: + try: + filename = hooks.build_sdist(td, {}) + log.info('build_sdist returned %r', filename) + except: + log.info('Failure in build_sdist', exc_info=True) + return False + + if not filename.endswith('.tar.gz'): + log.error("Filename %s doesn't have .tar.gz extension", filename) + return False + + path = pjoin(td, filename) + if isfile(path): + log.info("Output file %s exists", path) + else: + log.error("Output file %s does not exist", path) + return False + + if tarfile.is_tarfile(path): + log.info("Output file is a tar file") + else: + log.error("Output file is not a tar file") + return False + + finally: + shutil.rmtree(td) + + return True + +def check_build_wheel(hooks): + with BuildEnvironment() as env: + try: + env.pip_install(hooks.build_sys_requires) + log.info('Installed static build dependencies') + except CalledProcessError: + log.error('Failed to install static build dependencies') + return False + + try: + reqs = hooks.get_requires_for_build_wheel({}) + log.info('Got build requires: %s', reqs) + except: + log.error('Failure in get_requires_for_build_sdist', exc_info=True) + return False + + try: + env.pip_install(reqs) + log.info('Installed dynamic build dependencies') + except CalledProcessError: + log.error('Failed to install dynamic build dependencies') + return False + + td = mkdtemp() + log.info('Trying to build wheel in %s', td) + try: + try: + filename = hooks.build_wheel(td, {}) + log.info('build_wheel returned %r', filename) + except: + log.info('Failure in build_wheel', exc_info=True) + return False + + if not filename.endswith('.whl'): + log.error("Filename %s doesn't have .whl extension", filename) + return False + + path = pjoin(td, filename) + if isfile(path): + log.info("Output file %s exists", path) + else: + log.error("Output file %s does not exist", path) + return False + + if zipfile.is_zipfile(path): + log.info("Output file is a zip file") + else: + log.error("Output file is not a zip file") + return False + + finally: + shutil.rmtree(td) + + return True + + +def check(source_dir): + pyproject = pjoin(source_dir, 'pyproject.toml') + if isfile(pyproject): + log.info('Found pyproject.toml') + else: + log.error('Missing pyproject.toml') + return False + + try: + with open(pyproject) as f: + pyproject_data = toml_load(f) + # Ensure the mandatory data can be loaded + buildsys = pyproject_data['build-system'] + requires = buildsys['requires'] + backend = buildsys['build-backend'] + log.info('Loaded pyproject.toml') + except (TomlError, KeyError): + log.error("Invalid pyproject.toml", exc_info=True) + return False + + hooks = Pep517HookCaller(source_dir, backend) + + sdist_ok = check_build_sdist(hooks) + wheel_ok = check_build_wheel(hooks) + + if not sdist_ok: + log.warning('Sdist checks failed; scroll up to see') + if not wheel_ok: + log.warning('Wheel checks failed') + + return sdist_ok + + +def main(argv=None): + ap = argparse.ArgumentParser() + ap.add_argument('source_dir', + help="A directory containing pyproject.toml") + args = ap.parse_args(argv) + + enable_colourful_output() + + ok = check(args.source_dir) + + if ok: + print(ansi('Checks passed', 'green')) + else: + print(ansi('Checks failed', 'red')) + sys.exit(1) + +ansi_codes = { + 'reset': '\x1b[0m', + 'bold': '\x1b[1m', + 'red': '\x1b[31m', + 'green': '\x1b[32m', +} +def ansi(s, attr): + if os.name != 'nt' and sys.stdout.isatty(): + return ansi_codes[attr] + str(s) + ansi_codes['reset'] + else: + return str(s) + +if __name__ == '__main__': + main() diff --git a/pipenv/patched/notpip/_vendor/pep517/colorlog.py b/pipenv/patched/notpip/_vendor/pep517/colorlog.py new file mode 100644 index 00000000..26cf7480 --- /dev/null +++ b/pipenv/patched/notpip/_vendor/pep517/colorlog.py @@ -0,0 +1,110 @@ +"""Nicer log formatting with colours. + +Code copied from Tornado, Apache licensed. +""" +# Copyright 2012 Facebook +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import sys + +try: + import curses +except ImportError: + curses = None + +def _stderr_supports_color(): + color = False + if curses and hasattr(sys.stderr, 'isatty') and sys.stderr.isatty(): + try: + curses.setupterm() + if curses.tigetnum("colors") > 0: + color = True + except Exception: + pass + return color + +class LogFormatter(logging.Formatter): + """Log formatter with colour support + """ + DEFAULT_COLORS = { + logging.INFO: 2, # Green + logging.WARNING: 3, # Yellow + logging.ERROR: 1, # Red + logging.CRITICAL: 1, + } + + def __init__(self, color=True, datefmt=None): + r""" + :arg bool color: Enables color support. + :arg string fmt: Log message format. + It will be applied to the attributes dict of log records. The + text between ``%(color)s`` and ``%(end_color)s`` will be colored + depending on the level if color support is on. + :arg dict colors: color mappings from logging level to terminal color + code + :arg string datefmt: Datetime format. + Used for formatting ``(asctime)`` placeholder in ``prefix_fmt``. + .. versionchanged:: 3.2 + Added ``fmt`` and ``datefmt`` arguments. + """ + logging.Formatter.__init__(self, datefmt=datefmt) + self._colors = {} + if color and _stderr_supports_color(): + # The curses module has some str/bytes confusion in + # python3. Until version 3.2.3, most methods return + # bytes, but only accept strings. In addition, we want to + # output these strings with the logging module, which + # works with unicode strings. The explicit calls to + # unicode() below are harmless in python2 but will do the + # right conversion in python 3. + fg_color = (curses.tigetstr("setaf") or + curses.tigetstr("setf") or "") + if (3, 0) < sys.version_info < (3, 2, 3): + fg_color = str(fg_color, "ascii") + + for levelno, code in self.DEFAULT_COLORS.items(): + self._colors[levelno] = str(curses.tparm(fg_color, code), "ascii") + self._normal = str(curses.tigetstr("sgr0"), "ascii") + + scr = curses.initscr() + self.termwidth = scr.getmaxyx()[1] + curses.endwin() + else: + self._normal = '' + # Default width is usually 80, but too wide is worse than too narrow + self.termwidth = 70 + + def formatMessage(self, record): + l = len(record.message) + right_text = '{initial}-{name}'.format(initial=record.levelname[0], + name=record.name) + if l + len(right_text) < self.termwidth: + space = ' ' * (self.termwidth - (l + len(right_text))) + else: + space = ' ' + + if record.levelno in self._colors: + start_color = self._colors[record.levelno] + end_color = self._normal + else: + start_color = end_color = '' + + return record.message + space + start_color + right_text + end_color + +def enable_colourful_output(level=logging.INFO): + handler = logging.StreamHandler() + handler.setFormatter(LogFormatter()) + logging.root.addHandler(handler) + logging.root.setLevel(level) diff --git a/pipenv/patched/notpip/_vendor/pep517/compat.py b/pipenv/patched/notpip/_vendor/pep517/compat.py new file mode 100644 index 00000000..01c66fc7 --- /dev/null +++ b/pipenv/patched/notpip/_vendor/pep517/compat.py @@ -0,0 +1,23 @@ +"""Handle reading and writing JSON in UTF-8, on Python 3 and 2.""" +import json +import sys + +if sys.version_info[0] >= 3: + # Python 3 + def write_json(obj, path, **kwargs): + with open(path, 'w', encoding='utf-8') as f: + json.dump(obj, f, **kwargs) + + def read_json(path): + with open(path, 'r', encoding='utf-8') as f: + return json.load(f) + +else: + # Python 2 + def write_json(obj, path, **kwargs): + with open(path, 'wb') as f: + json.dump(obj, f, encoding='utf-8', **kwargs) + + def read_json(path): + with open(path, 'rb') as f: + return json.load(f) diff --git a/pipenv/patched/notpip/_vendor/pep517/envbuild.py b/pipenv/patched/notpip/_vendor/pep517/envbuild.py new file mode 100644 index 00000000..c54d3585 --- /dev/null +++ b/pipenv/patched/notpip/_vendor/pep517/envbuild.py @@ -0,0 +1,150 @@ +"""Build wheels/sdists by installing build deps to a temporary environment. +""" + +import os +import logging +from pipenv.patched.notpip._vendor import pytoml +import shutil +from subprocess import check_call +import sys +from sysconfig import get_paths +from tempfile import mkdtemp + +from .wrappers import Pep517HookCaller + +log = logging.getLogger(__name__) + +def _load_pyproject(source_dir): + with open(os.path.join(source_dir, 'pyproject.toml')) as f: + pyproject_data = pytoml.load(f) + buildsys = pyproject_data['build-system'] + return buildsys['requires'], buildsys['build-backend'] + + +class BuildEnvironment(object): + """Context manager to install build deps in a simple temporary environment + + Based on code I wrote for pip, which is MIT licensed. + """ + # Copyright (c) 2008-2016 The pip developers (see AUTHORS.txt file) + # + # Permission is hereby granted, free of charge, to any person obtaining + # a copy of this software and associated documentation files (the + # "Software"), to deal in the Software without restriction, including + # without limitation the rights to use, copy, modify, merge, publish, + # distribute, sublicense, and/or sell copies of the Software, and to + # permit persons to whom the Software is furnished to do so, subject to + # the following conditions: + # + # The above copyright notice and this permission notice shall be + # included in all copies or substantial portions of the Software. + # + # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + path = None + + def __init__(self, cleanup=True): + self._cleanup = cleanup + + def __enter__(self): + self.path = mkdtemp(prefix='pep517-build-env-') + log.info('Temporary build environment: %s', self.path) + + self.save_path = os.environ.get('PATH', None) + self.save_pythonpath = os.environ.get('PYTHONPATH', None) + + install_scheme = 'nt' if (os.name == 'nt') else 'posix_prefix' + install_dirs = get_paths(install_scheme, vars={ + 'base': self.path, + 'platbase': self.path, + }) + + scripts = install_dirs['scripts'] + if self.save_path: + os.environ['PATH'] = scripts + os.pathsep + self.save_path + else: + os.environ['PATH'] = scripts + os.pathsep + os.defpath + + if install_dirs['purelib'] == install_dirs['platlib']: + lib_dirs = install_dirs['purelib'] + else: + lib_dirs = install_dirs['purelib'] + os.pathsep + \ + install_dirs['platlib'] + if self.save_pythonpath: + os.environ['PYTHONPATH'] = lib_dirs + os.pathsep + \ + self.save_pythonpath + else: + os.environ['PYTHONPATH'] = lib_dirs + + return self + + def pip_install(self, reqs): + """Install dependencies into this env by calling pip in a subprocess""" + if not reqs: + return + log.info('Calling pip to install %s', reqs) + check_call([sys.executable, '-m', 'pip', 'install', '--ignore-installed', + '--prefix', self.path] + list(reqs)) + + def __exit__(self, exc_type, exc_val, exc_tb): + if self._cleanup and (self.path is not None) and os.path.isdir(self.path): + shutil.rmtree(self.path) + + if self.save_path is None: + os.environ.pop('PATH', None) + else: + os.environ['PATH'] = self.save_path + + if self.save_pythonpath is None: + os.environ.pop('PYTHONPATH', None) + else: + os.environ['PYTHONPATH'] = self.save_pythonpath + +def build_wheel(source_dir, wheel_dir, config_settings=None): + """Build a wheel from a source directory using PEP 517 hooks. + + :param str source_dir: Source directory containing pyproject.toml + :param str wheel_dir: Target directory to create wheel in + :param dict config_settings: Options to pass to build backend + + This is a blocking function which will run pip in a subprocess to install + build requirements. + """ + if config_settings is None: + config_settings = {} + requires, backend = _load_pyproject(source_dir) + hooks = Pep517HookCaller(source_dir, backend) + + with BuildEnvironment() as env: + env.pip_install(requires) + reqs = hooks.get_requires_for_build_wheel(config_settings) + env.pip_install(reqs) + return hooks.build_wheel(wheel_dir, config_settings) + + +def build_sdist(source_dir, sdist_dir, config_settings=None): + """Build an sdist from a source directory using PEP 517 hooks. + + :param str source_dir: Source directory containing pyproject.toml + :param str sdist_dir: Target directory to place sdist in + :param dict config_settings: Options to pass to build backend + + This is a blocking function which will run pip in a subprocess to install + build requirements. + """ + if config_settings is None: + config_settings = {} + requires, backend = _load_pyproject(source_dir) + hooks = Pep517HookCaller(source_dir, backend) + + with BuildEnvironment() as env: + env.pip_install(requires) + reqs = hooks.get_requires_for_build_sdist(config_settings) + env.pip_install(reqs) + return hooks.build_sdist(sdist_dir, config_settings) diff --git a/pipenv/patched/notpip/_vendor/pep517/wrappers.py b/pipenv/patched/notpip/_vendor/pep517/wrappers.py new file mode 100644 index 00000000..28260f32 --- /dev/null +++ b/pipenv/patched/notpip/_vendor/pep517/wrappers.py @@ -0,0 +1,134 @@ +from contextlib import contextmanager +import os +from os.path import dirname, abspath, join as pjoin +import shutil +from subprocess import check_call +import sys +from tempfile import mkdtemp + +from . import compat + +_in_proc_script = pjoin(dirname(abspath(__file__)), '_in_process.py') + +@contextmanager +def tempdir(): + td = mkdtemp() + try: + yield td + finally: + shutil.rmtree(td) + +class UnsupportedOperation(Exception): + """May be raised by build_sdist if the backend indicates that it can't.""" + +class Pep517HookCaller(object): + """A wrapper around a source directory to be built with a PEP 517 backend. + + source_dir : The path to the source directory, containing pyproject.toml. + backend : The build backend spec, as per PEP 517, from pyproject.toml. + """ + def __init__(self, source_dir, build_backend): + self.source_dir = abspath(source_dir) + self.build_backend = build_backend + + def get_requires_for_build_wheel(self, config_settings=None): + """Identify packages required for building a wheel + + Returns a list of dependency specifications, e.g.: + ["wheel >= 0.25", "setuptools"] + + This does not include requirements specified in pyproject.toml. + It returns the result of calling the equivalently named hook in a + subprocess. + """ + return self._call_hook('get_requires_for_build_wheel', { + 'config_settings': config_settings + }) + + def prepare_metadata_for_build_wheel(self, metadata_directory, config_settings=None): + """Prepare a *.dist-info folder with metadata for this project. + + Returns the name of the newly created folder. + + If the build backend defines a hook with this name, it will be called + in a subprocess. If not, the backend will be asked to build a wheel, + and the dist-info extracted from that. + """ + return self._call_hook('prepare_metadata_for_build_wheel', { + 'metadata_directory': abspath(metadata_directory), + 'config_settings': config_settings, + }) + + def build_wheel(self, wheel_directory, config_settings=None, metadata_directory=None): + """Build a wheel from this project. + + Returns the name of the newly created file. + + In general, this will call the 'build_wheel' hook in the backend. + However, if that was previously called by + 'prepare_metadata_for_build_wheel', and the same metadata_directory is + used, the previously built wheel will be copied to wheel_directory. + """ + if metadata_directory is not None: + metadata_directory = abspath(metadata_directory) + return self._call_hook('build_wheel', { + 'wheel_directory': abspath(wheel_directory), + 'config_settings': config_settings, + 'metadata_directory': metadata_directory, + }) + + def get_requires_for_build_sdist(self, config_settings=None): + """Identify packages required for building a wheel + + Returns a list of dependency specifications, e.g.: + ["setuptools >= 26"] + + This does not include requirements specified in pyproject.toml. + It returns the result of calling the equivalently named hook in a + subprocess. + """ + return self._call_hook('get_requires_for_build_sdist', { + 'config_settings': config_settings + }) + + def build_sdist(self, sdist_directory, config_settings=None): + """Build an sdist from this project. + + Returns the name of the newly created file. + + This calls the 'build_sdist' backend hook in a subprocess. + """ + return self._call_hook('build_sdist', { + 'sdist_directory': abspath(sdist_directory), + 'config_settings': config_settings, + }) + + + def _call_hook(self, hook_name, kwargs): + env = os.environ.copy() + + # On Python 2, pytoml returns Unicode values (which is correct) but the + # environment passed to check_call needs to contain string values. We + # convert here by encoding using ASCII (the backend can only contain + # letters, digits and _, . and : characters, and will be used as a + # Python identifier, so non-ASCII content is wrong on Python 2 in + # any case). + if sys.version_info[0] == 2: + build_backend = self.build_backend.encode('ASCII') + else: + build_backend = self.build_backend + + env['PEP517_BUILD_BACKEND'] = build_backend + with tempdir() as td: + compat.write_json({'kwargs': kwargs}, pjoin(td, 'input.json'), + indent=2) + + # Run the hook in a subprocess + check_call([sys.executable, _in_proc_script, hook_name, td], + cwd=self.source_dir, env=env) + + data = compat.read_json(pjoin(td, 'output.json')) + if data.get('unsupported'): + raise UnsupportedOperation + return data['return_val'] + diff --git a/pipenv/patched/notpip/_vendor/pkg_resources/__init__.py b/pipenv/patched/notpip/_vendor/pkg_resources/__init__.py index ed57821d..ac893b66 100644 --- a/pipenv/patched/notpip/_vendor/pkg_resources/__init__.py +++ b/pipenv/patched/notpip/_vendor/pkg_resources/__init__.py @@ -47,6 +47,11 @@ except ImportError: # Python 3.2 compatibility import imp as _imp +try: + FileExistsError +except NameError: + FileExistsError = OSError + from pipenv.patched.notpip._vendor import six from pipenv.patched.notpip._vendor.six.moves import urllib, map, filter @@ -78,8 +83,11 @@ __import__('pipenv.patched.notpip._vendor.packaging.requirements') __import__('pipenv.patched.notpip._vendor.packaging.markers') -if (3, 0) < sys.version_info < (3, 3): - raise RuntimeError("Python 3.3 or later is required") +__metaclass__ = type + + +if (3, 0) < sys.version_info < (3, 4): + raise RuntimeError("Python 3.4 or later is required") if six.PY2: # Those builtin exceptions are only defined in Python 3 @@ -537,7 +545,7 @@ class IResourceProvider(IMetadataProvider): """List of resource names in the directory (like ``os.listdir()``)""" -class WorkingSet(object): +class WorkingSet: """A collection of active distributions on sys.path (or a similar list)""" def __init__(self, entries=None): @@ -637,13 +645,12 @@ class WorkingSet(object): distributions in the working set, otherwise only ones matching both `group` and `name` are yielded (in distribution order). """ - for dist in self: - entries = dist.get_entry_map(group) - if name is None: - for ep in entries.values(): - yield ep - elif name in entries: - yield entries[name] + return ( + entry + for dist in self + for entry in dist.get_entry_map(group).values() + if name is None or name == entry.name + ) def run_script(self, requires, script_name): """Locate distribution for `requires` and run `script_name` script""" @@ -944,7 +951,7 @@ class _ReqExtras(dict): return not req.marker or any(extra_evals) -class Environment(object): +class Environment: """Searchable snapshot of distributions on a search path""" def __init__( @@ -959,7 +966,7 @@ class Environment(object): `platform` is an optional string specifying the name of the platform that platform-specific distributions must be compatible with. If unspecified, it defaults to the current platform. `python` is an - optional string naming the desired version of Python (e.g. ``'3.3'``); + optional string naming the desired version of Python (e.g. ``'3.6'``); it defaults to the current version. You may explicitly set `platform` (and/or `python`) to ``None`` if you @@ -2087,7 +2094,12 @@ def _handle_ns(packageName, path_item): importer = get_importer(path_item) if importer is None: return None - loader = importer.find_module(packageName) + + # capture warnings due to #1111 + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + loader = importer.find_module(packageName) + if loader is None: return None module = sys.modules.get(packageName) @@ -2132,12 +2144,13 @@ def _rebuild_mod_path(orig_path, package_name, module): parts = path_parts[:-module_parts] return safe_sys_path_index(_normalize_cached(os.sep.join(parts))) - if not isinstance(orig_path, list): - # Is this behavior useful when module.__path__ is not a list? - return + new_path = sorted(orig_path, key=position_in_sys_path) + new_path = [_normalize_cached(p) for p in new_path] - orig_path.sort(key=position_in_sys_path) - module.__path__[:] = [_normalize_cached(p) for p in orig_path] + if isinstance(module.__path__, list): + module.__path__[:] = new_path + else: + module.__path__ = new_path def declare_namespace(packageName): @@ -2148,9 +2161,10 @@ def declare_namespace(packageName): if packageName in _namespace_packages: return - path, parent = sys.path, None - if '.' in packageName: - parent = '.'.join(packageName.split('.')[:-1]) + path = sys.path + parent, _, _ = packageName.rpartition('.') + + if parent: declare_namespace(parent) if parent not in _namespace_packages: __import__(parent) @@ -2161,7 +2175,7 @@ def declare_namespace(packageName): # Track what packages are namespaces, so when new path items are added, # they can be updated - _namespace_packages.setdefault(parent, []).append(packageName) + _namespace_packages.setdefault(parent or None, []).append(packageName) _namespace_packages.setdefault(packageName, []) for path_item in path: @@ -2279,7 +2293,7 @@ EGG_NAME = re.compile( ).match -class EntryPoint(object): +class EntryPoint: """Object representing an advertised importable object""" def __init__(self, name, module_name, attrs=(), extras=(), dist=None): @@ -2433,7 +2447,7 @@ def _version_from_file(lines): return safe_version(value.strip()) or None -class Distribution(object): +class Distribution: """Wrap an actual or potential sys.path entry w/metadata""" PKG_INFO = 'PKG-INFO' @@ -3027,7 +3041,10 @@ def _bypass_ensure_directory(path): dirname, filename = split(path) if dirname and filename and not isdir(dirname): _bypass_ensure_directory(dirname) - mkdir(dirname, 0o755) + try: + mkdir(dirname, 0o755) + except FileExistsError: + pass def split_sections(s): diff --git a/pipenv/patched/notpip/_vendor/pkg_resources/py31compat.py b/pipenv/patched/notpip/_vendor/pkg_resources/py31compat.py index 331a51bb..3a44fa1c 100644 --- a/pipenv/patched/notpip/_vendor/pkg_resources/py31compat.py +++ b/pipenv/patched/notpip/_vendor/pkg_resources/py31compat.py @@ -2,6 +2,8 @@ import os import errno import sys +from pipenv.patched.notpip._vendor import six + def _makedirs_31(path, exist_ok=False): try: @@ -15,8 +17,7 @@ def _makedirs_31(path, exist_ok=False): # and exists_ok considerations are disentangled. # See https://github.com/pypa/setuptools/pull/1083#issuecomment-315168663 needs_makedirs = ( - sys.version_info < (3, 2, 5) or - (3, 3) <= sys.version_info < (3, 3, 6) or + six.PY2 or (3, 4) <= sys.version_info < (3, 4, 1) ) makedirs = _makedirs_31 if needs_makedirs else os.makedirs diff --git a/pipenv/patched/notpip/_vendor/pyparsing.py b/pipenv/patched/notpip/_vendor/pyparsing.py index b5ba3d5f..455d1151 100644 --- a/pipenv/patched/notpip/_vendor/pyparsing.py +++ b/pipenv/patched/notpip/_vendor/pyparsing.py @@ -1,6 +1,6 @@ # module pyparsing.py # -# Copyright (c) 2003-2016 Paul T. McGuire +# Copyright (c) 2003-2018 Paul T. McGuire # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -25,6 +25,7 @@ __doc__ = \ """ pyparsing module - Classes and methods to define and execute parsing grammars +============================================================================= The pyparsing module is an alternative approach to creating and executing simple grammars, vs. the traditional lex/yacc approach, or the use of regular expressions. With pyparsing, you @@ -58,10 +59,23 @@ The pyparsing module handles some of the problems that are typically vexing when - extra or missing whitespace (the above program will also handle "Hello,World!", "Hello , World !", etc.) - quoted strings - embedded comments + + +Getting Started - +----------------- +Visit the classes L{ParserElement} and L{ParseResults} to see the base classes that most other pyparsing +classes inherit from. Use the docstrings for examples of how to: + - construct literal match expressions from L{Literal} and L{CaselessLiteral} classes + - construct character word-group expressions using the L{Word} class + - see how to create repetitive expressions using L{ZeroOrMore} and L{OneOrMore} classes + - use L{'+'}, L{'|'}, L{'^'}, and L{'&'} operators to combine simple expressions into more complex ones + - associate names with your parsed results using L{ParserElement.setResultsName} + - find some helpful expression short-cuts like L{delimitedList} and L{oneOf} + - find more useful common expressions in the L{pyparsing_common} namespace class """ -__version__ = "2.2.0" -__versionTime__ = "06 Mar 2017 02:06 UTC" +__version__ = "2.2.1" +__versionTime__ = "18 Sep 2018 00:49 UTC" __author__ = "Paul McGuire " import string @@ -82,6 +96,15 @@ try: except ImportError: from threading import RLock +try: + # Python 3 + from collections.abc import Iterable + from collections.abc import MutableMapping +except ImportError: + # Python 2.7 + from collections import Iterable + from collections import MutableMapping + try: from collections import OrderedDict as _OrderedDict except ImportError: @@ -940,7 +963,7 @@ class ParseResults(object): def __dir__(self): return (dir(type(self)) + list(self.keys())) -collections.MutableMapping.register(ParseResults) +MutableMapping.register(ParseResults) def col (loc,strg): """Returns current column within a string, counting newlines as line separators. @@ -1025,11 +1048,11 @@ def _trim_arity(func, maxargs=2): # special handling for Python 3.5.0 - extra deep call stack by 1 offset = -3 if system_version == (3,5,0) else -2 frame_summary = traceback.extract_stack(limit=-offset+limit-1)[offset] - return [(frame_summary.filename, frame_summary.lineno)] + return [frame_summary[:2]] def extract_tb(tb, limit=0): frames = traceback.extract_tb(tb, limit=limit) frame_summary = frames[-1] - return [(frame_summary.filename, frame_summary.lineno)] + return [frame_summary[:2]] else: extract_stack = traceback.extract_stack extract_tb = traceback.extract_tb @@ -1374,7 +1397,7 @@ class ParserElement(object): else: preloc = loc tokensStart = preloc - if self.mayIndexError or loc >= len(instring): + if self.mayIndexError or preloc >= len(instring): try: loc,tokens = self.parseImpl( instring, preloc, doActions ) except IndexError: @@ -1408,7 +1431,6 @@ class ParserElement(object): self.resultsName, asList=self.saveAsList and isinstance(tokens,(ParseResults,list)), modal=self.modalResults ) - if debugging: #~ print ("Matched",self,"->",retTokens.asList()) if (self.debugActions[1] ): @@ -3242,7 +3264,7 @@ class ParseExpression(ParserElement): if isinstance( exprs, basestring ): self.exprs = [ ParserElement._literalStringClass( exprs ) ] - elif isinstance( exprs, collections.Iterable ): + elif isinstance( exprs, Iterable ): exprs = list(exprs) # if sequence of strings provided, wrap with Literal if all(isinstance(expr, basestring) for expr in exprs): @@ -4393,7 +4415,7 @@ def traceParseAction(f): @traceParseAction def remove_duplicate_chars(tokens): - return ''.join(sorted(set(''.join(tokens))) + return ''.join(sorted(set(''.join(tokens)))) wds = OneOrMore(wd).setParseAction(remove_duplicate_chars) print(wds.parseString("slkdjs sld sldd sdlf sdljf")) @@ -4583,7 +4605,7 @@ def oneOf( strs, caseless=False, useRegex=True ): symbols = [] if isinstance(strs,basestring): symbols = strs.split() - elif isinstance(strs, collections.Iterable): + elif isinstance(strs, Iterable): symbols = list(strs) else: warnings.warn("Invalid argument to oneOf, expected string or iterable", @@ -4734,7 +4756,7 @@ stringEnd = StringEnd().setName("stringEnd") _escapedPunc = Word( _bslash, r"\[]-*.$+^?()~ ", exact=2 ).setParseAction(lambda s,l,t:t[0][1]) _escapedHexChar = Regex(r"\\0?[xX][0-9a-fA-F]+").setParseAction(lambda s,l,t:unichr(int(t[0].lstrip(r'\0x'),16))) _escapedOctChar = Regex(r"\\0[0-7]+").setParseAction(lambda s,l,t:unichr(int(t[0][1:],8))) -_singleChar = _escapedPunc | _escapedHexChar | _escapedOctChar | Word(printables, excludeChars=r'\]', exact=1) | Regex(r"\w", re.UNICODE) +_singleChar = _escapedPunc | _escapedHexChar | _escapedOctChar | CharsNotIn(r'\]', exact=1) _charRange = Group(_singleChar + Suppress("-") + _singleChar) _reBracketExpr = Literal("[") + Optional("^").setResultsName("negate") + Group( OneOrMore( _charRange | _singleChar ) ).setResultsName("body") + "]" diff --git a/pipenv/patched/notpip/_vendor/pytoml/parser.py b/pipenv/patched/notpip/_vendor/pytoml/parser.py index e03a03fb..9f94e923 100644 --- a/pipenv/patched/notpip/_vendor/pytoml/parser.py +++ b/pipenv/patched/notpip/_vendor/pytoml/parser.py @@ -223,8 +223,8 @@ _float_re = re.compile(r'[+-]?(?:0|[1-9](?:_?\d)*)(?:\.\d(?:_?\d)*)?(?:[eE][+-]? _datetime_re = re.compile(r'(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(\.\d+)?(?:Z|([+-]\d{2}):(\d{2}))') _basicstr_ml_re = re.compile(r'(?:(?:|"|"")[^"\\\000-\011\013-\037])*') -_litstr_re = re.compile(r"[^'\000-\037]*") -_litstr_ml_re = re.compile(r"(?:(?:|'|'')(?:[^'\000-\011\013-\037]))*") +_litstr_re = re.compile(r"[^'\000\010\012-\037]*") +_litstr_ml_re = re.compile(r"(?:(?:|'|'')(?:[^'\000-\010\013-\037]))*") def _p_value(s, object_pairs_hook): pos = s.pos() diff --git a/pipenv/patched/notpip/_vendor/requests/__init__.py b/pipenv/patched/notpip/_vendor/requests/__init__.py index 8e7576c1..af3d4b1d 100644 --- a/pipenv/patched/notpip/_vendor/requests/__init__.py +++ b/pipenv/patched/notpip/_vendor/requests/__init__.py @@ -91,7 +91,7 @@ except (AssertionError, ValueError): RequestsDependencyWarning) # Attempt to enable urllib3's SNI support, if possible -from pipenv.patched.notpip._internal.compat import WINDOWS +from pipenv.patched.notpip._internal.utils.compat import WINDOWS if not WINDOWS: try: from pipenv.patched.notpip._vendor.urllib3.contrib import pyopenssl diff --git a/pipenv/patched/notpip/_vendor/vendor.txt b/pipenv/patched/notpip/_vendor/vendor.txt index b9854e9a..9389dd94 100644 --- a/pipenv/patched/notpip/_vendor/vendor.txt +++ b/pipenv/patched/notpip/_vendor/vendor.txt @@ -9,14 +9,15 @@ msgpack-python==0.5.6 lockfile==0.12.2 progress==1.4 ipaddress==1.0.22 # Only needed on 2.6 and 2.7 -packaging==17.1 -pyparsing==2.2.0 -pytoml==0.1.16 +packaging==18.0 +pep517==0.2 +pyparsing==2.2.1 +pytoml==0.1.19 retrying==1.3.3 requests==2.19.1 chardet==3.0.4 idna==2.7 urllib3==1.23 - certifi==2018.4.16 -setuptools==39.2.0 + certifi==2018.8.24 +setuptools==40.4.3 webencodings==0.5.1 diff --git a/pipenv/patched/patched.txt b/pipenv/patched/patched.txt index 3ad4c1e9..e7dadd8e 100644 --- a/pipenv/patched/patched.txt +++ b/pipenv/patched/patched.txt @@ -1,7 +1,5 @@ safety -git+https://github.com/jumpscale7/python-consistent-toml.git#egg=contoml crayons==0.1.2 pipfile==0.0.2 pip-tools==3.1.0 -prettytoml==0.3 -pip==18.0 +pip==18.1 diff --git a/pipenv/patched/piptools/_compat/pip_compat.py b/pipenv/patched/piptools/_compat/pip_compat.py index de9b4353..c466ef04 100644 --- a/pipenv/patched/piptools/_compat/pip_compat.py +++ b/pipenv/patched/piptools/_compat/pip_compat.py @@ -1,49 +1,55 @@ # -*- coding=utf-8 -*- -import importlib -from pip_shims import pip_version -import pkg_resources +__all__ = [ + "InstallRequirement", + "parse_requirements", + "RequirementSet", + "user_cache_dir", + "FAVORITE_HASH", + "is_file_url", + "url_to_path", + "PackageFinder", + "FormatControl", + "Wheel", + "Command", + "cmdoptions", + "get_installed_distributions", + "PyPI", + "SafeFileCache", + "InstallationError", + "parse_version", + "pip_version", + "install_req_from_editable", + "install_req_from_line", + "user_cache_dir" +] -def do_import(module_path, subimport=None, old_path=None, vendored_name=None): - old_path = old_path or module_path - prefix = vendored_name if vendored_name else "pip" - prefixes = ["{0}._internal".format(prefix), "{0}".format(prefix)] - paths = [module_path, old_path] - search_order = ["{0}.{1}".format(p, pth) for p in prefixes for pth in paths if pth is not None] - package = subimport if subimport else None - for to_import in search_order: - if not subimport: - to_import, _, package = to_import.rpartition(".") - try: - imported = importlib.import_module(to_import) - except ImportError: - continue - else: - return getattr(imported, package) - - -InstallRequirement = do_import('req.req_install', 'InstallRequirement', vendored_name="notpip") -parse_requirements = do_import('req.req_file', 'parse_requirements', vendored_name="notpip") -RequirementSet = do_import('req.req_set', 'RequirementSet', vendored_name="notpip") -user_cache_dir = do_import('utils.appdirs', 'user_cache_dir', vendored_name="notpip") -FAVORITE_HASH = do_import('utils.hashes', 'FAVORITE_HASH', vendored_name="notpip") -is_file_url = do_import('download', 'is_file_url', vendored_name="notpip") -url_to_path = do_import('download', 'url_to_path', vendored_name="notpip") -PackageFinder = do_import('index', 'PackageFinder', vendored_name="notpip") -FormatControl = do_import('index', 'FormatControl', vendored_name="notpip") -Wheel = do_import('wheel', 'Wheel', vendored_name="notpip") -Command = do_import('cli.base_command', 'Command', old_path='basecommand', vendored_name="notpip") -cmdoptions = do_import('cli.cmdoptions', old_path='cmdoptions', vendored_name="notpip") -get_installed_distributions = do_import('utils.misc', 'get_installed_distributions', old_path='utils', vendored_name="notpip") -PyPI = do_import('models.index', 'PyPI', vendored_name='notpip') -SafeFileCache = do_import('download', 'SafeFileCache', vendored_name='notpip') -InstallationError = do_import('exceptions', 'InstallationError', vendored_name='notpip') +from pipenv.vendor.appdirs import user_cache_dir +from pip_shims.shims import ( + InstallRequirement, + parse_requirements, + RequirementSet, + FAVORITE_HASH, + is_file_url, + url_to_path, + PackageFinder, + FormatControl, + Wheel, + Command, + cmdoptions, + get_installed_distributions, + PyPI, + SafeFileCache, + InstallationError, + parse_version, + pip_version, +) # pip 18.1 has refactored InstallRequirement constructors use by pip-tools. -if pkg_resources.parse_version(pip_version) < pkg_resources.parse_version('18.1'): +if parse_version(pip_version) < parse_version('18.1'): install_req_from_line = InstallRequirement.from_line install_req_from_editable = InstallRequirement.from_editable else: - install_req_from_line = do_import('req.constructors', 'install_req_from_line', vendored_name="notpip") - install_req_from_editable = do_import('req.constructors', 'install_req_from_editable', vendored_name="notpip") - + from pip_shims.shims import ( + install_req_from_editable, install_req_from_line + ) diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py index c1871670..2a0743a3 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -19,11 +19,10 @@ from .._compat import ( InstallRequirement, SafeFileCache ) -os.environ["PIP_SHIMS_BASE_MODULE"] = str("notpip") +os.environ["PIP_SHIMS_BASE_MODULE"] = str("pipenv.patched.notpip") from pip_shims.shims import do_import, VcsSupport, WheelCache from packaging.requirements import Requirement from packaging.specifiers import SpecifierSet, Specifier -from packaging.markers import Op, Value, Variable, Marker InstallationError = do_import(("exceptions.InstallationError", "7.0", "9999")) from pipenv.patched.notpip._internal.resolve import Resolver as PipResolver @@ -31,7 +30,7 @@ from pipenv.patched.notpip._internal.resolve import Resolver as PipResolver from pipenv.environments import PIPENV_CACHE_DIR as CACHE_DIR from ..exceptions import NoCandidateFound from ..utils import (fs_str, is_pinned_requirement, lookup_table, dedup, - make_install_requirement, clean_requires_python) + make_install_requirement, clean_requires_python) from .base import BaseRepository try: @@ -243,6 +242,7 @@ class PyPIRepository(BaseRepository): dist = None ireq.isolated = False ireq._wheel_cache = wheel_cache + try: from pipenv.patched.notpip._internal.operations.prepare import RequirementPreparer except ImportError: @@ -295,7 +295,18 @@ class PyPIRepository(BaseRepository): resolver = PipResolver(**resolver_kwargs) resolver.require_hashes = False results = resolver._resolve_one(reqset, ireq) - reqset.cleanup_files() + + cleanup_fn = getattr(reqset, "cleanup_files", None) + if cleanup_fn is not None: + try: + cleanup_fn() + except OSError: + pass + + if ireq.editable and (not ireq.source_dir or not os.path.exists(ireq.source_dir)): + if ireq.editable: + self._source_dir = TemporaryDirectory(fs_str("source")) + ireq.ensure_has_source_dir(self.source_dir) if ireq.editable and (ireq.source_dir and os.path.exists(ireq.source_dir)): # Collect setup_requires info from local eggs. diff --git a/pipenv/patched/prettytoml/__init__.py b/pipenv/patched/prettytoml/__init__.py deleted file mode 100644 index d7310748..00000000 --- a/pipenv/patched/prettytoml/__init__.py +++ /dev/null @@ -1,25 +0,0 @@ -from ._version import VERSION - -__version__ = VERSION - - -def prettify(toml_text): - """ - Prettifies and returns the TOML file content provided. - """ - from .parser import parse_tokens - from .lexer import tokenize - from .prettifier import prettify as element_prettify - - tokens = tokenize(toml_text, is_top_level=True) - elements = parse_tokens(tokens) - prettified = element_prettify(elements) - return ''.join(pretty_element.serialized() for pretty_element in prettified) - - -def prettify_from_file(file_path): - """ - Reads, prettifies and returns the TOML file specified by the file_path. - """ - with open(file_path, 'r') as fp: - return prettify(fp.read()) diff --git a/pipenv/patched/prettytoml/_version.py b/pipenv/patched/prettytoml/_version.py deleted file mode 100644 index e0f15470..00000000 --- a/pipenv/patched/prettytoml/_version.py +++ /dev/null @@ -1 +0,0 @@ -VERSION = 'master' diff --git a/pipenv/patched/prettytoml/elements/__init__.py b/pipenv/patched/prettytoml/elements/__init__.py deleted file mode 100644 index ece21123..00000000 --- a/pipenv/patched/prettytoml/elements/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ - -""" - TOML file elements (a higher abstraction layer than individual lexical tokens). -""" - -from .traversal import TraversalMixin -from .errors import InvalidElementError -from .table import TableElement -from .tableheader import TableHeaderElement -from .common import TYPE_METADATA, TYPE_ATOMIC, TYPE_CONTAINER, TYPE_MARKUP - -from . import traversal -from . import factory diff --git a/pipenv/patched/prettytoml/elements/abstracttable.py b/pipenv/patched/prettytoml/elements/abstracttable.py deleted file mode 100644 index 6a2c174a..00000000 --- a/pipenv/patched/prettytoml/elements/abstracttable.py +++ /dev/null @@ -1,92 +0,0 @@ -try: - from collections.abc import Mapping -except ImportError: - from collections import Mapping - -from prettytoml.elements.common import ContainerElement -from prettytoml.elements import traversal - - -class AbstractTable(ContainerElement, traversal.TraversalMixin, Mapping): - """ - Common code for handling tables as key-value pairs with metadata elements sprinkled all over. - - Assumes input sub_elements are correct. - """ - - def __init__(self, sub_elements): - ContainerElement.__init__(self, sub_elements) - self._fallback = None - - def _enumerate_items(self): - """ - Returns ((key_index, key_element), (value_index, value_element)) for all the element key-value pairs. - """ - non_metadata = self._enumerate_non_metadata_sub_elements() - while True: - try: - yield next(non_metadata), next(non_metadata) - except StopIteration: - return - - def items(self): - for (key_i, key), (value_i, value) in self._enumerate_items(): - yield key.value, value.value - if self._fallback: - for key, value in self._fallback.items(): - yield key, value - - def keys(self): - return tuple(key for (key, _) in self.items()) - - def values(self): - return tuple(value for (_, value) in self.items()) - - def __len__(self): - return len(tuple(self._enumerate_items())) - - def __iter__(self): - return (key for key, _ in self.items()) - - def __contains__(self, item): - return item in self.keys() - - def _find_key_and_value(self, key): - """ - Returns (key_i, value_i) corresponding to the given key value. - - Raises KeyError if no matching key found. - """ - for (key_i, key_element), (value_i, value_element) in self._enumerate_items(): - if key_element.value == key: - return key_i, value_i - raise KeyError - - def __getitem__(self, item): - for key, value in self.items(): - if key == item: - return value - raise KeyError - - def get(self, key, default=None): - try: - return self[key] - except KeyError: - return default - - def set_fallback(self, fallback): - """ - Sets a fallback dict-like instance to be used to look up values after they are not found - in this instance. - """ - self._fallback = fallback - - @property - def primitive_value(self): - """ - Returns a primitive Python value without any formatting or markup metadata. - """ - return { - key: - value.primitive_value if hasattr(value, 'primitive_value') else value for key, value in self.items() - } diff --git a/pipenv/patched/prettytoml/elements/array.py b/pipenv/patched/prettytoml/elements/array.py deleted file mode 100644 index 36c648b9..00000000 --- a/pipenv/patched/prettytoml/elements/array.py +++ /dev/null @@ -1,136 +0,0 @@ -from prettytoml.elements import common, factory, traversal -from prettytoml.elements.common import Element, ContainerElement -from prettytoml.elements.factory import create_element -from prettytoml.elements.metadata import NewlineElement -from prettytoml.elements.errors import InvalidElementError - - -class ArrayElement(ContainerElement, traversal.TraversalMixin): - """ - A sequence-like container element containing other atomic elements or other containers. - - Implements list-like interface. - - Assumes input sub_elements are correct for an array element. - - Raises an InvalidElementError if contains heterogeneous values. - """ - - def __init__(self, sub_elements): - common.ContainerElement.__init__(self, sub_elements) - self._check_homogeneity() - - def _check_homogeneity(self): - if len(set(type(v) for v in self.primitive_value)) > 1: - raise InvalidElementError('Array should be homogeneous') - - def __len__(self): - return len(tuple(self._enumerate_non_metadata_sub_elements())) - - def __getitem__(self, i): - """ - Returns the ith entry, which can be a primitive value, a seq-lie, or a dict-like object. - """ - return self._find_value(i)[1].value - - def __setitem__(self, i, value): - value_i, _ = self._find_value(i) - new_element = value if isinstance(value, Element) else factory.create_element(value) - self._sub_elements = self.sub_elements[:value_i] + [new_element] + self.sub_elements[value_i+1:] - - @property - def value(self): - return self # self is a sequence-like value - - @property - def primitive_value(self): - """ - Returns a primitive Python value without any formatting or markup metadata. - """ - return list( - self[i].primitive_value if hasattr(self[i], 'primitive_value') - else self[i] - for i in range(len(self))) - - def __str__(self): - return "Array{}".format(self.primitive_value) - - def append(self, v): - new_entry = [create_element(v)] - - if self: # If not empty, we need a comma and whitespace prefix! - new_entry = [ - factory.create_operator_element(','), - factory.create_whitespace_element(), - ] + new_entry - - insertion_index = self._find_closing_square_bracket() - self._sub_elements = self._sub_elements[:insertion_index] + new_entry + \ - self._sub_elements[insertion_index:] - - def _find_value(self, i): - """ - Returns (value_index, value) of ith value in this sequence. - - Raises IndexError if not found. - """ - return tuple(self._enumerate_non_metadata_sub_elements())[i] - - def __delitem__(self, i): - value_i, value = self._find_value(i) - - begin, end = value_i, value_i+1 - - # Rules: - # 1. begin should be index to the preceding comma to the value - # 2. end should be index to the following comma, or the closing bracket - # 3. If no preceding comma found but following comma found then end should be the index of the following value - - preceding_comma = self._find_preceding_comma(value_i) - found_preceding_comma = preceding_comma >= 0 - if found_preceding_comma: - begin = preceding_comma - - following_comma = self._find_following_comma(value_i) - if following_comma >= 0: - if not found_preceding_comma: - end = self._find_following_non_metadata(following_comma) - else: - end = following_comma - else: - end = self._find_following_closing_square_bracket() - - self._sub_elements = self.sub_elements[:begin] + self._sub_elements[end:] - - @property - def is_multiline(self): - return any(isinstance(e, (NewlineElement)) for e in self.elements) - - def turn_into_multiline(self): - """ - Turns this array into a multi-line array with each element lying on its own line. - """ - if self.is_multiline: - return - - i = self._find_following_comma(-1) - - def next_entry_i(): - return self._find_following_non_metadata(i) - - def next_newline_i(): - return self._find_following_newline(i) - - def next_closing_bracket_i(): - return self._find_following_closing_square_bracket(i) - - def next_comma_i(): - return self._find_following_comma(i) - - while i < len(self.elements)-1: - if next_newline_i() < next_entry_i(): - self.elements.insert(i+1, factory.create_newline_element()) - if float('-inf') < next_comma_i() < next_closing_bracket_i(): - i = next_comma_i() - else: - i = next_closing_bracket_i() diff --git a/pipenv/patched/prettytoml/elements/atomic.py b/pipenv/patched/prettytoml/elements/atomic.py deleted file mode 100644 index 571810d9..00000000 --- a/pipenv/patched/prettytoml/elements/atomic.py +++ /dev/null @@ -1,52 +0,0 @@ -from ..tokens import py2toml, toml2py -from . import common -from prettytoml.util import is_dict_like, is_sequence_like -from .errors import InvalidElementError - - -class AtomicElement(common.TokenElement): - """ - An element containing a sequence of tokens representing a single atomic value that can be updated in place. - - Raises: - InvalidElementError: when passed an invalid sequence of tokens. - """ - - def __init__(self, _tokens): - common.TokenElement.__init__(self, _tokens, common.TYPE_ATOMIC) - - def _validate_tokens(self, _tokens): - if len([token for token in _tokens if not token.type.is_metadata]) != 1: - raise InvalidElementError('Tokens making up an AtomicElement must contain only one non-metadata token') - - def serialized(self): - return ''.join(token.source_substring for token in self.tokens) - - def _value_token_index(self): - """ - Finds the token where the value is stored. - """ - # TODO: memoize this value - for i, token in enumerate(self.tokens): - if not token.type.is_metadata: - return i - raise RuntimeError('could not find a value token') - - @property - def value(self): - """ - Returns a Python value contained in this atomic element. - """ - return toml2py.deserialize(self._tokens[self._value_token_index()]) - - @property - def primitive_value(self): - return self.value - - def set(self, value): - """ - Sets the contained value to the given one. - """ - assert (not is_sequence_like(value)) and (not is_dict_like(value)), 'the value must be an atomic primitive' - token_index = self._value_token_index() - self._tokens[token_index] = py2toml.create_primitive_token(value) diff --git a/pipenv/patched/prettytoml/elements/common.py b/pipenv/patched/prettytoml/elements/common.py deleted file mode 100644 index 7508047a..00000000 --- a/pipenv/patched/prettytoml/elements/common.py +++ /dev/null @@ -1,101 +0,0 @@ -from abc import abstractmethod - -TYPE_METADATA = 'element-metadata' -TYPE_ATOMIC = 'element-atomic' -TYPE_CONTAINER = 'element-container' -TYPE_MARKUP = 'element-markup' - -class Element: - """ - An Element: - - is one or more Token instances, or one or more other Element instances. Not both. - - knows how to serialize its value back to valid TOML code. - - A non-metadata Element is an Element that: - - knows how to deserialize its content into usable Python primitive, seq-like, or dict-like value. - - knows how to update its content from a Python primitive, seq-like, or dict-like value - while maintaining its formatting. - """ - - def __init__(self, _type): - self._type = _type - - @property - def type(self): - return self._type - - @abstractmethod - def serialized(self): - """ - TOML serialization of this element as str. - """ - raise NotImplementedError - - -class TokenElement(Element): - """ - An Element made up of tokens - """ - - def __init__(self, _tokens, _type): - Element.__init__(self, _type) - self._validate_tokens(_tokens) - self._tokens = list(_tokens) - - @property - def tokens(self): - return self._tokens - - @property - def first_token(self): - return self._tokens[0] - - @abstractmethod - def _validate_tokens(self, _tokens): - raise NotImplementedError - - def serialized(self): - return ''.join(token.source_substring for token in self._tokens) - - def __repr__(self): - return repr(self.tokens) - - @property - def primitive_value(self): - """ - Returns a primitive Python value without any formatting or markup metadata. - """ - raise NotImplementedError - - -class ContainerElement(Element): - """ - An Element containing exclusively other elements. - """ - - def __init__(self, sub_elements): - Element.__init__(self, TYPE_CONTAINER) - self._sub_elements = list(sub_elements) - - @property - def sub_elements(self): - return self._sub_elements - - @property - def elements(self): - return self.sub_elements - - def serialized(self): - return ''.join(element.serialized() for element in self.sub_elements) - - def __repr__(self): - return repr(self.primitive_value) - - @property - def primitive_value(self): - """ - Returns a primitive Python value without any formatting or markup metadata. - """ - raise NotImplementedError - - diff --git a/pipenv/patched/prettytoml/elements/errors.py b/pipenv/patched/prettytoml/elements/errors.py deleted file mode 100644 index 0fcf2e99..00000000 --- a/pipenv/patched/prettytoml/elements/errors.py +++ /dev/null @@ -1,13 +0,0 @@ - -class InvalidElementError(Exception): - """ - Raised by Element factories when the given sequence of tokens or sub-elements are invalid for the - specific type of Element being created. - """ - - def __init__(self, message): - self.message = message - - def __repr__(self): - return "InvalidElementError: {}".format(self.message) - diff --git a/pipenv/patched/prettytoml/elements/factory.py b/pipenv/patched/prettytoml/elements/factory.py deleted file mode 100644 index 177738db..00000000 --- a/pipenv/patched/prettytoml/elements/factory.py +++ /dev/null @@ -1,152 +0,0 @@ -import datetime -import functools -import six -from prettytoml import tokens -from prettytoml.tokens import py2toml -from prettytoml.elements.atomic import AtomicElement -from prettytoml.elements.metadata import PunctuationElement, WhitespaceElement, NewlineElement -from prettytoml.elements.tableheader import TableHeaderElement -from prettytoml.util import join_with, is_sequence_like - - -def create_element(value, multiline_strings_allowed=True): - """ - Creates and returns the appropriate elements.Element instance from the given Python primitive, sequence-like, - or dict-like value. - """ - from prettytoml.elements.array import ArrayElement - - if isinstance(value, (int, float, bool, datetime.datetime, datetime.date) + six.string_types) or value is None: - primitive_token = py2toml.create_primitive_token(value, multiline_strings_allowed=multiline_strings_allowed) - return AtomicElement((primitive_token,)) - - elif isinstance(value, (list, tuple)): - preamble = [create_operator_element('[')] - postable = [create_operator_element(']')] - stuffing_elements = [create_element(v) for v in value] - spaced_stuffing = join_with(stuffing_elements, - separator=[create_operator_element(','), create_whitespace_element()]) - - return ArrayElement(preamble + spaced_stuffing + postable) - - elif isinstance(value, dict): - return create_inline_table(value, multiline_table=False, multiline_strings_allowed=multiline_strings_allowed) - - else: - raise RuntimeError('Value type unaccounted for: {} of type {}'.format(value, type(value))) - - -def create_inline_table(from_dict, multiline_table=False, multiline_strings_allowed=True): - """ - Creates an InlineTable element from the given dict instance. - """ - - from prettytoml.elements.inlinetable import InlineTableElement - - preamble = [create_operator_element('{')] - postable = [create_operator_element('}')] - - stuffing_elements = ( - ( - create_string_element(k, bare_allowed=True), - create_whitespace_element(), - create_operator_element('='), - create_whitespace_element(), - create_element(v, multiline_strings_allowed=False) - ) for (k, v) in from_dict.items()) - - pair_separator = [create_operator_element(','), - create_newline_element() if multiline_table else create_whitespace_element()] - spaced_elements = join_with(stuffing_elements, separator=pair_separator) - - return InlineTableElement(preamble + spaced_elements + postable) - - -def create_string_element(value, bare_allowed=False): - """ - Creates and returns an AtomicElement wrapping a string value. - """ - return AtomicElement((py2toml.create_string_token(value, bare_allowed),)) - - -def create_operator_element(operator): - """ - Creates a PunctuationElement instance containing an operator token of the specified type. The operator - should be a TOML source str. - """ - operator_type_map = { - ',': tokens.TYPE_OP_COMMA, - '=': tokens.TYPE_OP_ASSIGNMENT, - '[': tokens.TYPE_OP_SQUARE_LEFT_BRACKET, - ']': tokens.TYPE_OP_SQUARE_RIGHT_BRACKET, - '[[': tokens.TYPE_OP_DOUBLE_SQUARE_LEFT_BRACKET, - ']]': tokens.TYPE_OP_DOUBLE_SQUARE_RIGHT_BRACKET, - '{': tokens.TYPE_OP_CURLY_LEFT_BRACKET, - '}': tokens.TYPE_OP_CURLY_RIGHT_BRACKET, - } - - ts = (tokens.Token(operator_type_map[operator], operator),) - return PunctuationElement(ts) - - -def create_newline_element(): - """ - Creates and returns a single NewlineElement. - """ - ts = (tokens.Token(tokens.TYPE_NEWLINE, '\n'),) - return NewlineElement(ts) - - -def create_whitespace_element(length=1, char=' '): - """ - Creates and returns a WhitespaceElement containing spaces. - """ - ts = (tokens.Token(tokens.TYPE_WHITESPACE, char),) * length - return WhitespaceElement(ts) - - -def create_table_header_element(names): - - name_elements = [] - - if isinstance(names, six.string_types): - name_elements = [py2toml.create_string_token(names, bare_string_allowed=True)] - else: - for (i, name) in enumerate(names): - name_elements.append(py2toml.create_string_token(name, bare_string_allowed=True)) - if i < (len(names)-1): - name_elements.append(py2toml.operator_token(tokens.TYPE_OPT_DOT)) - - return TableHeaderElement( - [py2toml.operator_token(tokens.TYPE_OP_SQUARE_LEFT_BRACKET)] + name_elements + - [py2toml.operator_token(tokens.TYPE_OP_SQUARE_RIGHT_BRACKET), py2toml.operator_token(tokens.TYPE_NEWLINE)], - ) - - -def create_array_of_tables_header_element(name): - return TableHeaderElement(( - py2toml.operator_token(tokens.TYPE_OP_DOUBLE_SQUARE_LEFT_BRACKET), - py2toml.create_string_token(name, bare_string_allowed=True), - py2toml.operator_token(tokens.TYPE_OP_DOUBLE_SQUARE_RIGHT_BRACKET), - py2toml.operator_token(tokens.TYPE_NEWLINE), - )) - - -def create_table(dict_value): - """ - Creates a TableElement out of a dict instance. - """ - from prettytoml.elements.table import TableElement - - if not isinstance(dict_value, dict): - raise ValueError('input must be a dict instance.') - - table_element = TableElement([create_newline_element()]) - for k, v in dict_value.items(): - table_element[k] = create_element(v) - - return table_element - - -def create_multiline_string(text, maximum_line_length): - return AtomicElement(_tokens=[py2toml.create_multiline_string(text, maximum_line_length)]) diff --git a/pipenv/patched/prettytoml/elements/inlinetable.py b/pipenv/patched/prettytoml/elements/inlinetable.py deleted file mode 100644 index 7b985fc1..00000000 --- a/pipenv/patched/prettytoml/elements/inlinetable.py +++ /dev/null @@ -1,78 +0,0 @@ -from prettytoml.elements import factory, abstracttable -from prettytoml.elements.common import Element - - -class InlineTableElement(abstracttable.AbstractTable): - """ - An Element containing key-value pairs, representing an inline table. - - Implements dict-like interface. - - Assumes input sub_elements are correct for an inline table element. - """ - - def __init__(self, sub_elements): - abstracttable.AbstractTable.__init__(self, sub_elements) - - def __setitem__(self, key, value): - - new_element = value if isinstance(value, Element) else factory.create_element(value) - - try: - - key_i, value_i = self._find_key_and_value(key) - # Found, then replace the value element with a new one - self._sub_elements = self.sub_elements[:value_i] + [new_element] + self.sub_elements[value_i+1:] - - except KeyError: # Key does not exist, adding anew! - - new_entry = [ - factory.create_string_element(key, bare_allowed=True), - factory.create_whitespace_element(), - factory.create_operator_element('='), - factory.create_whitespace_element(), - new_element, - ] - - if self: # If not empty - new_entry = [ - factory.create_operator_element(','), - factory.create_whitespace_element(), - ] + new_entry - - insertion_index = self._find_closing_curly_bracket() - self._sub_elements = self.sub_elements[:insertion_index] + new_entry + self.sub_elements[insertion_index:] - - def __delitem__(self, key): - - key_i, value_i = self._find_key_and_value(key) - - begin, end = key_i, value_i+1 - - # Rules: - # 1. begin should be index to the preceding comma to the key - # 2. end should be index to the following comma, or the closing bracket - # 3. If no preceding comma found but following comma found then end should be the index of the following key - - preceding_comma = self._find_preceding_comma(begin) - found_preceding_comma = preceding_comma >= 0 - if found_preceding_comma: - begin = preceding_comma - - following_comma = self._find_following_comma(value_i) - if following_comma >= 0: - if not found_preceding_comma: - end = self._find_following_non_metadata(following_comma) - else: - end = following_comma - else: - end = self._find_closing_curly_bracket() - - self._sub_elements = self.sub_elements[:begin] + self.sub_elements[end:] - - def multiline_equivalent(self): - return factory.create_inline_table(self.primitive_value, multiline_table=True, multiline_strings_allowed=True) - - @property - def value(self): - return self # self is a dict-like value that is perfectly usable diff --git a/pipenv/patched/prettytoml/elements/metadata.py b/pipenv/patched/prettytoml/elements/metadata.py deleted file mode 100644 index d5ee1061..00000000 --- a/pipenv/patched/prettytoml/elements/metadata.py +++ /dev/null @@ -1,80 +0,0 @@ -from prettytoml import tokens -from prettytoml.elements import common -from .errors import InvalidElementError - - -class WhitespaceElement(common.TokenElement): - """ - An element that contains tokens of whitespace - """ - - def __init__(self, _tokens): - common.TokenElement.__init__(self, _tokens, common.TYPE_METADATA) - - def _validate_tokens(self, _tokens): - for token in _tokens: - if token.type != tokens.TYPE_WHITESPACE: - raise InvalidElementError('Tokens making up a WhitespaceElement must all be whitespace') - - @property - def length(self): - """ - The whitespace length of this element - """ - return len(self.tokens) - - -class NewlineElement(common.TokenElement): - """ - An element containing newline tokens - - Raises: - InvalidElementError: when passed an invalid sequence of tokens. - """ - - def __init__(self, _tokens): - common.TokenElement.__init__(self, _tokens, common.TYPE_METADATA) - - def _validate_tokens(self, _tokens): - for token in _tokens: - if token.type != tokens.TYPE_NEWLINE: - raise InvalidElementError('Tokens making a NewlineElement must all be newlines') - - -class CommentElement(common.TokenElement): - """ - An element containing a single comment token followed by a newline. - - Raises: - InvalidElementError: when passed an invalid sequence of tokens. - """ - - def __init__(self, _tokens): - common.TokenElement.__init__(self, _tokens, common.TYPE_METADATA) - - def _validate_tokens(self, _tokens): - if len(_tokens) != 2 or _tokens[0].type != tokens.TYPE_COMMENT or _tokens[1].type != tokens.TYPE_NEWLINE: - raise InvalidElementError('CommentElement needs one comment token followed by one newline token') - - -class PunctuationElement(common.TokenElement): - """ - An element containing a single punctuation token. - - Raises: - InvalidElementError: when passed an invalid sequence of tokens. - """ - - def __init__(self, _tokens): - common.TokenElement.__init__(self, _tokens, common.TYPE_METADATA) - - @property - def token(self): - """ - Returns the token contained in this Element. - """ - return self.tokens[0] - - def _validate_tokens(self, _tokens): - if not _tokens or not tokens.is_operator(_tokens[0]): - raise InvalidElementError('PunctuationElement must be made of only a single operator token') diff --git a/pipenv/patched/prettytoml/elements/table.py b/pipenv/patched/prettytoml/elements/table.py deleted file mode 100644 index cdc3ed4c..00000000 --- a/pipenv/patched/prettytoml/elements/table.py +++ /dev/null @@ -1,122 +0,0 @@ -from prettytoml.elements import abstracttable, factory -from prettytoml.elements.errors import InvalidElementError -from prettytoml.elements.common import Element -from prettytoml.elements.metadata import CommentElement, NewlineElement, WhitespaceElement -from . import common - - -class TableElement(abstracttable.AbstractTable): - """ - An Element containing an unnamed top-level table. - - Implements dict-like interface. - - Assumes input sub_elements are correct. - - Raises InvalidElementError on duplicate keys. - """ - - def __init__(self, sub_elements): - abstracttable.AbstractTable.__init__(self, sub_elements) - - self._check_for_duplicate_keys() - - def _check_for_duplicate_keys(self): - if len(set(self.keys())) < len(self.keys()): - raise InvalidElementError('Duplicate keys found') - - def __setitem__(self, key, value): - if key in self: - self._update(key, value) - else: - self._insert(key, value) - - def _update(self, key, value): - _, value_i = self._find_key_and_value(key) - self._sub_elements[value_i] = value if isinstance(value, Element) else factory.create_element(value) - - def _find_insertion_index(self): - """ - Returns the self.sub_elements index in which new entries should be inserted. - """ - - non_metadata_elements = tuple(self._enumerate_non_metadata_sub_elements()) - - if not non_metadata_elements: - return 0 - - last_entry_i = non_metadata_elements[-1][0] - following_newline_i = self._find_following_line_terminator(last_entry_i) - - return following_newline_i + 1 - - def _detect_indentation_size(self): - """ - Detects the level of indentation used in this table. - """ - - def lines(): - # Returns a sequence of sequences of elements belonging to each line - start = 0 - for i, element in enumerate(self.elements): - if isinstance(element, (CommentElement, NewlineElement)): - yield self.elements[start:i+1] - start = i+1 - - def indentation(line): - # Counts the number of whitespace tokens at the beginning of this line - try: - first_non_whitespace_i = next(i for (i, e) in enumerate(line) if not isinstance(e, WhitespaceElement)) - return sum(space.length for space in line[:first_non_whitespace_i]) - except StopIteration: - return 0 - - def is_empty_line(line): - return all(e.type == common.TYPE_METADATA for e in line) - - try: - return min(indentation(line) for line in lines() if len(line) > 1 and not is_empty_line(line)) - except ValueError: # Raised by ValueError when no matching lines found - return 0 - - def _insert(self, key, value): - - value_element = value if isinstance(value, Element) else factory.create_element(value) - - indentation_size = self._detect_indentation_size() - indentation = [factory.create_whitespace_element(self._detect_indentation_size())] if indentation_size else [] - - inserted_elements = indentation + [ - factory.create_string_element(key, bare_allowed=True), - factory.create_whitespace_element(), - factory.create_operator_element('='), - factory.create_whitespace_element(), - value_element, - factory.create_newline_element(), - ] - - insertion_index = self._find_insertion_index() - - self._sub_elements = \ - self.sub_elements[:insertion_index] + inserted_elements + self.sub_elements[insertion_index:] - - def __delitem__(self, key): - begin, _ = self._find_key_and_value(key) - preceding_newline = self._find_preceding_newline(begin) - if preceding_newline >= 0: - begin = preceding_newline - end = self._find_following_line_terminator(begin) - if end < 0: - end = len(tuple(self._sub_elements)) - self._sub_elements = self.sub_elements[:begin] + self.sub_elements[end:] - - def pop(self, key): - v = self[key] - del self[key] - return v - - def value(self): - return self - - def __str__(self): - return str(self.primitive_value) diff --git a/pipenv/patched/prettytoml/elements/tableheader.py b/pipenv/patched/prettytoml/elements/tableheader.py deleted file mode 100644 index eacd88b9..00000000 --- a/pipenv/patched/prettytoml/elements/tableheader.py +++ /dev/null @@ -1,95 +0,0 @@ -from prettytoml import tokens -from prettytoml.tokens import toml2py -from prettytoml.elements import common -from prettytoml.elements.common import Element, TokenElement -from prettytoml.elements.errors import InvalidElementError - -_opening_bracket_types = (tokens.TYPE_OP_SQUARE_LEFT_BRACKET, tokens.TYPE_OP_DOUBLE_SQUARE_LEFT_BRACKET) -_closing_bracket_types = (tokens.TYPE_OP_SQUARE_RIGHT_BRACKET, tokens.TYPE_OP_DOUBLE_SQUARE_RIGHT_BRACKET) -_name_types = ( - tokens.TYPE_BARE_STRING, - tokens.TYPE_LITERAL_STRING, - tokens.TYPE_STRING, -) - - -class TableHeaderElement(TokenElement): - """ - An element containing opening and closing single and double square brackets, strings and dots and ending with - a newline. - - Raises InvalidElementError. - """ - - def __init__(self, _tokens): - TokenElement.__init__(self, _tokens, common.TYPE_MARKUP) - self._names = tuple(toml2py.deserialize(token) for token in self._tokens if token.type in _name_types) - - @property - def is_array_of_tables(self): - opening_bracket = next(token for i, token in enumerate(self._tokens) if token.type in _opening_bracket_types) - return opening_bracket.type == tokens.TYPE_OP_DOUBLE_SQUARE_LEFT_BRACKET - - @property - def names(self): - """ - Returns a sequence of string names making up this table header name. - """ - return self._names - - def has_name_prefix(self, names): - """ - Returns True if the header names is prefixed by the given sequence of names. - """ - for i, name in enumerate(names): - if self.names[i] != name: - return False - return True - - def serialized(self): - return ''.join(token.source_substring for token in self._tokens) - - def is_named(self, names): - """ - Returns True if the given name sequence matches the full name of this header. - """ - return tuple(names) == self.names - - def _validate_tokens(self, _tokens): - - opening_bracket_i = next((i for i, token in enumerate(_tokens) - if token.type in _opening_bracket_types), float('-inf')) - - if opening_bracket_i < 0: - raise InvalidElementError('Expected an opening bracket') - - _tokens = _tokens[opening_bracket_i+1:] - first_name_i = next((i for i, token in enumerate(_tokens) if token.type in _name_types), float('-inf')) - if first_name_i < 0: - raise InvalidElementError('Expected a table header name') - - _tokens = _tokens[first_name_i+1:] - - while True: - - next_dot_i = next((i for i, token in enumerate(_tokens) if token.type == tokens.TYPE_OPT_DOT), - float('-inf')) - if next_dot_i < 0: - break - - _tokens = _tokens[next_dot_i+1:] - - next_name_i = next((i for i, token in enumerate(_tokens) if token.type in _name_types), float('-inf')) - if next_name_i < 0: - raise InvalidElementError('Expected a name after the dot') - - _tokens = _tokens[next_name_i+1:] - - closing_bracket_i = next((i for i, token in enumerate(_tokens) if token.type in _closing_bracket_types), - float('-inf')) - - if closing_bracket_i < 0: - raise InvalidElementError('Expected a closing bracket') - - if _tokens[-1].type != tokens.TYPE_NEWLINE: - raise InvalidElementError('Must end with a newline') diff --git a/pipenv/patched/prettytoml/elements/test_array.py b/pipenv/patched/prettytoml/elements/test_array.py deleted file mode 100644 index 3ccc98b5..00000000 --- a/pipenv/patched/prettytoml/elements/test_array.py +++ /dev/null @@ -1,67 +0,0 @@ -import pytest -from prettytoml import lexer -from prettytoml.elements.array import ArrayElement -from prettytoml.elements.atomic import AtomicElement -from prettytoml.elements.metadata import PunctuationElement, WhitespaceElement, NewlineElement - - -def test_array_element(): - tokens = tuple(lexer.tokenize('[4, 8, 42, \n 23, 15]')) - assert len(tokens) == 17 - sub_elements = ( - PunctuationElement(tokens[:1]), - - AtomicElement(tokens[1:2]), - PunctuationElement(tokens[2:3]), - WhitespaceElement(tokens[3:4]), - - AtomicElement(tokens[4:5]), - PunctuationElement(tokens[5:6]), - WhitespaceElement(tokens[6:7]), - - AtomicElement(tokens[7:8]), - PunctuationElement(tokens[8:9]), - WhitespaceElement(tokens[9:10]), - NewlineElement(tokens[10:11]), - WhitespaceElement(tokens[11:12]), - - AtomicElement(tokens[12:13]), - PunctuationElement(tokens[13:14]), - - WhitespaceElement(tokens[14:15]), - AtomicElement(tokens[15:16]), - PunctuationElement(tokens[16:17]) - ) - - array_element = ArrayElement(sub_elements) - - # Test length - assert len(array_element) == 5 - - # Test getting a value - assert array_element[0] == 4 - assert array_element[1] == 8 - assert array_element[2] == 42 - assert array_element[3] == 23 - assert array_element[-1] == 15 - - # Test assignment with a negative index - array_element[-1] = 12 - - # Test persistence of formatting - assert '[4, 8, 42, \n 23, 12]' == array_element.serialized() - - # Test raises IndexError on invalid index - with pytest.raises(IndexError) as _: - print(array_element[5]) - - # Test appending a new value - array_element.append(77) - assert '[4, 8, 42, \n 23, 12, 77]' == array_element.serialized() - - # Test deleting a value - del array_element[3] - assert '[4, 8, 42, 12, 77]' == array_element.serialized() - - # Test primitive_value - assert [4, 8, 42, 12, 77] == array_element.primitive_value diff --git a/pipenv/patched/prettytoml/elements/test_atomic.py b/pipenv/patched/prettytoml/elements/test_atomic.py deleted file mode 100644 index 940ddd27..00000000 --- a/pipenv/patched/prettytoml/elements/test_atomic.py +++ /dev/null @@ -1,9 +0,0 @@ -from prettytoml import lexer -from prettytoml.elements.atomic import AtomicElement - - -def test_atomic_element(): - element = AtomicElement(tuple(lexer.tokenize(' \t 42 '))) - assert element.value == 42 - element.set(23) - assert element.serialized() == ' \t 23 ' diff --git a/pipenv/patched/prettytoml/elements/test_common.py b/pipenv/patched/prettytoml/elements/test_common.py deleted file mode 100644 index 9f5dd4c8..00000000 --- a/pipenv/patched/prettytoml/elements/test_common.py +++ /dev/null @@ -1,89 +0,0 @@ -from prettytoml import tokens, lexer -from prettytoml.elements import traversal -from prettytoml.elements.atomic import AtomicElement -from prettytoml.elements.metadata import NewlineElement, PunctuationElement, WhitespaceElement, CommentElement -from prettytoml.elements.table import TableElement -from prettytoml.elements.tableheader import TableHeaderElement - -atomic_token_types = ( - tokens.TYPE_INTEGER, - tokens.TYPE_FLOAT, - tokens.TYPE_BARE_STRING, - tokens.TYPE_STRING, - tokens.TYPE_LITERAL_STRING, - tokens.TYPE_MULTILINE_STRING, - tokens.TYPE_MULTILINE_LITERAL_STRING, -) - -punctuation_token_types = ( - tokens.TYPE_OPT_DOT, - tokens.TYPE_OP_CURLY_LEFT_BRACKET, - tokens.TYPE_OP_SQUARE_LEFT_BRACKET, - tokens.TYPE_OP_DOUBLE_SQUARE_LEFT_BRACKET, - tokens.TYPE_OP_SQUARE_RIGHT_BRACKET, - tokens.TYPE_OP_CURLY_RIGHT_BRACKET, - tokens.TYPE_OP_DOUBLE_SQUARE_RIGHT_BRACKET, - tokens.TYPE_OP_ASSIGNMENT, -) - -def primitive_token_to_primitive_element(token): - if token.type == tokens.TYPE_NEWLINE: - return NewlineElement((token,)) - elif token.type in atomic_token_types: - return AtomicElement((token,)) - elif token.type == tokens.TYPE_NEWLINE: - return NewlineElement((token,)) - elif token.type in punctuation_token_types: - return PunctuationElement((token,)) - elif token.type == tokens.TYPE_WHITESPACE: - return WhitespaceElement((token,)) - elif token.type == tokens.TYPE_COMMENT: - return CommentElement((token,)) - else: - raise RuntimeError("{} has no mapped primitive element".format(token)) - - -def primitive_tokens_to_primitive_elements(tokens): - return list(map(primitive_token_to_primitive_element, tokens)) - - -def dummy_file_elements(): - tokens_ = tuple(lexer.tokenize(""" -name = fawzy -another_name=another_fawzy - -[details] -id= 42 -section =fourth - -[[person]] -personname= lefawzy -dest=north - -[[person]] -dest=south -personname=lafawzy - -[details.extended] -number = 313 -type =complex""")) - - elements = \ - [TableElement(primitive_tokens_to_primitive_elements(tokens_[:12]))] + \ - [TableHeaderElement(tokens_[12:16])] + \ - [TableElement(primitive_tokens_to_primitive_elements(tokens_[16:25]))] + \ - [TableHeaderElement(tokens_[25:31])] + \ - [TableElement(primitive_tokens_to_primitive_elements(tokens_[31:39]))] + \ - [TableHeaderElement(tokens_[39:45])] + \ - [TableElement(primitive_tokens_to_primitive_elements(tokens_[45:53]))] + \ - [TableHeaderElement(tokens_[53:60])] + \ - [TableElement(primitive_tokens_to_primitive_elements(tokens_[60:]))] - - return elements - - -class DummyFile(traversal.TraversalMixin): - - @property - def elements(self): - return dummy_file_elements() diff --git a/pipenv/patched/prettytoml/elements/test_factory.py b/pipenv/patched/prettytoml/elements/test_factory.py deleted file mode 100644 index 08a42883..00000000 --- a/pipenv/patched/prettytoml/elements/test_factory.py +++ /dev/null @@ -1,22 +0,0 @@ -from collections import OrderedDict -from prettytoml.elements import factory -from prettytoml.elements.array import ArrayElement -from prettytoml.elements.atomic import AtomicElement -from prettytoml.elements.inlinetable import InlineTableElement - - -def test_creating_elements(): - - atomic = factory.create_element(42) - assert isinstance(atomic, AtomicElement) - assert atomic.value == 42 - - seq = factory.create_element(['a', 'p', 'p', 'l', 'e']) - assert isinstance(seq, ArrayElement) - assert seq.serialized() == '["a", "p", "p", "l", "e"]' - assert ''.join(seq.primitive_value) == 'apple' - - mapping = factory.create_element(OrderedDict((('one', 1), ('two', 2)))) - assert isinstance(mapping, InlineTableElement) - assert mapping.serialized() == '{one = 1, two = 2}' - diff --git a/pipenv/patched/prettytoml/elements/test_inlinetable.py b/pipenv/patched/prettytoml/elements/test_inlinetable.py deleted file mode 100644 index 3c663873..00000000 --- a/pipenv/patched/prettytoml/elements/test_inlinetable.py +++ /dev/null @@ -1,52 +0,0 @@ -from prettytoml import lexer -from prettytoml.elements.atomic import AtomicElement -from prettytoml.elements.inlinetable import InlineTableElement -from prettytoml.elements.metadata import PunctuationElement, WhitespaceElement - - -def test_inline_table(): - tokens = tuple(lexer.tokenize('{ name= "first", id=42}')) - - elements = ( - PunctuationElement(tokens[:1]), - WhitespaceElement(tokens[1:2]), - AtomicElement(tokens[2:3]), - PunctuationElement(tokens[3:4]), - WhitespaceElement(tokens[4:5]), - AtomicElement(tokens[5:6]), - PunctuationElement(tokens[6:7]), - WhitespaceElement(tokens[7:8]), - AtomicElement(tokens[8:9]), - PunctuationElement(tokens[9:10]), - AtomicElement(tokens[10:11]), - PunctuationElement(tokens[11:12]) - ) - - table = InlineTableElement(elements) - - assert table['name'] == 'first' - assert table['id'] == 42 - - table['name'] = 'fawzy' - table['nickname'] = 'nickfawzy' - - assert set(table.items()) == {('name', 'fawzy'), ('id', 42), ('nickname', 'nickfawzy')} - - assert table.serialized() == '{ name= "fawzy", id=42, nickname = "nickfawzy"}' - - del table['name'] - - assert table.serialized() == '{ id=42, nickname = "nickfawzy"}' - - del table['nickname'] - - assert table.serialized() == '{ id=42}' - - del table['id'] - - assert table.serialized() == '{ }' - - table['item1'] = 11 - table['item2'] = 22 - - assert table.serialized() == '{ item1 = 11, item2 = 22}' diff --git a/pipenv/patched/prettytoml/elements/test_metadata.py b/pipenv/patched/prettytoml/elements/test_metadata.py deleted file mode 100644 index 6e49fedd..00000000 --- a/pipenv/patched/prettytoml/elements/test_metadata.py +++ /dev/null @@ -1,25 +0,0 @@ -from prettytoml import lexer -from prettytoml.elements.metadata import WhitespaceElement, NewlineElement, CommentElement, PunctuationElement - - -def test_whitespace_element(): - element = WhitespaceElement(tuple(lexer.tokenize(' \t '))) - assert element.serialized() == ' \t ' - - -def test_newline_element(): - element = NewlineElement(tuple(lexer.tokenize('\n\n\n'))) - assert element.serialized() == '\n\n\n' - - -def test_comment_element(): - element = CommentElement(tuple(lexer.tokenize('# This is my insightful remark\n'))) - assert element.serialized() == '# This is my insightful remark\n' - - -def test_punctuation_element(): - PunctuationElement(tuple(lexer.tokenize('['))) - PunctuationElement(tuple(lexer.tokenize('[['))) - PunctuationElement(tuple(lexer.tokenize('.'))) - PunctuationElement(tuple(lexer.tokenize(']'))) - PunctuationElement(tuple(lexer.tokenize(']]'))) diff --git a/pipenv/patched/prettytoml/elements/test_table.py b/pipenv/patched/prettytoml/elements/test_table.py deleted file mode 100644 index ef0dba81..00000000 --- a/pipenv/patched/prettytoml/elements/test_table.py +++ /dev/null @@ -1,59 +0,0 @@ -from prettytoml import lexer -from prettytoml.elements.atomic import AtomicElement -from prettytoml.elements.metadata import WhitespaceElement, PunctuationElement, NewlineElement, CommentElement -from prettytoml.elements.table import TableElement - - -def test_table(): - - initial_toml = """name = "first" -id=42 # My id - - -""" - - tokens = tuple(lexer.tokenize(initial_toml)) - - elements = ( - AtomicElement(tokens[:1]), - WhitespaceElement(tokens[1:2]), - PunctuationElement(tokens[2:3]), - WhitespaceElement(tokens[3:4]), - AtomicElement(tokens[4:5]), - NewlineElement(tokens[5:6]), - - AtomicElement(tokens[6:7]), - PunctuationElement(tokens[7:8]), - AtomicElement(tokens[8:9]), - WhitespaceElement(tokens[9:10]), - CommentElement(tokens[10:12]), - - NewlineElement(tokens[12:13]), - NewlineElement(tokens[13:14]), - ) - - table = TableElement(elements) - - assert set(table.items()) == {('name', 'first'), ('id', 42)} - - assert table['name'] == 'first' - assert table['id'] == 42 - - table['relation'] = 'another' - - assert set(table.items()) == {('name', 'first'), ('id', 42), ('relation', 'another')} - - table['name'] = 'fawzy' - - assert set(table.items()) == {('name', 'fawzy'), ('id', 42), ('relation', 'another')} - - expected_toml = """name = "fawzy" -id=42 # My id -relation = "another" - - -""" - - assert table.serialized() == expected_toml - - diff --git a/pipenv/patched/prettytoml/elements/test_tableheader.py b/pipenv/patched/prettytoml/elements/test_tableheader.py deleted file mode 100644 index 67b1ded4..00000000 --- a/pipenv/patched/prettytoml/elements/test_tableheader.py +++ /dev/null @@ -1,12 +0,0 @@ -from prettytoml import lexer -from prettytoml.elements.tableheader import TableHeaderElement - - -def test_tableheader(): - tokens = tuple(lexer.tokenize('\n\t [[personal. information.details]] \n')) - element = TableHeaderElement(tokens) - - assert element.is_array_of_tables - assert ('personal', 'information', 'details') == element.names - - assert element.has_name_prefix(('personal', 'information')) diff --git a/pipenv/patched/prettytoml/elements/test_traversal.py b/pipenv/patched/prettytoml/elements/test_traversal.py deleted file mode 100644 index 2f002571..00000000 --- a/pipenv/patched/prettytoml/elements/test_traversal.py +++ /dev/null @@ -1,18 +0,0 @@ -from prettytoml.elements.test_common import DummyFile - - -def test_traversal(): - dummy_file = DummyFile() - - assert dummy_file._find_following_table_header(-1) == 1 - assert dummy_file._find_following_table_header(1) == 3 - assert dummy_file._find_following_table_header(3) == 5 - assert dummy_file._find_following_table_header(5) == 7 - assert dummy_file._find_following_table_header(7) < 0 - - assert dummy_file._find_preceding_table(30) == 8 - assert dummy_file._find_preceding_table(8) == 6 - assert dummy_file._find_preceding_table(6) == 4 - assert dummy_file._find_preceding_table(4) == 2 - assert dummy_file._find_preceding_table(2) == 0 - assert dummy_file._find_preceding_table(0) < 0 diff --git a/pipenv/patched/prettytoml/elements/traversal/__init__.py b/pipenv/patched/prettytoml/elements/traversal/__init__.py deleted file mode 100644 index c93506e2..00000000 --- a/pipenv/patched/prettytoml/elements/traversal/__init__.py +++ /dev/null @@ -1,175 +0,0 @@ -from prettytoml import tokens -from prettytoml.elements import common -from prettytoml.elements.metadata import PunctuationElement, NewlineElement -from prettytoml.elements.traversal import predicates - - -class TraversalMixin: - """ - A mix-in that provides convenient sub-element traversal to any class with - an `elements` member that is a sequence of Element instances - """ - - def __find_following_element(self, index, predicate): - """ - Finds and returns the index of element in self.elements that evaluates the given predicate to True - and whose index is higher than the given index, or returns -Infinity on failure. - """ - return find_following(self.elements, predicate, index) - - def __find_preceding_element(self, index, predicate): - """ - Finds and returns the index of the element in self.elements that evaluates the given predicate to True - and whose index is lower than the given index. - """ - i = find_previous(self.elements, predicate, index) - if i == float('inf'): - return float('-inf') - return i - - def __must_find_following_element(self, predicate): - """ - Finds and returns the index to the element in self.elements that evaluatest the predicate to True, or raises - an error. - """ - i = self.__find_following_element(-1, predicate) - if i < 0: - raise RuntimeError('Could not find non-optional element') - return i - - def _enumerate_non_metadata_sub_elements(self): - """ - Returns a sequence of of (index, sub_element) of the non-metadata sub-elements. - """ - return ((i, element) for i, element in enumerate(self.elements) if element.type != common.TYPE_METADATA) - - def _find_preceding_comma(self, index): - """ - Returns the index of the preceding comma element to the given index, or -Infinity. - """ - return self.__find_preceding_element(index, predicates.op_comma) - - def _find_following_comma(self, index): - """ - Returns the index of the following comma element after the given index, or -Infinity. - """ - def predicate(element): - return isinstance(element, PunctuationElement) and element.token.type == tokens.TYPE_OP_COMMA - return self.__find_following_element(index, predicate) - - def _find_following_newline(self, index): - """ - Returns the index of the following newline element after the given index, or -Infinity. - """ - return self.__find_following_element(index, lambda e: isinstance(e, NewlineElement)) - - def _find_following_comment(self, index): - """ - Returns the index of the following comment element after the given index, or -Infinity. - """ - return self.__find_following_element(index, predicates.comment) - - def _find_following_line_terminator(self, index): - """ - Returns the index of the following comment or newline element after the given index, or -Infinity. - """ - following_comment = self._find_following_comment(index) - following_newline = self._find_following_newline(index) - - if following_comment == float('-inf'): - return following_newline - if following_newline == float('-inf'): - return following_comment - - if following_newline < following_comment: - return following_newline - else: - return following_comment - - def _find_preceding_newline(self, index): - """ - Returns the index of the preceding newline element to the given index, or -Infinity. - """ - return self.__find_preceding_element(index, predicates.newline) - - def _find_following_non_metadata(self, index): - """ - Returns the index to the following non-metadata element after the given index, or -Infinity. - """ - return self.__find_following_element(index, predicates.non_metadata) - - def _find_closing_square_bracket(self): - """ - Returns the index to the closing square bracket, or raises an Error. - """ - - return self.__must_find_following_element(predicates.closing_square_bracket) - - def _find_following_opening_square_bracket(self, index): - """ - Returns the index to the opening square bracket, or -Infinity. - """ - return self.__find_following_element(index, predicates.opening_square_bracket) - - def _find_following_closing_square_bracket(self, index): - """ - Returns the index to the closing square bracket, or -Infinity. - """ - return self.__find_following_element(index, predicates.closing_square_bracket) - - def _find_following_table(self, index): - """ - Returns the index to the next TableElement after the specified index, or -Infinity. - """ - return self.__find_following_element(index, predicates.table) - - def _find_preceding_table(self, index): - """ - Returns the index to the preceding TableElement to the specified index, or -Infinity. - """ - return self.__find_preceding_element(index,predicates.table) - - def _find_closing_curly_bracket(self): - """ - Returns the index to the closing curly bracket, or raises an Error. - """ - def predicate(element): - return isinstance(element, PunctuationElement) and element.token.type == tokens.TYPE_OP_CURLY_RIGHT_BRACKET - return self.__must_find_following_element(predicate) - - def _find_following_table_header(self, index): - """ - Returns the index to the table header after the given element index, or -Infinity. - """ - return self.__find_following_element(index, predicates.table_header) - - -def find_following(element_seq, predicate, index=None): - """ - Finds and returns the index of the next element fulfilling the specified predicate after the specified - index, or -Infinity. - - Starts searching linearly from the start_from index. - """ - - if isinstance(index, (int, float)) and index < 0: - index = None - - for i, element in tuple(enumerate(element_seq))[index+1 if index is not None else index:]: - if predicate(element): - return i - return float('-inf') - - -def find_previous(element_seq, predicate, index=None): - """ - Finds and returns the index of the previous element fulfilling the specified predicate preceding to the specified - index, or Infinity. - """ - if isinstance(index, (int, float)) and index >= len(element_seq): - index = None - - for i, element in reversed(tuple(enumerate(element_seq))[:index]): - if predicate(element): - return i - return float('inf') diff --git a/pipenv/patched/prettytoml/elements/traversal/predicates.py b/pipenv/patched/prettytoml/elements/traversal/predicates.py deleted file mode 100644 index f18616bf..00000000 --- a/pipenv/patched/prettytoml/elements/traversal/predicates.py +++ /dev/null @@ -1,48 +0,0 @@ - -""" - The following predicates can be used in the traversal functions directly. -""" - -from ..atomic import AtomicElement -from ..metadata import PunctuationElement, CommentElement, NewlineElement, WhitespaceElement -from prettytoml import tokens -from .. import common - - -atomic = lambda e: isinstance(e, AtomicElement) - - -op_assignment = lambda e: isinstance(e, PunctuationElement) and e.token.type == tokens.TYPE_OP_ASSIGNMENT - - -op_comma = lambda e: isinstance(e, PunctuationElement) and e.token.type == tokens.TYPE_OP_COMMA - - -comment = lambda e: isinstance(e, CommentElement) - - -newline = lambda e: isinstance(e, NewlineElement) - - -non_metadata = lambda e: e.type != common.TYPE_METADATA - - -closing_square_bracket = \ - lambda e: isinstance(e, PunctuationElement) and e.token.type == tokens.TYPE_OP_SQUARE_RIGHT_BRACKET - - -opening_square_bracket = \ - lambda e: isinstance(e, PunctuationElement) and e.token.type == tokens.TYPE_OP_SQUARE_LEFT_BRACKET - - -def table(e): - from ..table import TableElement - return isinstance(e, TableElement) - - -def table_header(e): - from prettytoml.elements.tableheader import TableHeaderElement - return isinstance(e, TableHeaderElement) - - -whitespace = lambda e: isinstance(e, WhitespaceElement) diff --git a/pipenv/patched/prettytoml/errors.py b/pipenv/patched/prettytoml/errors.py deleted file mode 100644 index 23e69eb5..00000000 --- a/pipenv/patched/prettytoml/errors.py +++ /dev/null @@ -1,32 +0,0 @@ - - -class TOMLError(Exception): - """ - All errors raised by this module are descendants of this type. - """ - - -class InvalidTOMLFileError(TOMLError): - pass - - -class NoArrayFoundError(TOMLError): - """ - An array of tables was requested but none exist by the given name. - """ - - -class InvalidValueError(TOMLError): - pass - - -class DuplicateKeysError(TOMLError): - """ - Duplicate keys detected in the parsed file. - """ - - -class DuplicateTablesError(TOMLError): - """ - Duplicate tables detected in the parsed file. - """ diff --git a/pipenv/patched/prettytoml/lexer/__init__.py b/pipenv/patched/prettytoml/lexer/__init__.py deleted file mode 100644 index da32963a..00000000 --- a/pipenv/patched/prettytoml/lexer/__init__.py +++ /dev/null @@ -1,123 +0,0 @@ - -""" -A regular expression based Lexer/tokenizer for TOML. -""" - -from collections import namedtuple -import re -from prettytoml import tokens -from prettytoml.errors import TOMLError - -TokenSpec = namedtuple('TokenSpec', ('type', 're')) - -# Specs of all the valid tokens -_LEXICAL_SPECS = ( - TokenSpec(tokens.TYPE_COMMENT, re.compile(r'^(#.*)\n')), - TokenSpec(tokens.TYPE_STRING, re.compile(r'^("(([^"]|\\")+?[^\\]|([^"]|\\")|)")')), # Single line only - TokenSpec(tokens.TYPE_MULTILINE_STRING, re.compile(r'^(""".*?""")', re.DOTALL)), - TokenSpec(tokens.TYPE_LITERAL_STRING, re.compile(r"^('.*?')")), - TokenSpec(tokens.TYPE_MULTILINE_LITERAL_STRING, re.compile(r"^('''.*?''')", re.DOTALL)), - TokenSpec(tokens.TYPE_BARE_STRING, re.compile(r'^([A-Za-z0-9_-]+)')), - TokenSpec(tokens.TYPE_DATE, re.compile( - r'^([0-9]{4}-[0-9]{2}-[0-9]{2}(T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]*)?)?(([zZ])|((\+|-)[0-9]{2}:[0-9]{2}))?)')), - TokenSpec(tokens.TYPE_WHITESPACE, re.compile(r'^( |\t)', re.DOTALL)), - TokenSpec(tokens.TYPE_INTEGER, re.compile(r'^(((\+|-)[0-9_]+)|([0-9][0-9_]*))')), - TokenSpec(tokens.TYPE_FLOAT, - re.compile(r'^((((\+|-)[0-9_]+)|([1-9][0-9_]*))(\.[0-9_]+)?([eE](\+|-)?[0-9_]+)?)')), - TokenSpec(tokens.TYPE_BOOLEAN, re.compile(r'^(true|false)')), - TokenSpec(tokens.TYPE_OP_SQUARE_LEFT_BRACKET, re.compile(r'^(\[)')), - TokenSpec(tokens.TYPE_OP_SQUARE_RIGHT_BRACKET, re.compile(r'^(\])')), - TokenSpec(tokens.TYPE_OP_CURLY_LEFT_BRACKET, re.compile(r'^(\{)')), - TokenSpec(tokens.TYPE_OP_CURLY_RIGHT_BRACKET, re.compile(r'^(\})')), - TokenSpec(tokens.TYPE_OP_ASSIGNMENT, re.compile(r'^(=)')), - TokenSpec(tokens.TYPE_OP_COMMA, re.compile(r'^(,)')), - TokenSpec(tokens.TYPE_OP_DOUBLE_SQUARE_LEFT_BRACKET, re.compile(r'^(\[\[)')), - TokenSpec(tokens.TYPE_OP_DOUBLE_SQUARE_RIGHT_BRACKET, re.compile(r'^(\]\])')), - TokenSpec(tokens.TYPE_OPT_DOT, re.compile(r'^(\.)')), - TokenSpec(tokens.TYPE_NEWLINE, re.compile('^(\n|\r\n)')), -) - - -def _next_token_candidates(source): - matches = [] - for token_spec in _LEXICAL_SPECS: - match = token_spec.re.search(source) - if match: - matches.append(tokens.Token(token_spec.type, match.group(1))) - return matches - - -def _choose_from_next_token_candidates(candidates): - - if len(candidates) == 1: - return candidates[0] - elif len(candidates) > 1: - # Return the maximal-munch with ties broken by natural order of token type. - maximal_munch_length = max(len(token.source_substring) for token in candidates) - maximal_munches = [token for token in candidates if len(token.source_substring) == maximal_munch_length] - return sorted(maximal_munches)[0] # Return the first in sorting by priority - - -def _munch_a_token(source): - """ - Munches a single Token instance if it could recognize one at the beginning of the - given source text, or None if no token type could be recognized. - """ - candidates = _next_token_candidates(source) - return _choose_from_next_token_candidates(candidates) - - -class LexerError(TOMLError): - - def __init__(self, message): - self._message = message - - def __repr__(self): - return self._message - - def __str__(self): - return self._message - - -def tokenize(source, is_top_level=False): - """ - Tokenizes the input TOML source into a stream of tokens. - - If is_top_level is set to True, will make sure that the input source has a trailing newline character - before it is tokenized. - - Raises a LexerError when it fails recognize another token while not at the end of the source. - """ - - # Newlines are going to be normalized to UNIX newlines. - source = source.replace('\r\n', '\n') - - if is_top_level and source and source[-1] != '\n': - source += '\n' - - next_row = 1 - next_col = 1 - next_index = 0 - - while next_index < len(source): - - new_token = _munch_a_token(source[next_index:]) - - if not new_token: - raise LexerError("failed to read the next token at ({}, {}): {}".format( - next_row, next_col, source[next_index:])) - - # Set the col and row on the new token - new_token = tokens.Token(new_token.type, new_token.source_substring, next_col, next_row) - - # Advance the index, row and col count - next_index += len(new_token.source_substring) - for c in new_token.source_substring: - if c == '\n': - next_row += 1 - next_col = 1 - else: - next_col += 1 - - yield new_token - diff --git a/pipenv/patched/prettytoml/lexer/test_lexer.py b/pipenv/patched/prettytoml/lexer/test_lexer.py deleted file mode 100644 index df10b46d..00000000 --- a/pipenv/patched/prettytoml/lexer/test_lexer.py +++ /dev/null @@ -1,153 +0,0 @@ -# -*- coding: utf-8 -*- - -from prettytoml.lexer import _munch_a_token -from prettytoml.lexer import * - -# A mapping from token types to a sequence of pairs of (source_text, expected_matched_text) -valid_tokens = { - tokens.TYPE_COMMENT: ( - ( - '# My very insightful comment about the state of the universe\n# And now for something completely different!', - '# My very insightful comment about the state of the universe', - ), - ), - tokens.TYPE_STRING: ( - ('"a valid hug3 text" "some other string" = 42', '"a valid hug3 text"'), - ( - r'"I\'m a string. \"You can quote me\". Name\tJos\u00E9\nLocation\tSF." "some other string" = 42', - r'"I\'m a string. \"You can quote me\". Name\tJos\u00E9\nLocation\tSF."' - ), - ('"ʎǝʞ" key', '"ʎǝʞ"'), - ('""', '""'), - ('"t"', '"t"'), - ), - tokens.TYPE_MULTILINE_STRING: ( - ('"""\nRoses are red\nViolets are blue""" """other text"""', '"""\nRoses are red\nViolets are blue"""'), - ), - tokens.TYPE_LITERAL_STRING: ( - (r"'This is \ \n a \\ literal string' 'another \ literal string'", r"'This is \ \n a \\ literal string'"), - ), - tokens.TYPE_MULTILINE_LITERAL_STRING: ( - ( - "'''\nThe first newline is\ntrimmed in raw strings.\n All other whitespace\n is preserved.\n''' '''some other\n\n\t string'''", - "'''\nThe first newline is\ntrimmed in raw strings.\n All other whitespace\n is preserved.\n'''" - ), - ), - tokens.TYPE_DATE: ( - ('1979-05-27 5345', '1979-05-27'), - ('1979-05-27T07:32:00Z something', '1979-05-27T07:32:00Z'), - ('1979-05-27T00:32:00-07:00 ommm', '1979-05-27T00:32:00-07:00'), - ('1979-05-27T00:32:00.999999-07:00 2346', '1979-05-27T00:32:00.999999-07:00'), - ), - tokens.TYPE_WHITESPACE: ( - (' \t\n \r some_text', ' '), - ), - tokens.TYPE_INTEGER: ( - ('+99 "number"', "+99"), - ('42 fwfwef', "42"), - ('-17 fh34g34g', "-17"), - ('5_349_221 apples', "5_349_221"), - ('-1_2_3_4_5 steps', '-1_2_3_4_5') - ), - tokens.TYPE_FLOAT: ( - ('1.0 fwef', '1.0'), - ('3.1415 g4g', '3.1415'), - ('-0.01 433re', '-0.01'), - ('5e+2_2 ersdvf', '5e+2_2'), - ('1e6 ewe23', '1e6'), - ('-2E-2.2 3 rf23', '-2E-2'), - ('6.626e-34 +234f', '6.626e-34'), - ('9_224_617.445_991_228_313 f1ewer 23f4h = nonesense', '9_224_617.445_991_228_313'), - ('1e1_000 2346f,ef2!!', '1e1_000'), - ), - tokens.TYPE_BOOLEAN: ( - ('false business = true', 'false'), - ('true true', 'true'), - ), - tokens.TYPE_OP_SQUARE_LEFT_BRACKET: ( - ('[table_name]', '['), - ), - tokens.TYPE_OP_SQUARE_RIGHT_BRACKET: ( - (']\nbusiness = awesome', ']'), - ), - tokens.TYPE_OP_CURLY_LEFT_BRACKET: ( - ('{item_exists = no}', '{'), - ), - tokens.TYPE_OP_CURLY_RIGHT_BRACKET: ( - ('} moving on', '}'), - ), - tokens.TYPE_OP_COMMA: ( - (',item2,item4', ','), - ), - tokens.TYPE_OP_ASSIGNMENT: ( - ('== 42', '='), - ), - tokens.TYPE_OP_DOUBLE_SQUARE_LEFT_BRACKET: ( - ('[[array.of.tables]]', '[['), - ), - tokens.TYPE_OP_DOUBLE_SQUARE_RIGHT_BRACKET: ( - (']] item=3', ']]'), - ), - tokens.TYPE_BARE_STRING: ( - ('key another', 'key'), - ('bare_key 2fews', 'bare_key'), - ('bare-key kfcw', 'bare-key'), - ), - tokens.TYPE_OPT_DOT: ( - ('."another key"', '.'), - ('.subname', '.'), - ), - tokens.TYPE_NEWLINE: ( - ('\n\r \n', '\n'), - ) -} - -# A mapping from a token type to a sequence of (source, matched_text) pairs that shouldn't result from consuming the -# source text. -invalid_tokens = { - tokens.TYPE_INTEGER: ( - ('_234_423', ''), - ('0446234234', ''), - ), - tokens.TYPE_STRING: ( - ('"""', '"""'), - ), - tokens.TYPE_BOOLEAN: ( - ('True', 'True'), - ('True', 'true'), - ), - tokens.TYPE_FLOAT: ( - ('', ''), - ) -} - - -def test_valid_tokenizing(): - for token_type in valid_tokens: - for (source, expected_match) in valid_tokens[token_type]: - - token = _munch_a_token(source) - assert token, "Failed to tokenize: {}\nExpected: {}\nOut of: {}\nGot nothing!".format( - token_type, expected_match, source) - - assert token.type == token_type, \ - "Expected type: {}\nOut of: {}\nThat matched: {}\nOf type: {}".format( - token_type, source, token.source_substring, token.type) - assert token.source_substring == expected_match - - -def test_invalid_tokenizing(): - for token_type in invalid_tokens: - for source, expected_match in invalid_tokens[token_type]: - token = _munch_a_token(source) - if token: - assert not (token.type == token_type and token.source_substring == expected_match) - - -def test_token_type_order(): - type_a = tokens.TokenType('a', 5, is_metadata=False) - type_b = tokens.TokenType('b', 0, is_metadata=False) - type_c = tokens.TokenType('c', 3, is_metadata=False) - - assert type_b < type_c < type_a - assert type_a > type_c > type_b diff --git a/pipenv/patched/prettytoml/parser/__init__.py b/pipenv/patched/prettytoml/parser/__init__.py deleted file mode 100644 index 4cf600c0..00000000 --- a/pipenv/patched/prettytoml/parser/__init__.py +++ /dev/null @@ -1,34 +0,0 @@ - -""" - A parser for TOML tokens into TOML elements. -""" - - -from prettytoml.parser.errors import ParsingError - - -def parse_tokens(tokens): - """ - Parses the given token sequence into a sequence of top-level TOML elements. - - Raises ParserError on invalid TOML input. - """ - from .tokenstream import TokenStream - return _parse_token_stream(TokenStream(tokens)) - - -def _parse_token_stream(token_stream): - """ - Parses the given token_stream into a sequence of top-level TOML elements. - - Raises ParserError on invalid input TOML. - """ - from .parser import toml_file_elements - from .elementsanitizer import sanitize - - elements, pending = toml_file_elements(token_stream) - - if not pending.at_end: - raise ParsingError('Failed to parse line {}'.format(pending.head.row)) - - return sanitize(elements) diff --git a/pipenv/patched/prettytoml/parser/elementsanitizer.py b/pipenv/patched/prettytoml/parser/elementsanitizer.py deleted file mode 100644 index bec4893a..00000000 --- a/pipenv/patched/prettytoml/parser/elementsanitizer.py +++ /dev/null @@ -1,58 +0,0 @@ -from prettytoml import elements -from prettytoml.elements.table import TableElement -from prettytoml.elements.tableheader import TableHeaderElement -from prettytoml.errors import InvalidTOMLFileError -from prettytoml.util import PeekableIterator - - -def sanitize(_elements): - """ - Finds TableHeader elements that are not followed by TableBody elements and inserts empty TableElement - right after those. - """ - - output = list(_elements) - - def find_next_table_header(after=-1): - return next((i for (i, element) in enumerate(output) - if i > after and isinstance(element, TableHeaderElement)), float('-inf')) - - def find_next_table_body(after=-1): - return next((i for (i, element) in enumerate(output) - if i > after and isinstance(element, TableElement)), float('-inf')) - - next_table_header_i = find_next_table_header() - while next_table_header_i >= 0: - - following_table_header_i = find_next_table_header(next_table_header_i) - following_table_body_i = find_next_table_body(next_table_header_i) - - if (following_table_body_i < 0) or \ - (following_table_header_i >= 0 and (following_table_header_i < following_table_body_i)): - output.insert(next_table_header_i+1, TableElement(tuple())) - - next_table_header_i = find_next_table_header(next_table_header_i) - - return output - - -def validate_sanitized(_elements): - - # Non-metadata elements must start with an optional TableElement, followed by - # zero or more (TableHeaderElement, TableElement) pairs. - - if not _elements: - return - - it = PeekableIterator(e for e in _elements if e.type != elements.TYPE_METADATA) - - if isinstance(it.peek(), TableElement): - it.next() - - while it.peek(): - if not isinstance(it.peek(), TableHeaderElement): - raise InvalidTOMLFileError - it.next() - if not isinstance(it.peek(), TableElement): - raise InvalidTOMLFileError - it.next() diff --git a/pipenv/patched/prettytoml/parser/errors.py b/pipenv/patched/prettytoml/parser/errors.py deleted file mode 100644 index eca12f32..00000000 --- a/pipenv/patched/prettytoml/parser/errors.py +++ /dev/null @@ -1,17 +0,0 @@ -from prettytoml.errors import TOMLError - - -class ParsingError(TOMLError): - - def __init__(self, message='', token=None): - self.message = message - self.token = token - - def __repr__(self): - if self.message and self.token: - return "{} at row {} and col {}".format(self.message, self.token.row, self.token.col) - else: - return self.message - - def __str__(self): - return repr(self) diff --git a/pipenv/patched/prettytoml/parser/parser.py b/pipenv/patched/prettytoml/parser/parser.py deleted file mode 100644 index e61c0db5..00000000 --- a/pipenv/patched/prettytoml/parser/parser.py +++ /dev/null @@ -1,376 +0,0 @@ - -""" - A Recursive Descent implementation of a lexical parser for TOML. - - Grammar: - -------- - - Newline -> NEWLINE - Comment -> COMMENT Newline - LineTerminator -> Comment | Newline - Space -> WHITESPACE Space | WHITESPACE | EMPTY - TableHeader -> Space [ Space TableHeaderName Space ] Space LineTerminator | - Space [[ Space TableHeaderName Space ]] Space LineTerminator - TableHeaderName -> STRING Space '.' Space TableHeaderName | STRING - Atomic -> STRING | INTEGER | FLOAT | DATE | BOOLEAN - - Array -> '[' Space ArrayInternal Space ']' | '[' Space ArrayInternal Space LineTerminator Space ']' - ArrayInternal -> LineTerminator Space ArrayInternal | Value Space ',' Space LineTerminator Space ArrayInternal | - Value Space ',' Space ArrayInternal | LineTerminator | Value | EMPTY - - InlineTable -> '{' Space InlineTableInternal Space '}' - InlineTableKeyValuePair = STRING Space '=' Space Value - InlineTableInternal -> InlineTableKeyValuePair Space ',' Space InlineTableInternal | - InlineTableKeyValuePair | Empty - - Value -> Atomic | InlineTable | Array - KeyValuePair -> Space STRING Space '=' Space Value Space LineTerminator - - TableBody -> KeyValuePair TableBody | EmptyLine TableBody | EmptyLine | KeyValuePair - - EmptyLine -> Space LineTerminator - FileEntry -> TableHeader | TableBody - - TOMLFileElements -> FileEntry TOMLFileElements | FileEntry | EmptyLine | EMPTY -""" - -from prettytoml import tokens -from prettytoml.elements.array import ArrayElement -from prettytoml.elements.atomic import AtomicElement -from prettytoml.elements.inlinetable import InlineTableElement -from prettytoml.elements.metadata import NewlineElement, CommentElement, WhitespaceElement, PunctuationElement -from prettytoml.elements.table import TableElement -from prettytoml.elements.tableheader import TableHeaderElement - -from prettytoml.parser.recdesc import capture_from -from prettytoml.parser.errors import ParsingError -from prettytoml.parser.tokenstream import TokenStream - -""" - Non-terminals are represented as functions which return (RESULT, pending_token_stream), or raise ParsingError. -""" - - -def token(token_type): - def factory(ts): - t = ts.head - if t.type != token_type: - raise ParsingError('Expected a token of type {}'.format(token_type)) - return t, ts.tail - return factory - - -def newline_element(token_stream): - """ - Returns NewlineElement, pending_token_stream or raises ParsingError. - """ - captured = capture_from(token_stream).find(token(tokens.TYPE_NEWLINE)) - return NewlineElement(captured.value()), captured.pending_tokens - - -def comment_tokens(ts1): - c1 = capture_from(ts1).find(token(tokens.TYPE_COMMENT)).and_find(token(tokens.TYPE_NEWLINE)) - return c1.value(), c1.pending_tokens - - -def comment_element(token_stream): - """ - Returns CommentElement, pending_token_stream or raises ParsingError. - """ - captured = capture_from(token_stream).find(comment_tokens) - return CommentElement(captured.value()), captured.pending_tokens - - -def line_terminator_tokens(token_stream): - captured = capture_from(token_stream).find(comment_tokens).or_find(token(tokens.TYPE_NEWLINE)) - return captured.value(), captured.pending_tokens - - -def line_terminator_element(token_stream): - captured = capture_from(token_stream).find(comment_element).or_find(newline_element) - return captured.value('Expected a comment or a newline')[0], captured.pending_tokens - - -def zero_or_more_tokens(token_type): - - def factory(token_stream): - def more(ts): - c = capture_from(ts).find(token(token_type)).and_find(zero_or_more_tokens(token_type)) - return c.value(), c.pending_tokens - - def two(ts): - c = capture_from(ts).find(token(tokens.TYPE_WHITESPACE)) - return c.value(), c.pending - - def zero(ts): - return tuple(), ts - - captured = capture_from(token_stream).find(more).or_find(two).or_find(zero) - return captured.value(), captured.pending_tokens - - return factory - - -def space_element(token_stream): - captured = capture_from(token_stream).find(zero_or_more_tokens(tokens.TYPE_WHITESPACE)) - return WhitespaceElement([t for t in captured.value() if t]), captured.pending_tokens - - -def string_token(token_stream): - captured = capture_from(token_stream).\ - find(token(tokens.TYPE_BARE_STRING)).\ - or_find(token(tokens.TYPE_STRING)).\ - or_find(token(tokens.TYPE_LITERAL_STRING)).\ - or_find(token(tokens.TYPE_MULTILINE_STRING)).\ - or_find(token(tokens.TYPE_MULTILINE_LITERAL_STRING)) - return captured.value('Expected a string'), captured.pending_tokens - - -def string_element(token_stream): - captured = capture_from(token_stream).find(string_token) - return AtomicElement(captured.value()), captured.pending_tokens - - -def table_header_name_tokens(token_stream): - - def one(ts): - c = capture_from(ts).\ - find(string_token).\ - and_find(zero_or_more_tokens(tokens.TYPE_WHITESPACE)).\ - and_find(token(tokens.TYPE_OPT_DOT)).\ - and_find(zero_or_more_tokens(tokens.TYPE_WHITESPACE)).\ - and_find(table_header_name_tokens) - return c.value(), c.pending_tokens - - captured = capture_from(token_stream).find(one).or_find(string_token) - return captured.value(), captured.pending_tokens - - -def table_header_element(token_stream): - - def single(ts1): - c1 = capture_from(ts1).\ - find(zero_or_more_tokens(tokens.TYPE_WHITESPACE)).\ - and_find(token(tokens.TYPE_OP_SQUARE_LEFT_BRACKET)).\ - and_find(zero_or_more_tokens(tokens.TYPE_WHITESPACE)).\ - and_find(table_header_name_tokens).\ - and_find(zero_or_more_tokens(tokens.TYPE_WHITESPACE)).\ - and_find(token(tokens.TYPE_OP_SQUARE_RIGHT_BRACKET)).\ - and_find(zero_or_more_tokens(tokens.TYPE_WHITESPACE)).\ - and_find(line_terminator_tokens) - - return c1.value(), c1.pending_tokens - - def double(ts2): - c2 = capture_from(ts2).\ - find(zero_or_more_tokens(tokens.TYPE_WHITESPACE)).\ - and_find(token(tokens.TYPE_OP_DOUBLE_SQUARE_LEFT_BRACKET)).\ - and_find(zero_or_more_tokens(tokens.TYPE_WHITESPACE)).\ - and_find(table_header_name_tokens).\ - and_find(zero_or_more_tokens(tokens.TYPE_WHITESPACE)).\ - and_find(token(tokens.TYPE_OP_DOUBLE_SQUARE_RIGHT_BRACKET)).\ - and_find(zero_or_more_tokens(tokens.TYPE_WHITESPACE)).\ - and_find(line_terminator_tokens) - - return c2.value(), c2.pending_tokens - - captured = capture_from(token_stream).find(single).or_find(double) - return TableHeaderElement(captured.value()), captured.pending_tokens - - -def atomic_element(token_stream): - captured = capture_from(token_stream).\ - find(string_token).\ - or_find(token(tokens.TYPE_INTEGER)).\ - or_find(token(tokens.TYPE_FLOAT)).\ - or_find(token(tokens.TYPE_DATE)).\ - or_find(token(tokens.TYPE_BOOLEAN)) - return AtomicElement(captured.value('Expected an atomic primitive value')), captured.pending_tokens - - -def punctuation_element(token_type): - def factory(ts): - c = capture_from(ts).find(token(token_type)) - return PunctuationElement(c.value('Expected the punctuation element: {}'.format(token_type))), c.pending_tokens - return factory - - -def value(token_stream): - captured = capture_from(token_stream).\ - find(atomic_element).\ - or_find(array_element).\ - or_find(inline_table_element) - return captured.value('Expected a primitive value, array or an inline table'), captured.pending_tokens - - -def array_internal(ts): - - def zero(ts0): - c = capture_from(ts0).\ - and_find(line_terminator_element).\ - and_find(space_element).\ - and_find(array_internal) - return c.value(), c.pending_tokens - - def one(ts1): - c = capture_from(ts1).\ - find(value).\ - and_find(space_element).\ - and_find(punctuation_element(tokens.TYPE_OP_COMMA)).\ - and_find(space_element).\ - and_find(line_terminator_element).\ - and_find(space_element).\ - and_find(array_internal) - return c.value(), c.pending_tokens - - def two(ts2): - c = capture_from(ts2).\ - find(value).\ - and_find(space_element).\ - and_find(punctuation_element(tokens.TYPE_OP_COMMA)).\ - and_find(space_element).\ - and_find(array_internal) - return c.value(), c.pending_tokens - - def three(ts3): - c = capture_from(ts3).\ - find(space_element).\ - and_find(line_terminator_element) - return c.value(), c.pending_tokens - - captured = capture_from(ts).find(zero).or_find(one).or_find(two).or_find(three).or_find(value).or_empty() - return captured.value(), captured.pending_tokens - - -def array_element(token_stream): - - def one(ts1): - ca = capture_from(ts1).\ - find(punctuation_element(tokens.TYPE_OP_SQUARE_LEFT_BRACKET)).\ - and_find(space_element).\ - and_find(array_internal).\ - and_find(space_element).\ - and_find(punctuation_element(tokens.TYPE_OP_SQUARE_RIGHT_BRACKET)) - return ca.value(), ca.pending_tokens - - def two(ts2): - ca = capture_from(ts2).\ - find(punctuation_element(tokens.TYPE_OP_SQUARE_LEFT_BRACKET)).\ - and_find(space_element).\ - and_find(array_internal).\ - and_find(space_element).\ - and_find(line_terminator_element).\ - and_find(space_element).\ - and_find(punctuation_element(tokens.TYPE_OP_SQUARE_RIGHT_BRACKET)) - return ca.value(), ca.pending_tokens - - captured = capture_from(token_stream).find(one).or_find(two) - return ArrayElement(captured.value()), captured.pending_tokens - - -def inline_table_element(token_stream): - - # InlineTableElement -> '{' Space InlineTableInternal Space '}' - # InlineTableKeyValuePair = STRING Space '=' Space Value - # InlineTableInternal -> InlineTableKeyValuePair Space ',' Space InlineTableInternal | - # InlineTableKeyValuePair | Empty - - def key_value(ts): - ca = capture_from(ts).\ - find(string_element).\ - and_find(space_element).\ - and_find(punctuation_element(tokens.TYPE_OP_ASSIGNMENT)).\ - and_find(space_element).\ - and_find(value) - return ca.value(), ca.pending_tokens - - def internal(ts): - def one(ts1): - c1 = capture_from(ts1).\ - find(key_value).\ - and_find(space_element).\ - and_find(punctuation_element(tokens.TYPE_OP_COMMA)).\ - and_find(space_element).\ - and_find(internal) - return c1.value(), c1.pending_tokens - - c = capture_from(ts).find(one).or_find(key_value).or_empty() - return c.value(), c.pending_tokens - - captured = capture_from(token_stream).\ - find(punctuation_element(tokens.TYPE_OP_CURLY_LEFT_BRACKET)).\ - and_find(space_element).\ - and_find(internal).\ - and_find(space_element).\ - and_find(punctuation_element(tokens.TYPE_OP_CURLY_RIGHT_BRACKET)) - - return InlineTableElement(captured.value()), captured.pending_tokens - - -def key_value_pair(token_stream): - captured = capture_from(token_stream).\ - find(space_element).\ - and_find(string_element).\ - and_find(space_element).\ - and_find(punctuation_element(tokens.TYPE_OP_ASSIGNMENT)).\ - and_find(space_element).\ - and_find(value).\ - and_find(space_element).\ - and_find(line_terminator_element) - return captured.value(), captured.pending_tokens - - -def table_body_elements(token_stream): - - # TableBody -> KeyValuePair TableBody | EmptyLine TableBody | EmptyLine | KeyValuePair - - def one(ts1): - c = capture_from(ts1).\ - find(key_value_pair).\ - and_find(table_body_elements) - return c.value(), c.pending_tokens - - def two(ts2): - c = capture_from(ts2).\ - find(empty_line_elements).\ - and_find(table_body_elements) - return c.value(), c.pending_tokens - - captured = capture_from(token_stream).\ - find(one).\ - or_find(two).\ - or_find(empty_line_elements).\ - or_find(key_value_pair) - - return captured.value(), captured.pending_tokens - - -def table_body_element(token_stream): - captured = capture_from(token_stream).find(table_body_elements) - return TableElement(captured.value()), captured.pending_tokens - - -def empty_line_tokens(ts1): - c1 = capture_from(ts1).find(space_element).and_find(line_terminator_element) - return c1.value(), c1.pending_tokens - - -def empty_line_elements(token_stream): - captured = capture_from(token_stream).find(empty_line_tokens) - return captured.value(), captured.pending_tokens - - -def file_entry_element(token_stream): - captured = capture_from(token_stream).find(table_header_element).\ - or_find(table_body_element) - return captured.value(), captured.pending_tokens - - -def toml_file_elements(token_stream): - - def one(ts1): - c1 = capture_from(ts1).find(file_entry_element).and_find(toml_file_elements) - return c1.value(), c1.pending_tokens - - captured = capture_from(token_stream).find(one).or_find(file_entry_element).or_empty() - return captured.value(), captured.pending_tokens diff --git a/pipenv/patched/prettytoml/parser/recdesc.py b/pipenv/patched/prettytoml/parser/recdesc.py deleted file mode 100644 index 8731dba3..00000000 --- a/pipenv/patched/prettytoml/parser/recdesc.py +++ /dev/null @@ -1,114 +0,0 @@ -from prettytoml.parser.errors import ParsingError -from prettytoml.parser.tokenstream import TokenStream - - -class Capturer: - """ - Recursive-descent matching DSL. Yeah.. - """ - - def __init__(self, token_stream, value=tuple(), dormant_error=None): - self._token_stream = token_stream - self._value = value - self._dormant_error = dormant_error - - def find(self, finder): - """ - Searches the token stream using the given finder. - - `finder(ts)` is a function that accepts a `TokenStream` instance and returns `(element, pending_ts)` - where `element` is the found "something" or a sequence of "somethings", and `pending_ts` the unconsumed - `TokenStream`. - - `finder(ts)` can raise `ParsingError` to indicate that it couldn't find anything, or - a `TokenStream.EndOfStream` to indicate a premature end of the TokenStream. - - This method returns a Capturer instance that can be further used to find more and more "somethings". The value - at any given moment can be retrieved via the `Capturer.value()` method. - """ - - try: - - # Execute finder! - element, pending_ts = finder(self._token_stream) - - # If result is not a sequence, make it so - if not isinstance(element, (tuple, list)): - element = (element,) - - # Return a Capturer with accumulated findings - return Capturer(pending_ts, value=self.value() + element) - - except ParsingError as e: - - # Failed to find, store error in returned value - return Capturer(self._token_stream, dormant_error=e) - - except TokenStream.EndOfStream as e: - - # Premature end of stream, store error in returned value - return Capturer(self._token_stream, dormant_error=e) - - def value(self, parsing_expectation_msg=None): - """ - Returns the accumulated values found as a sequence of values, or raises an encountered dormant error. - - If parsing_expectation_msg is specified and a dormant_error is a ParsingError, the expectation message is used - instead in it. - """ - - if self._dormant_error: - if parsing_expectation_msg and isinstance(self._dormant_error, ParsingError): - raise ParsingError(parsing_expectation_msg, token=self._token_stream.head) - else: - raise self._dormant_error - return self._value - - @property - def pending_tokens(self): - """ - Returns a TokenStream with the pending tokens yet to be processed. - """ - return self._token_stream - - def or_find(self, finder): - """ - If a dormant_error is present, try this new finder instead. If not, does nothing. - """ - if self._dormant_error: - return Capturer(self._token_stream).find(finder) - else: - return self - - def or_end_of_file(self): - """ - Discards any errors if at end of the stream. - """ - if isinstance(self._dormant_error, TokenStream.EndOfStream): - return Capturer(self.pending_tokens, value=self._value) - else: - return self - - def or_empty(self): - """ - Discards any previously-encountered dormant error. - """ - if self._dormant_error: - return Capturer(self.pending_tokens, value=self._value) - else: - return self - - def and_find(self, finder): - """ - Accumulate new "somethings" to the stored value using the given finder. - """ - - if self._dormant_error: - return Capturer(self.pending_tokens, dormant_error=self._dormant_error) - - return Capturer(self.pending_tokens, self.value()).find(finder) - - -def capture_from(token_stream): - return Capturer(token_stream) - diff --git a/pipenv/patched/prettytoml/parser/test_parser.py b/pipenv/patched/prettytoml/parser/test_parser.py deleted file mode 100644 index 40dd3dba..00000000 --- a/pipenv/patched/prettytoml/parser/test_parser.py +++ /dev/null @@ -1,156 +0,0 @@ -from prettytoml.elements.array import ArrayElement -from prettytoml.elements.atomic import AtomicElement -from prettytoml.elements.metadata import CommentElement, NewlineElement, WhitespaceElement -from prettytoml.elements.tableheader import TableHeaderElement -from prettytoml.lexer import tokenize -from prettytoml.parser import parser -from prettytoml.parser.tokenstream import TokenStream - - -def test_line_terminator_1(): - tokens = tokenize('# Sup\n') - ts = TokenStream(tokens) - element, pending_ts = parser.line_terminator_element(ts) - - assert isinstance(element, CommentElement) - assert pending_ts.offset == 2 - assert ts.offset == 0 - - -def test_line_terminator_2(): - tokens = tokenize('\n') - ts = TokenStream(tokens) - element, pending_ts = parser.line_terminator_element(ts) - - assert isinstance(element, NewlineElement) - assert pending_ts.offset == 1 - assert ts.offset == 0 - - -def test_space_1(): - ts = TokenStream(tokenize(' noo')) - space_element, pending_ts = parser.space_element(ts) - - assert isinstance(space_element, WhitespaceElement) - assert len(space_element.tokens) == 2 - assert pending_ts.offset == 2 - assert ts.offset == 0 - - -def test_space_2(): - ts = TokenStream(tokenize(' noo')) - space_element, pending_ts = parser.space_element(ts) - - assert isinstance(space_element, WhitespaceElement) - assert len(space_element.tokens) == 1 - assert pending_ts.offset == 1 - assert ts.offset == 0 - - -def test_space_3(): - ts = TokenStream(tokenize('noo')) - space_element, pending_ts = parser.space_element(ts) - - assert isinstance(space_element, WhitespaceElement) - assert len(space_element.tokens) == 0 - assert pending_ts.offset == 0 - assert ts.offset == 0 - - -def test_table_header(): - ts = TokenStream(tokenize(" [ namez . namey . namex ] \n other things")) - table_header_element, pending_tokens = parser.table_header_element(ts) - - assert isinstance(table_header_element, TableHeaderElement) - assert len(pending_tokens) == 4 - - -def test_atomic_element(): - e1, p1 = parser.atomic_element(TokenStream(tokenize('42 not'))) - assert isinstance(e1, AtomicElement) and e1.value == 42 - assert len(p1) == 2 - - e2, p2 = parser.atomic_element(TokenStream(tokenize('not 42'))) - assert isinstance(e2, AtomicElement) and e2.value == 'not' - assert len(p2) == 2 - - -def test_array(): - array_element, pending_ts = parser.array_element(TokenStream(tokenize('[ 3, 4, 5,6,7] '))) - - assert isinstance(array_element, ArrayElement) - assert len(array_element) == 5 - assert len(pending_ts) == 1 - - -def test_array_2(): - - text = """[ - "alpha", - "omega" -]""" - - array_element, pending_ts = parser.array_element(TokenStream(tokenize(text))) - - assert array_element[0] == 'alpha' - assert array_element[1] == 'omega' - - -def test_empty_array(): - - text = '[]' - - array_element, pending_ts = parser.array_element(TokenStream(tokenize(text))) - - assert isinstance(array_element, ArrayElement) - assert pending_ts.at_end - - -def test_inline_table(): - inline_table, pending_ts = parser.inline_table_element(TokenStream(tokenize('{ "id"= 42,test = name} vroom'))) - - assert set(inline_table.keys()) == {'id', 'test'} - assert len(pending_ts) == 2 - assert inline_table['id'] == 42 - assert inline_table['test'] == 'name' - - -def test_table_body(): - table_body, pending_ts = parser.table_body_element(TokenStream(tokenize(' name= "test" # No way man!\nid =42\n vvv'))) - assert set(table_body.keys()) == {'name', 'id'} - assert len(pending_ts) == 2 - assert table_body['name'] == 'test' - assert table_body['id'] == 42 - - -def test_key_value_pair(): - text = """hosts = [ - "alpha", - "omega" -] -""" - - parsed, pending_ts = parser.key_value_pair(TokenStream(tokenize(text))) - - assert isinstance(parsed[1], AtomicElement) - assert isinstance(parsed[5], ArrayElement) - - -def test_table_body_2(): - - text = """ -data = [ ["gamma", "delta"], [1, 2] ] - -# Line breaks are OK when inside arrays -hosts = [ - "alpha", - "omega" -] - -str_multiline = wohoo -""" - - table_body, pending_ts = parser.table_body_element(TokenStream(tokenize(text))) - - assert len(pending_ts) == 0 - diff --git a/pipenv/patched/prettytoml/parser/tokenstream.py b/pipenv/patched/prettytoml/parser/tokenstream.py deleted file mode 100644 index 2a2fdc25..00000000 --- a/pipenv/patched/prettytoml/parser/tokenstream.py +++ /dev/null @@ -1,39 +0,0 @@ - -class TokenStream: - """ - An immutable subset of a token sequence - """ - - class EndOfStream(Exception): - pass - - Nothing = tuple() - - def __init__(self, _tokens, offset=0): - if isinstance(_tokens, tuple): - self._tokens = _tokens - else: - self._tokens = tuple(_tokens) - self._head_index = offset - - def __len__(self): - return len(self._tokens) - self.offset - - @property - def head(self): - try: - return self._tokens[self._head_index] - except IndexError: - raise TokenStream.EndOfStream - - @property - def tail(self): - return TokenStream(self._tokens, offset=self._head_index+1) - - @property - def offset(self): - return self._head_index - - @property - def at_end(self): - return self.offset >= len(self._tokens) diff --git a/pipenv/patched/prettytoml/prettifier/__init__.py b/pipenv/patched/prettytoml/prettifier/__init__.py deleted file mode 100644 index 97ac1619..00000000 --- a/pipenv/patched/prettytoml/prettifier/__init__.py +++ /dev/null @@ -1,39 +0,0 @@ -from . import deindentanonymoustable, tableindent, tableassignment -from prettytoml.prettifier import tablesep, commentspace, linelength, tableentrysort - -""" - TOMLFile prettifiers - - Each prettifier is a function that accepts a sequence of Element instances that make up a - TOML file and it is allowed to modify it as it pleases. -""" - - -UNIFORM_TABLE_INDENTATION = tableindent.table_entries_should_be_uniformly_indented -UNIFORM_TABLE_ASSIGNMENT_SPACING = tableassignment.table_assignment_spacing -ANONYMOUS_TABLE_INDENTATION = deindentanonymoustable.deindent_anonymous_table -COMMENT_SPACING = commentspace.comment_space -TABLE_SPACING = tablesep.table_separation -LINE_LENGTH_ENFORCERS = linelength.line_length_limiter -TABLE_ENTRY_SORTING = tableentrysort.sort_table_entries - - -ALL = ( - TABLE_SPACING, # Must be before COMMENT_SPACING - COMMENT_SPACING, # Must be after TABLE_SPACING - UNIFORM_TABLE_INDENTATION, - UNIFORM_TABLE_ASSIGNMENT_SPACING, - ANONYMOUS_TABLE_INDENTATION, - LINE_LENGTH_ENFORCERS, - TABLE_ENTRY_SORTING, -) - - -def prettify(toml_file_elements, prettifiers=ALL): - """ - Prettifies a sequence of element instances according to pre-defined set of formatting rules. - """ - elements = toml_file_elements[:] - for prettifier in prettifiers: - elements = prettifier(elements) - return elements diff --git a/pipenv/patched/prettytoml/prettifier/commentspace.py b/pipenv/patched/prettytoml/prettifier/commentspace.py deleted file mode 100644 index fd549349..00000000 --- a/pipenv/patched/prettytoml/prettifier/commentspace.py +++ /dev/null @@ -1,35 +0,0 @@ - -from prettytoml.elements import traversal as t, factory as element_factory -from prettytoml.elements.table import TableElement - - -def comment_space(toml_file_elements): - """ - Rule: Line-terminating comments should always be prefixed by a single tab character whitespace only. - """ - elements = toml_file_elements[:] - for element in elements: - if isinstance(element, TableElement): - _do_table(element.sub_elements) - return elements - - -def _do_table(table_elements): - - # Iterator index - i = float('-inf') - - def next_newline(): - return t.find_following(table_elements, t.predicates.newline, i) - - def next_comment(): - return t.find_following(table_elements, t.predicates.comment, i) - - def last_non_metadata(): - return t.find_previous(table_elements, t.predicates.non_metadata, next_comment()) - - while next_comment() >= 0: - if i < last_non_metadata() < next_comment() < next_newline(): - del table_elements[last_non_metadata()+1:next_comment()] - table_elements.insert(next_comment(), element_factory.create_whitespace_element(char='\t', length=1)) - i = next_newline() diff --git a/pipenv/patched/prettytoml/prettifier/common.py b/pipenv/patched/prettytoml/prettifier/common.py deleted file mode 100644 index dd1e01a2..00000000 --- a/pipenv/patched/prettytoml/prettifier/common.py +++ /dev/null @@ -1,54 +0,0 @@ - -from itertools import * -from prettytoml.elements.common import TokenElement -from prettytoml.elements.metadata import NewlineElement - - -def text_to_elements(toml_text): - from ..lexer import tokenize - from ..parser import parse_tokens - return parse_tokens(tokenize(toml_text)) - - -def elements_to_text(toml_elements): - return ''.join(e.serialized() for e in toml_elements) - - -def assert_prettifier_works(source_text, expected_text, prettifier_func): - assert expected_text == elements_to_text(prettifier_func(text_to_elements(source_text))) - - -def lines(elements): - """ - Splits a sequence of elements into a sub-sequence of each line. - - A line is defined as a sequence of elements terminated by a NewlineElement. - """ - - def __next_line(es): - # Returns the next line and the remaining sequence of elements - line = tuple(takewhile(lambda e: not isinstance(e, NewlineElement), es)) - line += (es[len(line)],) - return line, es[len(line):] - - left_elements = tuple(elements) - while left_elements: - line, left_elements = __next_line(left_elements) - yield line - - -def non_empty_elements(elements): - """ - Filters out TokenElement instances with zero tokens. - """ - return filter(lambda e: not (isinstance(e, TokenElement) and not e.tokens), elements) - - -def index(predicate, seq): - """ - Returns the index of the element satisfying the given predicate, or None. - """ - try: - return next(i for (i, e) in enumerate(seq) if predicate(e)) - except StopIteration: - return None diff --git a/pipenv/patched/prettytoml/prettifier/deindentanonymoustable.py b/pipenv/patched/prettytoml/prettifier/deindentanonymoustable.py deleted file mode 100644 index a661f704..00000000 --- a/pipenv/patched/prettytoml/prettifier/deindentanonymoustable.py +++ /dev/null @@ -1,43 +0,0 @@ -import operator -from prettytoml.elements import traversal as t, traversal -from itertools import * -from functools import * -from prettytoml.elements.metadata import WhitespaceElement -from prettytoml.elements.table import TableElement -from prettytoml.prettifier import common - - -def deindent_anonymous_table(toml_file_elements): - """ - Rule: Anonymous table should never be indented. - """ - - anonymous_table_index = _find_anonymous_table(toml_file_elements) - if anonymous_table_index is None: - return toml_file_elements - - return toml_file_elements[:anonymous_table_index] + \ - [_unindent_table(toml_file_elements[anonymous_table_index])] + \ - toml_file_elements[anonymous_table_index+1:] - - -def _unindent_table(table_element): - table_lines = tuple(common.lines(table_element.sub_elements)) - unindented_lines = tuple(tuple(dropwhile(lambda e: isinstance(e, WhitespaceElement), line)) for line in table_lines) - return TableElement(reduce(operator.concat, unindented_lines)) - - -def _find_anonymous_table(toml_file_elements): - """ - Finds and returns the index of the TableElement comprising the anonymous table or None. - """ - - first_table_index = common.index(t.predicates.table, toml_file_elements) - first_table_header_index = common.index(t.predicates.table_header, toml_file_elements) - - if first_table_header_index is None: - return first_table_index - elif first_table_index < first_table_header_index: - return first_table_index - - diff --git a/pipenv/patched/prettytoml/prettifier/linelength.py b/pipenv/patched/prettytoml/prettifier/linelength.py deleted file mode 100644 index 67d3a112..00000000 --- a/pipenv/patched/prettytoml/prettifier/linelength.py +++ /dev/null @@ -1,62 +0,0 @@ -import operator -from prettytoml import tokens -from prettytoml.prettifier import common -from prettytoml.elements import traversal as t, factory as element_factory -from prettytoml.elements.array import ArrayElement -from prettytoml.elements.atomic import AtomicElement -from prettytoml.elements.inlinetable import InlineTableElement -from prettytoml.elements.table import TableElement -from functools import * - - -MAXIMUM_LINE_LENGTH = 120 - - -def line_length_limiter(toml_file_elements): - """ - Rule: Lines whose lengths exceed 120 characters whose values are strings, arrays should have the array or - string value broken onto multiple lines - """ - return tuple(_fixed_table(e) if isinstance(e, TableElement) else e for e in toml_file_elements) - - -def _fixed_table(table_element): - """ - Returns a new TableElement. - """ - assert isinstance(table_element, TableElement) - lines = tuple(common.lines(table_element.sub_elements)) - fixed_lines = tuple(_fixed_line(l) if _line_length(l) > MAXIMUM_LINE_LENGTH else l for l in lines) - return TableElement(sub_elements=tuple(reduce(operator.concat, fixed_lines))) - - -def _line_length(line_elements): - """ - Returns the character length of the serialized elements of the given line. - """ - return sum(len(e.serialized()) for e in line_elements) - - -def _fixed_line(line_elements): - - def line_value_index(): - # Returns index of value element in the line - key_index = t.find_following(line_elements, t.predicates.non_metadata) - return t.find_following(line_elements, t.predicates.non_metadata, key_index) - - def multiline_equivalent(element): - if isinstance(element, AtomicElement) and tokens.is_string(element.first_token): - return element_factory.create_multiline_string(element.value, MAXIMUM_LINE_LENGTH) - elif isinstance(element, ArrayElement): - element.turn_into_multiline() - return element - else: - return element - - line_elements = tuple(line_elements) - value_index = line_value_index() - if value_index >= 0: - return line_elements[:value_index] + (multiline_equivalent(line_elements[value_index]),) + \ - line_elements[value_index+1:] - else: - return line_elements diff --git a/pipenv/patched/prettytoml/prettifier/tableassignment.py b/pipenv/patched/prettytoml/prettifier/tableassignment.py deleted file mode 100644 index 1d35d698..00000000 --- a/pipenv/patched/prettytoml/prettifier/tableassignment.py +++ /dev/null @@ -1,40 +0,0 @@ - -from prettytoml.elements import traversal as t, factory as element_factory - - -def table_assignment_spacing(toml_file_elements): - """ - Rule: Every key and value pair in any table should be separated the triplet - (single space character, an assignment character =, single space character) - """ - elements = toml_file_elements[:] - for table_element in (e for e in elements if t.predicates.table(e)): - _do_table(table_element) - return elements - - -def _do_table(table_element): - - elements = table_element.sub_elements - - # Our iterator index - i = float('-inf') - - def next_key(): - return t.find_following(elements, t.predicates.non_metadata, i) - - def next_assignment(): - return t.find_following(elements, t.predicates.op_assignment, next_key()) - - def next_value(): - return t.find_following(elements, t.predicates.non_metadata, next_assignment()) - - while next_key() >= 0: - - del elements[next_key()+1:next_assignment()] - del elements[next_assignment()+1:next_value()] - - elements.insert(next_assignment(), element_factory.create_whitespace_element(1)) - elements.insert(next_value(), element_factory.create_whitespace_element(1)) - - i = t.find_following(elements, t.predicates.newline, i) diff --git a/pipenv/patched/prettytoml/prettifier/tableentrysort.py b/pipenv/patched/prettytoml/prettifier/tableentrysort.py deleted file mode 100644 index 8cbd307b..00000000 --- a/pipenv/patched/prettytoml/prettifier/tableentrysort.py +++ /dev/null @@ -1,38 +0,0 @@ -import operator -from prettytoml import tokens -from prettytoml.elements.common import TokenElement -from prettytoml.elements.table import TableElement -from prettytoml.prettifier import common -from functools import * - - -def sort_table_entries(toml_file_elements): - """ - Rule: Entries within a single table should be ordered lexicographically by key - """ - return [_sorted_table(element) if isinstance(element, TableElement) else element for element in toml_file_elements] - - -def _line_key(line_elements): - """ - Given a sequence of elements comprising a single line, returns an orderable value to use in ordering lines. - """ - for e in line_elements: - if isinstance(e, TokenElement) and tokens.is_string(e.first_token): - return e.primitive_value - return 'z' * 10 # Metadata lines should be at the end - - -def _sorted_table(table): - """ - Returns another TableElement where the table entries are sorted lexicographically by key. - """ - assert isinstance(table, TableElement) - - # Discarding TokenElements with no tokens in them - table_elements = common.non_empty_elements(table.sub_elements) - lines = tuple(common.lines(table_elements)) - sorted_lines = sorted(lines, key=_line_key) - sorted_elements = reduce(operator.concat, sorted_lines) - - return TableElement(sorted_elements) diff --git a/pipenv/patched/prettytoml/prettifier/tableindent.py b/pipenv/patched/prettytoml/prettifier/tableindent.py deleted file mode 100644 index 3b60883d..00000000 --- a/pipenv/patched/prettytoml/prettifier/tableindent.py +++ /dev/null @@ -1,49 +0,0 @@ -from prettytoml import tokens -from prettytoml.elements import traversal as t, factory as element_factory -from prettytoml.tokens import py2toml - - -def table_entries_should_be_uniformly_indented(toml_file_elements): - """ - Rule: Nth-level table sections should be indented by (N-1)*2 spaces - """ - elements = toml_file_elements[:] - for (i, e) in enumerate(elements): - if t.predicates.table_header(e): - table = elements[t.find_following(elements, t.predicates.table, i)] - _do_table_header(e) - _do_table(table, len(e.names)) - return elements - - -def _do_table_header(table_header): - indent_start = 0 - indent_end = next(i for (i, token) in enumerate(table_header.tokens) if token.type != tokens.TYPE_WHITESPACE) - - del table_header.tokens[indent_start:indent_end] - table_header.tokens.insert(0, py2toml.create_whitespace(' ' * ((len(table_header.names)-1) * 2))) - - -def _do_table(table_element, table_level): - - elements = table_element.sub_elements - - # Iterator index - i = float('-inf') - - def first_indent(): - return t.find_following(elements, t.predicates.whitespace, i) - - def next_non_metadata(): - return t.find_following(elements, t.predicates.non_metadata, i) - - def next_newline(): - return t.find_following(elements, t.predicates.newline, next_non_metadata()) - - while next_non_metadata() >= 0: - if first_indent() >= 0: - del elements[first_indent():next_non_metadata()] - - elements.insert(next_non_metadata(), element_factory.create_whitespace_element((table_level-1)*2)) - - i = next_newline() diff --git a/pipenv/patched/prettytoml/prettifier/tablesep.py b/pipenv/patched/prettytoml/prettifier/tablesep.py deleted file mode 100644 index 059007f3..00000000 --- a/pipenv/patched/prettytoml/prettifier/tablesep.py +++ /dev/null @@ -1,31 +0,0 @@ - -from prettytoml.elements import traversal as t, factory as element_factory -from prettytoml.elements.metadata import WhitespaceElement, NewlineElement -from prettytoml.elements.table import TableElement - - -def table_separation(toml_file_elements): - """ - Rule: Tables should always be separated by an empty line. - """ - elements = toml_file_elements[:] - for element in elements: - if isinstance(element, TableElement): - _do_table(element.sub_elements) - return elements - - -def _do_table(table_elements): - - while table_elements and isinstance(table_elements[-1], WhitespaceElement): - del table_elements[-1] - - if not table_elements: - return - - if isinstance(table_elements[-1], NewlineElement): - last_non_metadata_i = t.find_previous(table_elements, t.predicates.non_metadata) - del table_elements[last_non_metadata_i+1:] - - table_elements.append(element_factory.create_newline_element()) - table_elements.append(element_factory.create_newline_element()) diff --git a/pipenv/patched/prettytoml/prettifier/test_commentspace.py b/pipenv/patched/prettytoml/prettifier/test_commentspace.py deleted file mode 100644 index 53d96d76..00000000 --- a/pipenv/patched/prettytoml/prettifier/test_commentspace.py +++ /dev/null @@ -1,28 +0,0 @@ - -from .common import assert_prettifier_works -from .commentspace import comment_space - - -def test_comment_space(): - - toml_text = """ -my_key = string -id = 12 # My special ID - -[section.name] -headerk = false -# Own-line comment should stay the same -other_key = "value" -""" - - expected_toml_text = """ -my_key = string -id = 12\t# My special ID - -[section.name] -headerk = false -# Own-line comment should stay the same -other_key = "value" -""" - - assert_prettifier_works(toml_text, expected_toml_text, comment_space) diff --git a/pipenv/patched/prettytoml/prettifier/test_deindentanonymoustable.py b/pipenv/patched/prettytoml/prettifier/test_deindentanonymoustable.py deleted file mode 100644 index 10a6d2c8..00000000 --- a/pipenv/patched/prettytoml/prettifier/test_deindentanonymoustable.py +++ /dev/null @@ -1,22 +0,0 @@ - -""" - This testing module depends on all the other modules. -""" - -from .deindentanonymoustable import deindent_anonymous_table -from .common import assert_prettifier_works - - -def test_anon_table_indent(): - toml_text = """ - key=value - another_key =44 -noname = me -""" - - expected_toml_text = """ -key=value -another_key =44 -noname = me -""" - assert_prettifier_works(toml_text, expected_toml_text, deindent_anonymous_table) diff --git a/pipenv/patched/prettytoml/prettifier/test_linelength.py b/pipenv/patched/prettytoml/prettifier/test_linelength.py deleted file mode 100644 index e4ab8fcb..00000000 --- a/pipenv/patched/prettytoml/prettifier/test_linelength.py +++ /dev/null @@ -1,39 +0,0 @@ -from .linelength import line_length_limiter -from .common import assert_prettifier_works, elements_to_text, text_to_elements -import pytoml - - -def test_splitting_string(): - toml_text = """ -k = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. In et lectus nec erat condimentum scelerisque gravida sed ipsum. Mauris non orci tincidunt, viverra enim eget, tincidunt orci. Sed placerat nibh vitae ante maximus egestas maximus eu quam. Praesent vehicula mauris vestibulum, mattis turpis sollicitudin, aliquam felis. Pellentesque volutpat pharetra purus vel finibus. Vestibulum sed tempus dui. Maecenas auctor sit amet diam et porta. Morbi id libero at elit ultricies porta vel vitae nullam. " -""" - - expected_toml_text = """ -k = \"\"\" -Lorem ipsum dolor sit amet, consectetur adipiscing elit. In et lectus nec erat condimentum scelerisque gravida sed \\ -ipsum. Mauris non orci tincidunt, viverra enim eget, tincidunt orci. Sed placerat nibh vitae ante maximus egestas \\ -maximus eu quam. Praesent vehicula mauris vestibulum, mattis turpis sollicitudin, aliquam felis. Pellentesque volutpat \\ -pharetra purus vel finibus. Vestibulum sed tempus dui. Maecenas auctor sit amet diam et porta. Morbi id libero at elit \\ -ultricies porta vel vitae nullam. \"\"\" -""" - assert_prettifier_works(toml_text, expected_toml_text, line_length_limiter) - - -def test_splitting_array(): - toml_text = """ - -somethingweird = false - -[section] -k = [4, 8, 15, 16, 23, 42, 4, 8, 15, 16, 23, 42, 4, 8, 15, 16, 23, 42, 4, 8, 15, 16, 23, 42, 4, 8, 15, 16, 23, 42, 4, 8, 15, 16, 23, 42, 4, 8, 15, 16, 23, 42] - - -[data] -id = 12 - -""" - - prettified = elements_to_text(line_length_limiter(text_to_elements(toml_text))) - - assert pytoml.loads(prettified) == pytoml.loads(toml_text) - assert all(len(line) < 120 for line in prettified.split('\n')) diff --git a/pipenv/patched/prettytoml/prettifier/test_tableassignment.py b/pipenv/patched/prettytoml/prettifier/test_tableassignment.py deleted file mode 100644 index 82fcc174..00000000 --- a/pipenv/patched/prettytoml/prettifier/test_tableassignment.py +++ /dev/null @@ -1,29 +0,0 @@ - -from .tableassignment import table_assignment_spacing -from .common import assert_prettifier_works - - -def test_table_assignment_spacing(): - toml_text = """ - key1= "my value" - key2 =42 - keys = [4, 5,1] - - [section] - key1= "my value" - key2 =42 - keys = [4, 5,1] -""" - - expected_prettified = """ - key1 = "my value" - key2 = 42 - keys = [4, 5,1] - - [section] - key1 = "my value" - key2 = 42 - keys = [4, 5,1] -""" - - assert_prettifier_works(toml_text, expected_prettified, table_assignment_spacing) diff --git a/pipenv/patched/prettytoml/prettifier/test_tableentrysort.py b/pipenv/patched/prettytoml/prettifier/test_tableentrysort.py deleted file mode 100644 index 0cc39f78..00000000 --- a/pipenv/patched/prettytoml/prettifier/test_tableentrysort.py +++ /dev/null @@ -1,45 +0,0 @@ - -from .tableentrysort import sort_table_entries -from .common import assert_prettifier_works - - -def test_table_sorting(): - toml_text = """description = "" -firstname = "adnan" -lastname = "fatayerji" -git_aydo = "" -groups = ["sales", "dubai", "mgmt"] -skype = "" -emails = ["adnan@incubaid.com", - "fatayera@incubaid.com", - "adnan.fatayerji@incubaid.com", - "adnan@greenitglobe.com", - "fatayera@greenitglobe.com", - "adnan.fatayerji@greenitglobe.com"] -# I really like this table -id = "fatayera" -git_github = "" -telegram = "971507192009" -mobiles = ["971507192009"] -""" - - prettified = """description = "" -emails = ["adnan@incubaid.com", - "fatayera@incubaid.com", - "adnan.fatayerji@incubaid.com", - "adnan@greenitglobe.com", - "fatayera@greenitglobe.com", - "adnan.fatayerji@greenitglobe.com"] -firstname = "adnan" -git_aydo = "" -git_github = "" -groups = ["sales", "dubai", "mgmt"] -# I really like this table -id = "fatayera" -lastname = "fatayerji" -mobiles = ["971507192009"] -skype = "" -telegram = "971507192009" -""" - - assert_prettifier_works(toml_text, prettified, sort_table_entries) diff --git a/pipenv/patched/prettytoml/prettifier/test_tableindent.py b/pipenv/patched/prettytoml/prettifier/test_tableindent.py deleted file mode 100644 index a37f73a7..00000000 --- a/pipenv/patched/prettytoml/prettifier/test_tableindent.py +++ /dev/null @@ -1,25 +0,0 @@ - -from .tableindent import table_entries_should_be_uniformly_indented -from .common import assert_prettifier_works - - -def test_table_entries_should_be_uniformly_indented(): - toml_text = """ - [firstlevel] -hello = "my name" - my_id = 12 - - [firstlevel.secondlevel] - my_truth = False -""" - - expected_toml_text = """ -[firstlevel] -hello = "my name" -my_id = 12 - - [firstlevel.secondlevel] - my_truth = False -""" - - assert_prettifier_works(toml_text, expected_toml_text, table_entries_should_be_uniformly_indented) diff --git a/pipenv/patched/prettytoml/prettifier/test_tablesep.py b/pipenv/patched/prettytoml/prettifier/test_tablesep.py deleted file mode 100644 index a8a81d52..00000000 --- a/pipenv/patched/prettytoml/prettifier/test_tablesep.py +++ /dev/null @@ -1,34 +0,0 @@ - -from .tablesep import table_separation -from .common import assert_prettifier_works - - -def test_table_separation(): - - toml_text = """key1 = "value1" -key2 = 22 -[section] -k = false -m= "true" - - - -[another.section] -l = "t" -creativity = "on vacation" -""" - - expected_toml_text = """key1 = "value1" -key2 = 22 - -[section] -k = false -m= "true" - -[another.section] -l = "t" -creativity = "on vacation" - -""" - - assert_prettifier_works(toml_text, expected_toml_text, table_separation) diff --git a/pipenv/patched/prettytoml/test_prettifier.py b/pipenv/patched/prettytoml/test_prettifier.py deleted file mode 100644 index f702fb01..00000000 --- a/pipenv/patched/prettytoml/test_prettifier.py +++ /dev/null @@ -1,12 +0,0 @@ - -from .prettifier import prettify -from .prettifier.common import assert_prettifier_works -import pytoml - - -def test_prettifying_against_humanly_verified_sample(): - toml_source = open('sample.toml').read() - expected = open('sample-prettified.toml').read() - - assert_prettifier_works(toml_source, expected, prettify) - assert pytoml.loads(toml_source) == pytoml.loads(expected) diff --git a/pipenv/patched/prettytoml/test_util.py b/pipenv/patched/prettytoml/test_util.py deleted file mode 100644 index b741abfa..00000000 --- a/pipenv/patched/prettytoml/test_util.py +++ /dev/null @@ -1,22 +0,0 @@ -from prettytoml.util import is_sequence_like, is_dict_like, chunkate_string - - -def test_is_sequence_like(): - assert is_sequence_like([1, 3, 4]) - assert not is_sequence_like(42) - - -def test_is_dict_like(): - assert is_dict_like({'name': False}) - assert not is_dict_like(42) - assert not is_dict_like([4, 8, 15]) - - -def test_chunkate_string(): - - text = """Lorem ipsum dolor sit amet, consectetur adipiscing elit. In et lectus nec erat condimentum scelerisque gravida sed ipsum. Mauris non orci tincidunt, viverra enim eget, tincidunt orci. Sed placerat nibh vitae ante maximus egestas maximus eu quam. Praesent vehicula mauris vestibulum, mattis turpis sollicitudin, aliquam felis. Pellentesque volutpat pharetra purus vel finibus. Vestibulum sed tempus dui. Maecenas auctor sit amet diam et porta. Morbi id libero at elit ultricies porta vel vitae nullam. """ - - chunks = chunkate_string(text, 50) - - assert ''.join(chunks) == text - assert all(len(chunk) <= 50 for chunk in chunks) diff --git a/pipenv/patched/prettytoml/tokens/__init__.py b/pipenv/patched/prettytoml/tokens/__init__.py deleted file mode 100644 index a5b2e9ae..00000000 --- a/pipenv/patched/prettytoml/tokens/__init__.py +++ /dev/null @@ -1,136 +0,0 @@ - -""" -TOML lexical tokens. -""" - -class TokenType: - """ - A TokenType is a concrete type of a source token along with a defined priority and a higher-order kind. - - The priority will be used in determining the tokenization behaviour of the lexer in the following manner: - whenever more than one token is recognizable as the next possible token and they are all of equal source - length, this priority is going to be used to break the tie by favoring the token type of the lowest priority - value. A TokenType instance is naturally ordered by its priority. - """ - - def __init__(self, name, priority, is_metadata): - self._priority = priority - self._name = name - self._is_metadata = is_metadata - - @property - def is_metadata(self): - return self._is_metadata - - @property - def priority(self): - return self._priority - - def __repr__(self): - return "{}-{}".format(self.priority, self._name) - - def __lt__(self, other): - return isinstance(other, TokenType) and self._priority < other.priority - -# Possible types of tokens -TYPE_BOOLEAN = TokenType('boolean', 0, is_metadata=False) -TYPE_INTEGER = TokenType('integer', 0, is_metadata=False) -TYPE_OP_COMMA = TokenType('comma', 0, is_metadata=True) -TYPE_OP_SQUARE_LEFT_BRACKET = TokenType('square_left_bracket', 0, is_metadata=True) -TYPE_OP_SQUARE_RIGHT_BRACKET = TokenType('square_right_bracket', 0, is_metadata=True) -TYPE_OP_CURLY_LEFT_BRACKET = TokenType('curly_left_bracket', 0, is_metadata=True) -TYPE_OP_CURLY_RIGHT_BRACKET = TokenType('curly_right_bracket', 0, is_metadata=True) -TYPE_OP_ASSIGNMENT = TokenType('assignment', 0, is_metadata=True) -TYPE_OP_DOUBLE_SQUARE_LEFT_BRACKET = TokenType('double_square_left_bracket', 0, is_metadata=True) -TYPE_OP_DOUBLE_SQUARE_RIGHT_BRACKET = TokenType('double_square_right_bracket', 0, is_metadata=True) -TYPE_FLOAT = TokenType('float', 1, is_metadata=False) -TYPE_DATE = TokenType('date', 40, is_metadata=False) -TYPE_OPT_DOT = TokenType('dot', 40, is_metadata=True) -TYPE_BARE_STRING = TokenType('bare_string', 50, is_metadata=False) -TYPE_STRING = TokenType('string', 90, is_metadata=False) -TYPE_MULTILINE_STRING = TokenType('multiline_string', 90, is_metadata=False) -TYPE_LITERAL_STRING = TokenType('literal_string', 90, is_metadata=False) -TYPE_MULTILINE_LITERAL_STRING = TokenType('multiline_literal_string', 90, is_metadata=False) -TYPE_NEWLINE = TokenType('newline', 91, is_metadata=True) -TYPE_WHITESPACE = TokenType('whitespace', 93, is_metadata=True) -TYPE_COMMENT = TokenType('comment', 95, is_metadata=True) - - -def is_operator(token): - """ - Returns True if the given token is an operator token. - """ - return token.type in ( - TYPE_OP_COMMA, - TYPE_OP_SQUARE_LEFT_BRACKET, - TYPE_OP_SQUARE_RIGHT_BRACKET, - TYPE_OP_DOUBLE_SQUARE_LEFT_BRACKET, - TYPE_OP_DOUBLE_SQUARE_RIGHT_BRACKET, - TYPE_OP_CURLY_LEFT_BRACKET, - TYPE_OP_CURLY_RIGHT_BRACKET, - TYPE_OP_ASSIGNMENT, - TYPE_OPT_DOT, - ) - - -def is_string(token): - return token.type in ( - TYPE_STRING, - TYPE_MULTILINE_STRING, - TYPE_LITERAL_STRING, - TYPE_BARE_STRING, - TYPE_MULTILINE_LITERAL_STRING - ) - - -class Token: - """ - A token/lexeme in a TOML source file. - - A Token instance is naturally ordered by its type. - """ - - def __init__(self, _type, source_substring, col=None, row=None): - self._source_substring = source_substring - self._type = _type - self._col = col - self._row = row - - def __eq__(self, other): - if not isinstance(other, Token): - return False - return self.source_substring == other.source_substring and self.type == other.type - - @property - def col(self): - """ - Column number (1-indexed). - """ - return self._col - - @property - def row(self): - """ - Row number (1-indexed). - """ - return self._row - - @property - def type(self): - """ - One of of the TOKEN_TYPE_* constants. - """ - return self._type - - @property - def source_substring(self): - """ - The substring of the initial source file containing this token. - """ - return self._source_substring - - def __lt__(self, other): - return isinstance(other, Token) and self.type < other.type - - def __repr__(self): - return "{}: {}".format(self.type, self.source_substring) diff --git a/pipenv/patched/prettytoml/tokens/errors.py b/pipenv/patched/prettytoml/tokens/errors.py deleted file mode 100644 index d40cb8e9..00000000 --- a/pipenv/patched/prettytoml/tokens/errors.py +++ /dev/null @@ -1,13 +0,0 @@ -from prettytoml.errors import TOMLError - - -class DeserializationError(TOMLError): - pass - - -class BadEscapeCharacter(TOMLError): - pass - - -class MalformedDateError(DeserializationError): - pass diff --git a/pipenv/patched/prettytoml/tokens/py2toml.py b/pipenv/patched/prettytoml/tokens/py2toml.py deleted file mode 100644 index 82991958..00000000 --- a/pipenv/patched/prettytoml/tokens/py2toml.py +++ /dev/null @@ -1,133 +0,0 @@ - -""" -A converter of python values to TOML Token instances. -""" -import codecs -import datetime -import six -from prettytoml import tokens -import re -from prettytoml.errors import TOMLError -from prettytoml.tokens import Token -from prettytoml.util import chunkate_string - - -class NotPrimitiveError(TOMLError): - pass - - -_operator_tokens_by_type = { - tokens.TYPE_OP_SQUARE_LEFT_BRACKET: tokens.Token(tokens.TYPE_OP_SQUARE_LEFT_BRACKET, u'['), - tokens.TYPE_OP_SQUARE_RIGHT_BRACKET: tokens.Token(tokens.TYPE_OP_SQUARE_RIGHT_BRACKET, u']'), - tokens.TYPE_OP_DOUBLE_SQUARE_LEFT_BRACKET: tokens.Token(tokens.TYPE_OP_DOUBLE_SQUARE_LEFT_BRACKET, u'[['), - tokens.TYPE_OP_DOUBLE_SQUARE_RIGHT_BRACKET: tokens.Token(tokens.TYPE_OP_DOUBLE_SQUARE_RIGHT_BRACKET, u']]'), - tokens.TYPE_OP_COMMA: tokens.Token(tokens.TYPE_OP_COMMA, u','), - tokens.TYPE_NEWLINE: tokens.Token(tokens.TYPE_NEWLINE, u'\n'), - tokens.TYPE_OPT_DOT: tokens.Token(tokens.TYPE_OPT_DOT, u'.'), -} - - -def operator_token(token_type): - return _operator_tokens_by_type[token_type] - - -def create_primitive_token(value, multiline_strings_allowed=True): - """ - Creates and returns a single token for the given primitive atomic value. - - Raises NotPrimitiveError when the given value is not a primitive atomic value - """ - if value is None: - return create_primitive_token('') - elif isinstance(value, bool): - return tokens.Token(tokens.TYPE_BOOLEAN, u'true' if value else u'false') - elif isinstance(value, int): - return tokens.Token(tokens.TYPE_INTEGER, u'{}'.format(value)) - elif isinstance(value, float): - return tokens.Token(tokens.TYPE_FLOAT, u'{}'.format(value)) - elif isinstance(value, (datetime.datetime, datetime.date, datetime.time)): - s = value.isoformat() - if s.endswith('+00:00'): - s = s[:-6] + 'Z' - return tokens.Token(tokens.TYPE_DATE, s) - elif isinstance(value, six.string_types): - return create_string_token(value, multiline_strings_allowed=multiline_strings_allowed) - - raise NotPrimitiveError("{} of type {}".format(value, type(value))) - - -_bare_string_regex = re.compile('^[a-zA-Z_-]*$') - - -def create_string_token(text, bare_string_allowed=False, multiline_strings_allowed=True): - """ - Creates and returns a single string token. - - Raises ValueError on non-string input. - """ - - if not isinstance(text, six.string_types): - raise ValueError('Given value must be a string') - - if text == '': - return tokens.Token(tokens.TYPE_STRING, '""'.format(_escape_single_line_quoted_string(text))) - elif bare_string_allowed and _bare_string_regex.match(text): - return tokens.Token(tokens.TYPE_BARE_STRING, text) - elif multiline_strings_allowed and (len(tuple(c for c in text if c == '\n')) >= 2 or len(text) > 80): - # If containing two or more newlines or is longer than 80 characters we'll use the multiline string format - return _create_multiline_string_token(text) - else: - return tokens.Token(tokens.TYPE_STRING, '"{}"'.format(_escape_single_line_quoted_string(text))) - - -def _escape_single_line_quoted_string(text): - if six.PY2: - return text.encode('unicode-escape').encode('string-escape').replace('"', '\\"').replace("\\'", "'") - else: - return codecs.encode(text, 'unicode-escape').decode().replace('"', '\\"') - - -def _create_multiline_string_token(text): - escaped = text.replace(u'"""', u'\"\"\"') - if len(escaped) > 50: - return tokens.Token(tokens.TYPE_MULTILINE_STRING, u'"""\n{}\\\n"""'.format(_break_long_text(escaped))) - else: - return tokens.Token(tokens.TYPE_MULTILINE_STRING, u'"""{}"""'.format(escaped)) - - -def _break_long_text(text, maximum_length=75): - """ - Breaks into lines of 75 character maximum length that are terminated by a backslash. - """ - - def next_line(remaining_text): - - # Returns a line and the remaining text - - if '\n' in remaining_text and remaining_text.index('\n') < maximum_length: - i = remaining_text.index('\n') - return remaining_text[:i+1], remaining_text[i+2:] - elif len(remaining_text) > maximum_length and ' ' in remaining_text: - i = remaining_text[:maximum_length].rfind(' ') - return remaining_text[:i+1] + '\\\n', remaining_text[i+2:] - else: - return remaining_text, '' - - remaining_text = text - lines = [] - while remaining_text: - line, remaining_text = next_line(remaining_text) - lines += [line] - - return ''.join(lines) - - -def create_whitespace(source_substring): - return Token(tokens.TYPE_WHITESPACE, source_substring) - - -def create_multiline_string(text, maximum_line_length=120): - def escape(t): - return t.replace(u'"""', six.u(r'\"\"\"')) - source_substring = u'"""\n{}"""'.format(u'\\\n'.join(chunkate_string(escape(text), maximum_line_length))) - return Token(tokens.TYPE_MULTILINE_STRING, source_substring) diff --git a/pipenv/patched/prettytoml/tokens/test_py2toml.py b/pipenv/patched/prettytoml/tokens/test_py2toml.py deleted file mode 100644 index 0d029c5d..00000000 --- a/pipenv/patched/prettytoml/tokens/test_py2toml.py +++ /dev/null @@ -1,69 +0,0 @@ -import datetime - -import strict_rfc3339 - -from prettytoml import tokens -from prettytoml.tokens import py2toml - - -def test_string(): - assert py2toml.create_string_token('fawzy', bare_string_allowed=True) == tokens.Token(tokens.TYPE_BARE_STRING, 'fawzy') - assert py2toml.create_primitive_token('I am a "cr\'azy" sentence.') == \ - tokens.Token(tokens.TYPE_STRING, '"I am a \\"cr\'azy\\" sentence."') - - -def test_multiline_string(): - text = 'The\nSuper\nT"""OML"""\n\nIs coming' - - primitive_token = py2toml.create_primitive_token(text) - - assert primitive_token.source_substring == '"""The\nSuper\nT\"\"\"OML\"\"\"\n\nIs coming"""' - - -def test_long_string(): - text = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Suspendisse faucibus nibh id urna euismod, " \ - "vitae blandit nisi blandit. Nam eu odio ex. Praesent iaculis sapien justo. Proin vehicula orci rhoncus " \ - "risus mattis cursus. Sed quis commodo diam. Morbi dictum fermentum ex. Ut augue lorem, facilisis eu " \ - "posuere ut, ullamcorper et quam. Donec porta neque eget erat lacinia, in convallis elit scelerisque. " \ - "Class aptent taciti sociosqu ad litora torquent per conubia nostra, per inceptos himenaeos. Praesent " \ - "felis metus, venenatis eu aliquam vel, fringilla in turpis. Praesent interdum pulvinar enim, et mattis " \ - "urna dapibus et. Sed ut egestas mauris. Etiam eleifend dui." - - primitive_token = py2toml.create_primitive_token(text) - - assert primitive_token.source_substring[3:-3] == r""" -Lorem ipsum dolor sit amet, consectetur adipiscing elit. Suspendisse \ -aucibus nibh id urna euismod, vitae blandit nisi blandit. Nam eu odio ex. \ -raesent iaculis sapien justo. Proin vehicula orci rhoncus risus mattis \ -ursus. Sed quis commodo diam. Morbi dictum fermentum ex. Ut augue lorem, \ -acilisis eu posuere ut, ullamcorper et quam. Donec porta neque eget erat \ -acinia, in convallis elit scelerisque. Class aptent taciti sociosqu ad \ -itora torquent per conubia nostra, per inceptos himenaeos. Praesent felis \ -etus, venenatis eu aliquam vel, fringilla in turpis. Praesent interdum \ -ulvinar enim, et mattis urna dapibus et. Sed ut egestas mauris. Etiam \ -leifend dui.\ -""" - - -def test_int(): - assert py2toml.create_primitive_token(42) == tokens.Token(tokens.TYPE_INTEGER, '42') - - -def test_float(): - assert py2toml.create_primitive_token(4.2) == tokens.Token(tokens.TYPE_FLOAT, '4.2') - - -def test_bool(): - assert py2toml.create_primitive_token(False) == tokens.Token(tokens.TYPE_BOOLEAN, 'false') - assert py2toml.create_primitive_token(True) == tokens.Token(tokens.TYPE_BOOLEAN, 'true') - - -def test_date(): - ts = strict_rfc3339.rfc3339_to_timestamp('1979-05-27T00:32:00-07:00') - dt = datetime.datetime.fromtimestamp(ts) - assert py2toml.create_primitive_token(dt) == tokens.Token(tokens.TYPE_DATE, '1979-05-27T07:32:00Z') - - -def test_none(): - t = py2toml.create_primitive_token(None) - assert t.type == tokens.TYPE_STRING and t.source_substring == '""' diff --git a/pipenv/patched/prettytoml/tokens/test_toml2py.py b/pipenv/patched/prettytoml/tokens/test_toml2py.py deleted file mode 100644 index ce166365..00000000 --- a/pipenv/patched/prettytoml/tokens/test_toml2py.py +++ /dev/null @@ -1,86 +0,0 @@ -from datetime import datetime - -import pytz - -from prettytoml import tokens -from prettytoml.tokens import toml2py -from prettytoml.tokens.errors import BadEscapeCharacter, DeserializationError - - -def test_integer(): - t1 = tokens.Token(tokens.TYPE_INTEGER, '42') - t2 = tokens.Token(tokens.TYPE_INTEGER, '1_001_2') - - assert toml2py.deserialize(t1) == 42 - assert toml2py.deserialize(t2) == 10012 - - -def test_float(): - tokens_and_values = ( - ('4.2', 4.2), - ('12e2', 12e2), - ('1_000e2', 1e5), - ('314.1e-2', 3.141) - ) - for token_string, value in tokens_and_values: - token = tokens.Token(tokens.TYPE_FLOAT, token_string) - assert toml2py.deserialize(token) == value - - -def test_string(): - - t0 = tokens.Token(tokens.TYPE_BARE_STRING, 'fawzy') - assert toml2py.deserialize(t0) == 'fawzy' - - t1 = tokens.Token(tokens.TYPE_STRING, '"I\'m a string. \\"You can quote me\\". Name\\tJos\\u00E9\\nLocation\\tSF."') - assert toml2py.deserialize(t1) == u'I\'m a string. "You can quote me". Name\tJos\xe9\nLocation\tSF.' - - t2 = tokens.Token(tokens.TYPE_MULTILINE_STRING, '"""\nRoses are red\nViolets are blue"""') - assert toml2py.deserialize(t2) == 'Roses are red\nViolets are blue' - - t3_str = '"""\nThe quick brown \\\n\n\n fox jumps over \\\n the lazy dog."""' - t3 = tokens.Token(tokens.TYPE_MULTILINE_STRING, t3_str) - assert toml2py.deserialize(t3) == 'The quick brown fox jumps over the lazy dog.' - - t4_str = '"""\\\n The quick brown \\\n fox jumps over \\\n the lazy dog.\\\n """' - t4 = tokens.Token(tokens.TYPE_MULTILINE_STRING, t4_str) - assert toml2py.deserialize(t4) == 'The quick brown fox jumps over the lazy dog.' - - t5 = tokens.Token(tokens.TYPE_LITERAL_STRING, r"'C:\Users\nodejs\templates'") - assert toml2py.deserialize(t5) == r'C:\Users\nodejs\templates' - - t6_str = "'''\nThe first newline is\ntrimmed in raw strings.\n All other whitespace\n is preserved.\n'''" - t6 = tokens.Token(tokens.TYPE_MULTILINE_LITERAL_STRING, t6_str) - assert toml2py.deserialize(t6) == 'The first newline is\ntrimmed in raw strings.\n All' \ - ' other whitespace\n is preserved.\n' - - -def test_date(): - t0 = tokens.Token(tokens.TYPE_DATE, '1979-05-27T07:32:00Z') - assert toml2py.deserialize(t0) == datetime(1979, 5, 27, 7, 32, tzinfo=pytz.utc) - - t1 = tokens.Token(tokens.TYPE_DATE, '1979-05-27T00:32:00-07:00') - assert toml2py.deserialize(t1) == datetime(1979, 5, 27, 7, 32, tzinfo=pytz.utc) - - t3 = tokens.Token(tokens.TYPE_DATE, '1987-07-05T17:45:00') - try: - toml2py.deserialize(t3) - assert False, 'Should detect malformed date' - except DeserializationError: - pass - - -def test_unescaping_a_string(): - - bad_escapes = ( - r"This string has a bad \a escape character.", - r'\x33', - ) - - for source in bad_escapes: - # Should complain about bad escape jobs - try: - toml2py._unescape_str(source) - assert False, "Should have thrown an exception for: " + source - except BadEscapeCharacter: - pass diff --git a/pipenv/patched/prettytoml/tokens/toml2py.py b/pipenv/patched/prettytoml/tokens/toml2py.py deleted file mode 100644 index 2bf9c1c2..00000000 --- a/pipenv/patched/prettytoml/tokens/toml2py.py +++ /dev/null @@ -1,131 +0,0 @@ -import re -import string -import iso8601 -from prettytoml import tokens -from prettytoml.tokens import TYPE_BOOLEAN, TYPE_INTEGER, TYPE_FLOAT, TYPE_DATE, \ - TYPE_MULTILINE_STRING, TYPE_BARE_STRING, TYPE_MULTILINE_LITERAL_STRING, TYPE_LITERAL_STRING, \ - TYPE_STRING -import codecs -import six -from prettytoml.tokens.errors import MalformedDateError -from .errors import BadEscapeCharacter -import functools -import operator - - -def deserialize(token): - """ - Deserializes the value of a single tokens.Token instance based on its type. - - Raises DeserializationError when appropriate. - """ - - if token.type == TYPE_BOOLEAN: - return _to_boolean(token) - elif token.type == TYPE_INTEGER: - return _to_int(token) - elif token.type == TYPE_FLOAT: - return _to_float(token) - elif token.type == TYPE_DATE: - return _to_date(token) - elif token.type in (TYPE_STRING, TYPE_MULTILINE_STRING, TYPE_BARE_STRING, - TYPE_LITERAL_STRING, TYPE_MULTILINE_LITERAL_STRING): - return _to_string(token) - else: - raise Exception('This should never happen!') - - -def _unescape_str(text): - """ - Unescapes a string according the TOML spec. Raises BadEscapeCharacter when appropriate. - """ - - # Detect bad escape jobs - bad_escape_regexp = re.compile(r'([^\\]|^)\\[^btnfr"\\uU]') - if bad_escape_regexp.findall(text): - raise BadEscapeCharacter - - # Do the unescaping - if six.PY2: - return _unicode_escaped_string(text).decode('string-escape').decode('unicode-escape') - else: - return codecs.decode(_unicode_escaped_string(text), 'unicode-escape') - - -def _unicode_escaped_string(text): - """ - Escapes all unicode characters in the given string - """ - - if six.PY2: - text = unicode(text) - - def is_unicode(c): - return c.lower() not in string.ascii_letters + string.whitespace + string.punctuation + string.digits - - def escape_unicode_char(x): - if six.PY2: - return x.encode('unicode-escape') - else: - return codecs.encode(x, 'unicode-escape') - - if any(is_unicode(c) for c in text): - homogeneous_chars = tuple(escape_unicode_char(c) if is_unicode(c) else c.encode() for c in text) - homogeneous_bytes = functools.reduce(operator.add, homogeneous_chars) - return homogeneous_bytes.decode() - else: - return text - - -def _to_string(token): - if token.type == tokens.TYPE_BARE_STRING: - return token.source_substring - - elif token.type == tokens.TYPE_STRING: - escaped = token.source_substring[1:-1] - return _unescape_str(escaped) - - elif token.type == tokens.TYPE_MULTILINE_STRING: - escaped = token.source_substring[3:-3] - - # Drop the first newline if existed - if escaped and escaped[0] == '\n': - escaped = escaped[1:] - - # Remove all occurrences of a slash-newline-zero-or-more-whitespace patterns - escaped = re.sub(r'\\\n\s*', repl='', string=escaped, flags=re.DOTALL) - return _unescape_str(escaped) - - elif token.type == tokens.TYPE_LITERAL_STRING: - return token.source_substring[1:-1] - - elif token.type == tokens.TYPE_MULTILINE_LITERAL_STRING: - text = token.source_substring[3:-3] - if text[0] == '\n': - text = text[1:] - return text - - raise RuntimeError('Control should never reach here.') - - -def _to_int(token): - return int(token.source_substring.replace('_', '')) - - -def _to_float(token): - assert token.type == tokens.TYPE_FLOAT - string = token.source_substring.replace('_', '') - return float(string) - - -def _to_boolean(token): - return token.source_substring == 'true' - - -_correct_date_format = re.compile(r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(Z|(\+|-)\d{2}:\d{2})') - - -def _to_date(token): - if not _correct_date_format.match(token.source_substring): - raise MalformedDateError - return iso8601.parse_date(token.source_substring) diff --git a/pipenv/patched/prettytoml/util.py b/pipenv/patched/prettytoml/util.py deleted file mode 100644 index 155f00e2..00000000 --- a/pipenv/patched/prettytoml/util.py +++ /dev/null @@ -1,141 +0,0 @@ -import math -import itertools - - -def is_sequence_like(x): - """ - Returns True if x exposes a sequence-like interface. - """ - required_attrs = ( - '__len__', - '__getitem__' - ) - return all(hasattr(x, attr) for attr in required_attrs) - - -def is_dict_like(x): - """ - Returns True if x exposes a dict-like interface. - """ - required_attrs = ( - '__len__', - '__getitem__', - 'keys', - 'values', - ) - return all(hasattr(x, attr) for attr in required_attrs) - - -def join_with(iterable, separator): - """ - Joins elements from iterable with separator and returns the produced sequence as a list. - - separator must be addable to a list. - """ - inputs = list(iterable) - b = [] - for i, element in enumerate(inputs): - if isinstance(element, (list, tuple, set)): - b += tuple(element) - else: - b += [element] - if i < len(inputs)-1: - b += separator - return b - - -def chunkate_string(text, length): - """ - Iterates over the given seq in chunks of at maximally the given length. Will never break a whole word. - """ - iterator_index = 0 - - def next_newline(): - try: - return next(i for (i, c) in enumerate(text) if i > iterator_index and c == '\n') - except StopIteration: - return len(text) - - def next_breaker(): - try: - return next(i for (i, c) in reversed(tuple(enumerate(text))) - if i >= iterator_index and - (i < iterator_index+length) and - c in (' ', '\t')) - except StopIteration: - return len(text) - - while iterator_index < len(text): - next_chunk = text[iterator_index:min(next_newline(), next_breaker()+1)] - iterator_index += len(next_chunk) - yield next_chunk - - -def flatten_nested(nested_dicts): - """ - Flattens dicts and sequences into one dict with tuples of keys representing the nested keys. - - Example - >>> dd = { \ - 'dict1': {'name': 'Jon', 'id': 42}, \ - 'dict2': {'name': 'Sam', 'id': 41}, \ - 'seq1': [{'one': 1, 'two': 2}] \ - } - - >>> flatten_nested(dd) == { \ - ('dict1', 'name'): 'Jon', ('dict1', 'id'): 42, \ - ('dict2', 'name'): 'Sam', ('dict2', 'id'): 41, \ - ('seq1', 0, 'one'): 1, ('seq1', 0, 'two'): 2, \ - } - True - """ - assert isinstance(nested_dicts, (dict, list, tuple)), 'Only works with a collection parameter' - - def items(c): - if isinstance(c, dict): - return c.items() - elif isinstance(c, (list, tuple)): - return enumerate(c) - else: - raise RuntimeError('c must be a collection') - - def flatten(dd): - output = {} - for k, v in items(dd): - if isinstance(v, (dict, list, tuple)): - for child_key, child_value in flatten(v).items(): - output[(k,) + child_key] = child_value - else: - output[(k,)] = v - return output - - return flatten(nested_dicts) - - -class PeekableIterator: - - # Returned by peek() when the iterator is exhausted. Truthiness is False. - Nothing = tuple() - - def __init__(self, iter): - self._iter = iter - - def __next__(self): - return next(self._iter) - - def next(self): - return self.__next__() - - def __iter__(self): - return self - - def peek(self): - """ - Returns PeekableIterator.Nothing when the iterator is exhausted. - """ - try: - v = next(self._iter) - self._iter = itertools.chain((v,), self._iter) - return v - except StopIteration: - return PeekableIterator.Nothing diff --git a/pipenv/pipenv.1 b/pipenv/pipenv.1 index 62808744..0876d34f 100644 --- a/pipenv/pipenv.1 +++ b/pipenv/pipenv.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "PIPENV" "1" "Oct 07, 2017" "8.2.7" "pipenv" +.TH "PIPENV" "1" "Nov 18, 2018" "2018.11.15.dev0" "pipenv" .SH NAME pipenv \- pipenv Documentation . @@ -38,39 +38,58 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .ce 0 .sp .sp -\fBPipenv\fP —\ the officially recommended Python packaging tool from \fI\%Python.org\fP, free (as in freedom). +\fBPipenv\fP is a tool that aims to bring the best of all packaging worlds (bundler, composer, npm, cargo, yarn, etc.) to the Python world. \fIWindows is a first\-class citizen, in our world.\fP .sp -Pipenv is a tool that aims to bring the best of all packaging worlds (bundler, composer, npm, cargo, yarn, etc.) to the Python world. \fIWindows is a first-class citizen, in our world.\fP +It automatically creates and manages a virtualenv for your projects, as well as adds/removes packages from your \fBPipfile\fP as you install/uninstall packages. It also generates the ever\-important \fBPipfile.lock\fP, which is used to produce deterministic builds. .sp -It automatically creates and manages a virtualenv for your projects, as well as adds/removes packages from your \fBPipfile\fP as you install/uninstall packages. It also generates the ever-important \fBPipfile.lock\fP, which is used to produce deterministic builds. +Pipenv is primarily meant to provide users and developers of applications with an easy method to setup a working environment. For the distinction between libraries and applications and the usage of \fBsetup.py\fP vs \fBPipfile\fP to define dependencies, see pipfile\-vs\-setuppy\&. .sp The problems that Pipenv seeks to solve are multi\-faceted: .INDENT 0.0 .IP \(bu 2 You no longer need to use \fBpip\fP and \fBvirtualenv\fP separately. They work together. .IP \(bu 2 -Managing a \fBrequirements.txt\fP file \fI\%can be problematic\fP, so Pipenv uses the upcoming \fBPipfile\fP and \fBPipfile.lock\fP instead, which is superior for basic use cases. +Managing a \fBrequirements.txt\fP file \fI\%can be problematic\fP, so Pipenv uses \fBPipfile\fP and \fBPipfile.lock\fP to separate abstract dependency declarations from the last tested combination. .IP \(bu 2 Hashes are used everywhere, always. Security. Automatically expose security vulnerabilities. .IP \(bu 2 +Strongly encourage the use of the latest versions of dependencies to minimize security risks \fI\%arising from outdated components\fP\&. +.IP \(bu 2 Give you insight into your dependency graph (e.g. \fB$ pipenv graph\fP). .IP \(bu 2 Streamline development workflow by loading \fB\&.env\fP files. .UNINDENT -.SH INSTALL PIPENV TODAY! +.sp +You can quickly play with Pipenv right in your browser: +\fI\%Try in browser\fP.SH INSTALL PIPENV TODAY! +.sp +If you\(aqre on MacOS, you can install Pipenv easily with Homebrew: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -$ pip install pipenv -✨🍰✨ +$ brew install pipenv .ft P .fi .UNINDENT .UNINDENT .sp -If you have excellent taste, there\(aqs also a \fI\%fancy installation method\fP\&. +Or, if you\(aqre using Fedora 28: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ sudo dnf install pipenv +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Otherwise, refer to the installing\-pipenv chapter for instructions. +.sp +✨🍰✨ .SS Pipenv & Virtual Environments [image] .sp @@ -91,7 +110,7 @@ This guide is written for Python 3, however, these instructions should work fine on Python 2.7—if you are still using it, for some reason. .UNINDENT .UNINDENT -.SS ☤ Make sure you\(aqve got Python & pip +.SS ☤ Make sure you\(aqve got Python & pip .sp Before you go any further, make sure you have Python and that it\(aqs available from your command line. You can check this by simply running: @@ -150,8 +169,11 @@ pip 9.0.1 .UNINDENT .sp If you installed Python from source, with an installer from \fI\%python.org\fP, or -via \fI\%Homebrew\fP you should already have pip9. If you\(aqre on Linux and installed +via \fI\%Homebrew\fP you should already have pip. If you\(aqre on Linux and installed using your OS package manager, you may have to \fI\%install pip\fP separately. +.sp +If you plan to install Pipenv using Homebrew you can skip this step. The +Homebrew installer takes care of pip for you. .SS ☤ Installing Pipenv .sp Pipenv is a dependency manager for Python projects. If you\(aqre familiar @@ -159,8 +181,42 @@ with Node.js\(aq \fI\%npm\fP or Ruby\(aqs \fI\%bundler\fP, it is similar in spir tools. While pip can install Python packages, Pipenv is recommended as it\(aqs a higher\-level tool that simplifies dependency management for common use cases. +.SS ☤ Homebrew Installation of Pipenv .sp -Use \fBpip\fP to install Pipenv: +Homebrew is a popular open\-source package management system for macOS. +.sp +Installing pipenv via Homebrew will keep pipenv and all of its dependencies in +an isolated virtual environment so it doesn\(aqt interfere with the rest of your +Python installation. +.sp +Once you have installed \fI\%Homebrew\fP simply run: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ brew install pipenv +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +To upgrade pipenv at any time: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ brew upgrade pipenv +.ft P +.fi +.UNINDENT +.UNINDENT +.SS ☤ Pragmatic Installation of Pipenv +.sp +If you have a working installation of pip, and maintain certain "toolchain" type Python modules as global utilities in your user environment, pip \fI\%user installs\fP allow for installation into your home directory. Note that due to interaction between dependencies, you should limit tools installed in this way to basic building blocks for a Python workflow like virtualenv, pipenv, tox, and similar software. +.sp +To install: .INDENT 0.0 .INDENT 3.5 .sp @@ -187,13 +243,40 @@ absolute path to your home directory) so you\(aqll need to add \fI\%modifying ~/.profile\fP\&. .sp On Windows you can find the user base binary directory by running -\fBpy \-m site \-\-user\-site\fP and replacing \fBsite\-packages\fP with +\fBpython \-m site \-\-user\-site\fP and replacing \fBsite\-packages\fP with \fBScripts\fP\&. For example, this could return \fBC:\eUsers\eUsername\eAppData\eRoaming\ePython36\esite\-packages\fP so you would need to set your \fBPATH\fP to include \fBC:\eUsers\eUsername\eAppData\eRoaming\ePython36\eScripts\fP\&. You can set your user \fBPATH\fP permanently in the \fI\%Control Panel\fP\&. You may need to log out for the \fBPATH\fP changes to take effect. +.sp +For more information, see the \fI\%user installs documentation\fP\&. +.UNINDENT +.UNINDENT +.sp +To upgrade pipenv at any time: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ pip install \-\-user \-\-upgrade pipenv +.ft P +.fi +.UNINDENT +.UNINDENT +.SS ☤ Crude Installation of Pipenv +.sp +If you don\(aqt even have pip installed, you can use this crude installation method, which will bootstrap your whole system: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ curl https://raw.githubusercontent.com/kennethreitz/pipenv/master/get\-pipenv.py | python +.ft P +.fi .UNINDENT .UNINDENT .SS ☤ Installing packages for your project @@ -214,7 +297,7 @@ $ pipenv install requests .UNINDENT .sp Pipenv will install the excellent \fI\%Requests\fP library and create a \fBPipfile\fP -for you in your project\(aqs directory. The Pipfile is used to track which +for you in your project\(aqs directory. The \fBPipfile\fP is used to track which dependencies your project needs in case you need to re\-install them, such as when you share your project with others. You should get output similar to this (although the exact paths shown will vary): @@ -297,119 +380,742 @@ Your IP is 8.8.8.8 Using \fB$ pipenv run\fP ensures that your installed packages are available to your script. It\(aqs also possible to spawn a new shell that ensures all commands have access to your installed packages with \fB$ pipenv shell\fP\&. +.SS ☤ Virtualenv mapping caveat +.INDENT 0.0 +.IP \(bu 2 +Pipenv automatically maps projects to their specific virtualenvs. +.IP \(bu 2 +The virtualenv is stored globally with the name of the project’s root directory plus the hash of the full path to the project\(aqs root (e.g., \fBmy_project\-a3de50\fP). +.IP \(bu 2 +If you change your project\(aqs path, you break such a default mapping and pipenv will no longer be able to find and to use the project\(aqs virtualenv. +.IP \(bu 2 +You might want to set \fBexport PIPENV_VENV_IN_PROJECT=1\fP in your .bashrc/.zshrc (or any shell configuration file) for creating the virtualenv inside your project\(aqs directory, avoiding problems with subsequent path changes. +.UNINDENT .SS ☤ Next steps .sp Congratulations, you now know how to install and use Python packages! ✨ 🍰 ✨ -.SS ☤ Fancy Installation of Pipenv -.sp -To install pipenv in a fancy way, we recommend using \fI\%pipsi\fP\&. -.sp -Pipsi is a powerful tool which allows you to install Python scripts into isolated virtual environments. -.sp -To install pipsi, first run this: +.SS Release and Version History +.SS 2018.11.14 (2018\-11\-14) +.SS Features & Improvements .INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -$ curl https://raw.githubusercontent.com/mitsuhiko/pipsi/master/get\-pipsi.py | python -.ft P -.fi +.IP \(bu 2 +Improved exceptions and error handling on failures. \fI\%#1977\fP +.IP \(bu 2 +Added persistent settings for all CLI flags via \fBPIPENV_{FLAG_NAME}\fP environment variables by enabling \fBauto_envvar_prefix=PIPENV\fP in click (implements PEEP\-0002). \fI\%#2200\fP +.IP \(bu 2 +Added improved messaging about available but skipped updates due to dependency conflicts when running \fBpipenv update \-\-outdated\fP\&. \fI\%#2411\fP +.IP \(bu 2 +Added environment variable \fIPIPENV_PYUP_API_KEY\fP to add ability +to override the bundled pyup.io API key. \fI\%#2825\fP +.IP \(bu 2 +Added additional output to \fBpipenv update \-\-outdated\fP to indicate that the operation succeded and all packages were already up to date. \fI\%#2828\fP +.IP \(bu 2 +Updated \fBcrayons\fP patch to enable colors on native powershell but swap native blue for magenta. \fI\%#3020\fP +.IP \(bu 2 +Added support for \fB\-\-bare\fP to \fBpipenv clean\fP, and fixed \fBpipenv sync \-\-bare\fP to actually reduce output. \fI\%#3041\fP +.IP \(bu 2 +Added windows\-compatible spinner via upgraded \fBvistir\fP dependency. \fI\%#3089\fP +.IP \(bu 2 +.INDENT 2.0 +.IP \(bu 2 +Added support for python installations managed by \fBasdf\fP\&. \fI\%#3096\fP .UNINDENT +.IP \(bu 2 +Improved runtime performance of no\-op commands such as \fBpipenv \-\-venv\fP by around 2/3. \fI\%#3158\fP +.IP \(bu 2 +Do not show error but success for running \fBpipenv uninstall \-\-all\fP in a fresh virtual environment. \fI\%#3170\fP +.IP \(bu 2 +Improved asynchronous installation and error handling via queued subprocess paralleization. \fI\%#3217\fP .UNINDENT -.sp -Follow the instructions, you\(aqll have to update your \fBPATH\fP\&. -.sp -Then, simply run: +.SS Bug Fixes .INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -$ pipsi install pipenv -.ft P -.fi +.IP \(bu 2 +Remote non\-PyPI artifacts and local wheels and artifacts will now include their own hashes rather than including hashes from \fBPyPI\fP\&. \fI\%#2394\fP +.IP \(bu 2 +Non\-ascii characters will now be handled correctly when parsed by pipenv\(aqs \fBToML\fP parsers. \fI\%#2737\fP +.IP \(bu 2 +Updated \fBpipenv uninstall\fP to respect the \fB\-\-skip\-lock\fP argument. \fI\%#2848\fP +.IP \(bu 2 +Fixed a bug which caused uninstallation to sometimes fail to successfullly remove packages from \fBPipfiles\fP with comments on preceding or following lines. \fI\%#2885\fP, +\fI\%#3099\fP +.IP \(bu 2 +Pipenv will no longer fail when encountering python versions on Windows that have been uninstalled. \fI\%#2983\fP +.IP \(bu 2 +Fixed unnecessary extras are added when translating markers \fI\%#3026\fP +.IP \(bu 2 +Fixed a virtualenv creation issue which could cause new virtualenvs to inadvertently attempt to read and write to global site packages. \fI\%#3047\fP +.IP \(bu 2 +Fixed an issue with virtualenv path derivation which could cause errors, particularly for users on WSL bash. \fI\%#3055\fP +.IP \(bu 2 +Fixed a bug which caused \fBUnexpected EOF\fP errors to be thrown when \fBpip\fP was waiting for input from users who had put login credentials in environment variables. \fI\%#3088\fP +.IP \(bu 2 +Fixed a bug in \fBrequirementslib\fP which prevented successful installation from mercurial repositories. \fI\%#3090\fP +.IP \(bu 2 +Fixed random resource warnings when using pyenv or any other subprocess calls. \fI\%#3094\fP +.IP \(bu 2 +.INDENT 2.0 +.IP \(bu 2 +Fixed a bug which sometimes prevented cloning and parsing \fBmercurial\fP requirements. \fI\%#3096\fP .UNINDENT +.IP \(bu 2 +Fixed an issue in \fBdelegator.py\fP related to subprocess calls when using \fBPopenSpawn\fP to stream output, which sometimes threw unexpected \fBEOF\fP errors. \fI\%#3102\fP, +\fI\%#3114\fP, +\fI\%#3117\fP +.IP \(bu 2 +Fix the path casing issue that makes \fIpipenv clean\fP fail on Windows \fI\%#3104\fP +.IP \(bu 2 +Pipenv will avoid leaving build artifacts in the current working directory. \fI\%#3106\fP +.IP \(bu 2 +Fixed issues with broken subprocess calls leaking resource handles and causing random and sporadic failures. \fI\%#3109\fP +.IP \(bu 2 +Fixed an issue which caused \fBpipenv clean\fP to sometimes clean packages from the base \fBsite\-packages\fP folder or fail entirely. \fI\%#3113\fP +.IP \(bu 2 +Updated \fBpythonfinder\fP to correct an issue with unnesting of nested paths when searching for python versions. \fI\%#3121\fP +.IP \(bu 2 +Added additional logic for ignoring and replacing non\-ascii characters when formatting console output on non\-UTF\-8 systems. \fI\%#3131\fP +.IP \(bu 2 +Fix virtual environment discovery when \fIPIPENV_VENV_IN_PROJECT\fP is set, but the in\-project \fI\&.venv\fP is a file. \fI\%#3134\fP +.IP \(bu 2 +Hashes for remote and local non\-PyPI artifacts will now be included in \fBPipfile.lock\fP during resolution. \fI\%#3145\fP +.IP \(bu 2 +Fix project path hashing logic in purpose to prevent collisions of virtual environments. \fI\%#3151\fP +.IP \(bu 2 +Fix package installation when the virtual environment path contains parentheses. \fI\%#3158\fP +.IP \(bu 2 +Azure Pipelines YAML files are updated to use the latest syntax and product name. \fI\%#3164\fP +.IP \(bu 2 +Fixed new spinner success message to write only one success message during resolution. \fI\%#3183\fP +.IP \(bu 2 +Pipenv will now correctly respect the \fB\-\-pre\fP option when used with \fBpipenv install\fP\&. \fI\%#3185\fP +.IP \(bu 2 +Fix a bug where exception is raised when run pipenv graph in a project without created virtualenv \fI\%#3201\fP +.IP \(bu 2 +When sources are missing names, names will now be derived from the supplied URL. \fI\%#3216\fP .UNINDENT -.sp -To upgrade pipenv at any time: +.SS Vendored Libraries .INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -$ pipsi upgrade pipenv -.ft P -.fi +.IP \(bu 2 +Updated \fBpythonfinder\fP to correct an issue with unnesting of nested paths when searching for python versions. \fI\%#3061\fP, +\fI\%#3121\fP +.IP \(bu 2 +.INDENT 2.0 +.TP +.B Updated vendored dependencies: +.INDENT 7.0 +.IP \(bu 2 +\fBcertifi 2018.08.24 => 2018.10.15\fP +.IP \(bu 2 +\fBurllib3 1.23 => 1.24\fP +.IP \(bu 2 +\fBrequests 2.19.1 => 2.20.0\fP +.IP \(bu 2 +\fBshellingham \(ga\(ga1.2.6 => 1.2.7\fP +.IP \(bu 2 +\fBtomlkit 0.4.4. => 0.4.6\fP +.IP \(bu 2 +\fBvistir 0.1.6 => 0.1.8\fP +.IP \(bu 2 +\fBpythonfinder 0.1.2 => 0.1.3\fP +.IP \(bu 2 +\fBrequirementslib 1.1.9 => 1.1.10\fP +.IP \(bu 2 +\fBbackports.functools_lru_cache 1.5.0 (new)\fP +.IP \(bu 2 +\fBcursor 1.2.0 (new)\fP \fI\%#3089\fP .UNINDENT .UNINDENT -.sp -This will install \fBpipenv\fP in an isolated virtualenv, so it doesn\(aqt interfere with the rest of your Python installation! -.SS ☤ Pragmatic Installation of Pipenv -.sp -If you have a working installation of pip, and maintain certain "toolchain" type Python modules as global utilities in your user environment, pip \fI\%user installs\fP allow for installation into your home directory. Note that due to interaction between dependencies, you should limit tools installed in this way to basic building blocks for a Python workflow like virtualenv, pipenv, tox, and similar software. -.sp -To install: +.IP \(bu 2 +.INDENT 2.0 +.TP +.B Updated vendored dependencies: +.INDENT 7.0 +.IP \(bu 2 +\fBrequests 2.19.1 => 2.20.1\fP +.IP \(bu 2 +\fBtomlkit 0.4.46 => 0.5.2\fP +.IP \(bu 2 +\fBvistir 0.1.6 => 0.2.4\fP +.IP \(bu 2 +\fBpythonfinder 1.1.2 => 1.1.8\fP +.IP \(bu 2 +\fBrequirementslib 1.1.10 => 1.3.0\fP \fI\%#3096\fP +.UNINDENT +.UNINDENT +.IP \(bu 2 +Switch to \fBtomlkit\fP for parsing and writing. Drop \fBprettytoml\fP and \fBcontoml\fP from vendors. \fI\%#3191\fP +.IP \(bu 2 +Updated \fBrequirementslib\fP to aid in resolution of local and remote archives. \fI\%#3196\fP +.UNINDENT +.SS Improved Documentation .INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -$ pip install \-\-user pipenv -.ft P -.fi +.IP \(bu 2 +Expanded development and testing documentation for contributors to get started. \fI\%#3074\fP .UNINDENT -.UNINDENT -.sp -For more information see the \fI\%user installs documentation\fP, but to add the installed cli tools from a pip user install to your path, add the output of: +.SS 2018.10.13 (2018\-10\-13) +.SS Bug Fixes .INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -$ python \-c "import site; import os; print(os.path.join(site.USER_BASE, \(aqbin\(aq))" -.ft P -.fi +.IP \(bu 2 +Fixed a bug in \fBpipenv clean\fP which caused global packages to sometimes be inadvertently targeted for cleanup. \fI\%#2849\fP +.IP \(bu 2 +Fix broken backport imports for vendored vistir. \fI\%#2950\fP, +\fI\%#2955\fP, +\fI\%#2961\fP +.IP \(bu 2 +Fixed a bug with importing local vendored dependencies when running \fBpipenv graph\fP\&. \fI\%#2952\fP +.IP \(bu 2 +Fixed a bug which caused executable discovery to fail when running inside a virtualenv. \fI\%#2957\fP +.IP \(bu 2 +Fix parsing of outline tables. \fI\%#2971\fP +.IP \(bu 2 +Fixed a bug which caused \fBverify_ssl\fP to fail to drop through to \fBpip install\fP correctly as \fBtrusted\-host\fP\&. \fI\%#2979\fP +.IP \(bu 2 +Fixed a bug which caused canonicalized package names to fail to resolve against PyPI. \fI\%#2989\fP +.IP \(bu 2 +Enhanced CI detection to detect Azure Devops builds. \fI\%#2993\fP +.IP \(bu 2 +Fixed a bug which prevented installing pinned versions which used redirection symbols from the command line. \fI\%#2998\fP +.IP \(bu 2 +Fixed a bug which prevented installing the local directory in non\-editable mode. \fI\%#3005\fP .UNINDENT -.UNINDENT -.sp -To upgrade pipenv at any time: +.SS Vendored Libraries .INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -$ pip install \-\-user \-\-upgrade pipenv -.ft P -.fi +.IP \(bu 2 +Updated \fBrequirementslib\fP to version \fB1.1.9\fP\&. \fI\%#2989\fP +.IP \(bu 2 +Upgraded \fBpythonfinder => 1.1.1\fP and \fBvistir => 0.1.7\fP\&. \fI\%#3007\fP .UNINDENT -.UNINDENT -.SS ☤ Crude Installation of Pipenv -.sp -If you don\(aqt even have pip installed, you can use this crude installation method, which will bootstrap your whole system: +.SS 2018.10.9 (2018\-10\-09) +.SS Features & Improvements .INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -$ curl https://raw.githubusercontent.com/kennethreitz/pipenv/master/get\-pipenv.py | python -.ft P -.fi +.IP \(bu 2 +Added environment variables \fIPIPENV_VERBOSE\fP and \fIPIPENV_QUIET\fP to control +output verbosity without needing to pass options. \fI\%#2527\fP +.IP \(bu 2 +Updated test\-pypi addon to better support json\-api access (forward compatibility). +Improved testing process for new contributors. \fI\%#2568\fP +.IP \(bu 2 +Greatly enhanced python discovery functionality: +.INDENT 2.0 +.IP \(bu 2 +Added pep514 (windows launcher/finder) support for python discovery. +.IP \(bu 2 +Introduced architecture discovery for python installations which support different architectures. \fI\%#2582\fP +.UNINDENT +.IP \(bu 2 +Added support for \fBpipenv shell\fP on msys and cygwin/mingw/git bash for Windows. \fI\%#2641\fP +.IP \(bu 2 +Enhanced resolution of editable and VCS dependencies. \fI\%#2643\fP +.IP \(bu 2 +Deduplicate and refactor CLI to use stateful arguments and object passing. See \fI\%this issue\fP for reference. \fI\%#2814\fP +.UNINDENT +.SS Behavior Changes +.INDENT 0.0 +.IP \(bu 2 +Virtual environment activation for \fBrun\fP is revised to improve interpolation +with other Python discovery tools. \fI\%#2503\fP +.IP \(bu 2 +Improve terminal coloring to display better in Powershell. \fI\%#2511\fP +.IP \(bu 2 +Invoke \fBvirtualenv\fP directly for virtual environment creation, instead of depending on \fBpew\fP\&. \fI\%#2518\fP +.IP \(bu 2 +\fBpipenv \-\-help\fP will now include short help descriptions. \fI\%#2542\fP +.IP \(bu 2 +Add \fBCOMSPEC\fP to fallback option (along with \fBSHELL\fP and \fBPYENV_SHELL\fP) +if shell detection fails, improving robustness on Windows. \fI\%#2651\fP +.IP \(bu 2 +Fallback to shell mode if \fIrun\fP fails with Windows error 193 to handle non\-executable commands. This should improve usability on Windows, where some users run non\-executable files without specifying a command, relying on Windows file association to choose the current command. \fI\%#2718\fP +.UNINDENT +.SS Bug Fixes +.INDENT 0.0 +.IP \(bu 2 +Fixed a bug which prevented installation of editable requirements using \fBssh://\fP style urls \fI\%#1393\fP +.IP \(bu 2 +VCS Refs for locked local editable dependencies will now update appropriately to the latest hash when running \fBpipenv update\fP\&. \fI\%#1690\fP +.IP \(bu 2 +\fB\&.tar.gz\fP and \fB\&.zip\fP artifacts will now have dependencies installed even when they are missing from the lockfile. \fI\%#2173\fP +.IP \(bu 2 +The command line parser will now handle multiple \fB\-e/\-\-editable\fP dependencies properly via click\(aqs option parser to help mitigate future parsing issues. \fI\%#2279\fP +.IP \(bu 2 +Fixed the ability of pipenv to parse \fBdependency_links\fP from \fBsetup.py\fP when \fBPIP_PROCESS_DEPENDENCY_LINKS\fP is enabled. \fI\%#2434\fP +.IP \(bu 2 +Fixed a bug which could cause \fB\-i/\-\-index\fP arguments to sometimes be incorrectly picked up in packages. This is now handled in the command line parser. \fI\%#2494\fP +.IP \(bu 2 +Fixed non\-deterministic resolution issues related to changes to the internal package finder in \fBpip 10\fP\&. \fI\%#2499\fP, +\fI\%#2529\fP, +\fI\%#2589\fP, +\fI\%#2666\fP, +\fI\%#2767\fP, +\fI\%#2785\fP, +\fI\%#2795\fP, +\fI\%#2801\fP, +\fI\%#2824\fP, +\fI\%#2862\fP, +\fI\%#2879\fP, +\fI\%#2894\fP, +\fI\%#2933\fP +.IP \(bu 2 +Fix subshell invocation on Windows for Python 2. \fI\%#2515\fP +.IP \(bu 2 +Fixed a bug which sometimes caused pipenv to throw a \fBTypeError\fP or to run into encoding issues when writing lockfiles on python 2. \fI\%#2561\fP +.IP \(bu 2 +Improve quoting logic for \fBpipenv run\fP so it works better with Windows +built\-in commands. \fI\%#2563\fP +.IP \(bu 2 +Fixed a bug related to parsing vcs requirements with both extras and subdirectory fragments. +Corrected an issue in the \fBrequirementslib\fP parser which led to some markers being discarded rather than evaluated. \fI\%#2564\fP +.IP \(bu 2 +Fixed multiple issues with finding the correct system python locations. \fI\%#2582\fP +.IP \(bu 2 +Catch JSON decoding error to prevent exception when the lock file is of +invalid format. \fI\%#2607\fP +.IP \(bu 2 +Fixed a rare bug which could sometimes cause errors when installing packages with custom sources. \fI\%#2610\fP +.IP \(bu 2 +Update requirementslib to fix a bug which could raise an \fBUnboundLocalError\fP when parsing malformed VCS URIs. \fI\%#2617\fP +.IP \(bu 2 +Fixed an issue which prevented passing multiple \fB\-\-ignore\fP parameters to \fBpipenv check\fP\&. \fI\%#2632\fP +.IP \(bu 2 +Fixed a bug which caused attempted hashing of \fBssh://\fP style URIs which could cause failures during installation of private ssh repositories. +\- Corrected path conversion issues which caused certain editable VCS paths to be converted to \fBssh://\fP URIs improperly. \fI\%#2639\fP +.IP \(bu 2 +Fixed a bug which caused paths to be formatted incorrectly when using \fBpipenv shell\fP in bash for windows. \fI\%#2641\fP +.IP \(bu 2 +Dependency links to private repositories defined via \fBssh://\fP schemes will now install correctly and skip hashing as long as \fBPIP_PROCESS_DEPENDENCY_LINKS=1\fP\&. \fI\%#2643\fP +.IP \(bu 2 +Fixed a bug which sometimes caused pipenv to parse the \fBtrusted_host\fP argument to pip incorrectly when parsing source URLs which specify \fBverify_ssl = false\fP\&. \fI\%#2656\fP +.IP \(bu 2 +Prevent crashing when a virtual environment in \fBWORKON_HOME\fP is faulty. \fI\%#2676\fP +.IP \(bu 2 +Fixed virtualenv creation failure when a .venv file is present in the project root. \fI\%#2680\fP +.IP \(bu 2 +Fixed a bug which could cause the \fB\-e/\-\-editable\fP argument on a dependency to be accidentally parsed as a dependency itself. \fI\%#2714\fP +.IP \(bu 2 +Correctly pass \fIverbose\fP and \fIdebug\fP flags to the resolver subprocess so it generates appropriate output. This also resolves a bug introduced by the fix to #2527. \fI\%#2732\fP +.IP \(bu 2 +All markers are now included in \fBpipenv lock \-\-requirements\fP output. \fI\%#2748\fP +.IP \(bu 2 +Fixed a bug in marker resolution which could cause duplicate and non\-deterministic markers. \fI\%#2760\fP +.IP \(bu 2 +Fixed a bug in the dependency resolver which caused regular issues when handling \fBsetup.py\fP based dependency resolution. \fI\%#2766\fP +.IP \(bu 2 +.INDENT 2.0 +.TP +.B Updated vendored dependencies: +.INDENT 7.0 +.IP \(bu 2 +\fBpip\-tools\fP (updated and patched to latest w/ \fBpip 18.0\fP compatibilty) +.IP \(bu 2 +\fBpip 10.0.1 => 18.0\fP +.IP \(bu 2 +\fBclick 6.7 => 7.0\fP +.IP \(bu 2 +\fBtoml 0.9.4 => 0.10.0\fP +.IP \(bu 2 +\fBpyparsing 2.2.0 => 2.2.2\fP +.IP \(bu 2 +\fBdelegator 0.1.0 => 0.1.1\fP +.IP \(bu 2 +\fBattrs 18.1.0 => 18.2.0\fP +.IP \(bu 2 +\fBdistlib 0.2.7 => 0.2.8\fP +.IP \(bu 2 +\fBpackaging 17.1.0 => 18.0\fP +.IP \(bu 2 +\fBpassa 0.2.0 => 0.3.1\fP +.IP \(bu 2 +\fBpip_shims 0.1.2 => 0.3.1\fP +.IP \(bu 2 +\fBplette 0.1.1 => 0.2.2\fP +.IP \(bu 2 +\fBpythonfinder 1.0.2 => 1.1.0\fP +.IP \(bu 2 +\fBpytoml 0.1.18 => 0.1.19\fP +.IP \(bu 2 +\fBrequirementslib 1.1.16 => 1.1.17\fP +.IP \(bu 2 +\fBshellingham 1.2.4 => 1.2.6\fP +.IP \(bu 2 +\fBtomlkit 0.4.2 => 0.4.4\fP +.IP \(bu 2 +\fBvistir 0.1.4 => 0.1.6\fP \fI\%#2802\fP, .UNINDENT .UNINDENT .sp -Congratulations, you now have pip and Pipenv installed! +\fI\%#2867\fP, +\fI\%#2880\fP +.IP \(bu 2 +Fixed a bug where \fIpipenv\fP crashes when the \fIWORKON_HOME\fP directory does not exist. \fI\%#2877\fP +.IP \(bu 2 +Fixed pip is not loaded from pipenv\(aqs patched one but the system one \fI\%#2912\fP +.IP \(bu 2 +Fixed various bugs related to \fBpip 18.1\fP release which prevented locking, installation, and syncing, and dumping to a \fBrequirements.txt\fP file. \fI\%#2924\fP +.UNINDENT +.SS Vendored Libraries +.INDENT 0.0 +.IP \(bu 2 +Pew is no longer vendored. Entry point \fBpewtwo\fP, packages \fBpipenv.pew\fP and +\fBpipenv.patched.pew\fP are removed. \fI\%#2521\fP +.IP \(bu 2 +Update \fBpythonfinder\fP to major release \fB1.0.0\fP for integration. \fI\%#2582\fP +.IP \(bu 2 +Update requirementslib to fix a bug which could raise an \fBUnboundLocalError\fP when parsing malformed VCS URIs. \fI\%#2617\fP +.IP \(bu 2 +.INDENT 2.0 +.IP \(bu 2 +Vendored new libraries \fBvistir\fP and \fBpip\-shims\fP, \fBtomlkit\fP, \fBmodutil\fP, and \fBplette\fP\&. +.IP \(bu 2 +Update vendored libraries: +\- \fBscandir\fP to \fB1.9.0\fP +\- \fBclick\-completion\fP to \fB0.4.1\fP +\- \fBsemver\fP to \fB2.8.1\fP +\- \fBshellingham\fP to \fB1.2.4\fP +\- \fBpytoml\fP to \fB0.1.18\fP +\- \fBcertifi\fP to \fB2018.8.24\fP +\- \fBptyprocess\fP to \fB0.6.0\fP +\- \fBrequirementslib\fP to \fB1.1.5\fP +\- \fBpythonfinder\fP to \fB1.0.2\fP +\- \fBpipdeptree\fP to \fB0.13.0\fP +\- \fBpython\-dotenv\fP to \fB0.9.1\fP \fI\%#2639\fP +.UNINDENT +.IP \(bu 2 +.INDENT 2.0 +.TP +.B Updated vendored dependencies: +.INDENT 7.0 +.IP \(bu 2 +\fBpip\-tools\fP (updated and patched to latest w/ \fBpip 18.0\fP compatibilty) +.IP \(bu 2 +\fBpip 10.0.1 => 18.0\fP +.IP \(bu 2 +\fBclick 6.7 => 7.0\fP +.IP \(bu 2 +\fBtoml 0.9.4 => 0.10.0\fP +.IP \(bu 2 +\fBpyparsing 2.2.0 => 2.2.2\fP +.IP \(bu 2 +\fBdelegator 0.1.0 => 0.1.1\fP +.IP \(bu 2 +\fBattrs 18.1.0 => 18.2.0\fP +.IP \(bu 2 +\fBdistlib 0.2.7 => 0.2.8\fP +.IP \(bu 2 +\fBpackaging 17.1.0 => 18.0\fP +.IP \(bu 2 +\fBpassa 0.2.0 => 0.3.1\fP +.IP \(bu 2 +\fBpip_shims 0.1.2 => 0.3.1\fP +.IP \(bu 2 +\fBplette 0.1.1 => 0.2.2\fP +.IP \(bu 2 +\fBpythonfinder 1.0.2 => 1.1.0\fP +.IP \(bu 2 +\fBpytoml 0.1.18 => 0.1.19\fP +.IP \(bu 2 +\fBrequirementslib 1.1.16 => 1.1.17\fP +.IP \(bu 2 +\fBshellingham 1.2.4 => 1.2.6\fP +.IP \(bu 2 +\fBtomlkit 0.4.2 => 0.4.4\fP +.IP \(bu 2 +\fBvistir 0.1.4 => 0.1.6\fP \fI\%#2902\fP, +.UNINDENT +.UNINDENT +.sp +\fI\%#2935\fP +.UNINDENT +.SS Improved Documentation +.INDENT 0.0 +.IP \(bu 2 +Simplified the test configuration process. \fI\%#2568\fP +.IP \(bu 2 +Updated documentation to use working fortune cookie addon. \fI\%#2644\fP +.IP \(bu 2 +Added additional information about troubleshooting \fBpipenv shell\fP by using the the \fB$PIPENV_SHELL\fP environment variable. \fI\%#2671\fP +.IP \(bu 2 +Added a link to \fBPEP\-440\fP version specifiers in the documentation for additional detail. \fI\%#2674\fP +.IP \(bu 2 +Added simple example to README.md for installing from git. \fI\%#2685\fP +.IP \(bu 2 +Stopped recommending \fI\-\-system\fP for Docker contexts. \fI\%#2762\fP +.IP \(bu 2 +Fixed the example url for doing "pipenv install \-e +some\-repo\-url#egg=something", it was missing the "egg=" in the fragment +identifier. \fI\%#2792\fP +.IP \(bu 2 +Fixed link to the "be cordial" essay in the contribution documentation. \fI\%#2793\fP +.IP \(bu 2 +Clarify \fIpipenv install\fP documentation \fI\%#2844\fP +.IP \(bu 2 +Replace reference to uservoice with PEEP\-000 \fI\%#2909\fP +.UNINDENT +.SS 2018.7.1 (2018\-07\-01) +.SS Features & Improvements +.INDENT 0.0 +.IP \(bu 2 +All calls to \fBpipenv shell\fP are now implemented from the ground up using \fI\%shellingham\fP, a custom library which was purpose built to handle edge cases and shell detection. \fI\%#2371\fP +.IP \(bu 2 +Added support for python 3.7 via a few small compatibility / bugfixes. \fI\%#2427\fP, +\fI\%#2434\fP, +\fI\%#2436\fP +.IP \(bu 2 +Added new flag \fBpipenv \-\-support\fP to replace the diagnostic command \fBpython \-m pipenv.help\fP\&. \fI\%#2477\fP, +\fI\%#2478\fP +.IP \(bu 2 +Improved import times and CLI runtimes with minor tweaks. \fI\%#2485\fP +.UNINDENT +.SS Bug Fixes +.INDENT 0.0 +.IP \(bu 2 +Fixed an ongoing bug which sometimes resolved incompatible versions into lockfiles. \fI\%#1901\fP +.IP \(bu 2 +Fixed a bug which caused errors when creating virtualenvs which contained leading dash characters. \fI\%#2415\fP +.IP \(bu 2 +Fixed a logic error which caused \fB\-\-deploy \-\-system\fP to overwrite editable vcs packages in the pipfile before installing, which caused any installation to fail by default. \fI\%#2417\fP +.IP \(bu 2 +Updated requirementslib to fix an issue with properly quoting markers in VCS requirements. \fI\%#2419\fP +.IP \(bu 2 +Installed new vendored jinja2 templates for \fBclick\-completion\fP which were causing template errors for users with completion enabled. \fI\%#2422\fP +.IP \(bu 2 +Added support for python 3.7 via a few small compatibility / bugfixes. \fI\%#2427\fP +.IP \(bu 2 +Fixed an issue reading package names from \fBsetup.py\fP files in projects which imported utilities such as \fBversioneer\fP\&. \fI\%#2433\fP +.IP \(bu 2 +Pipenv will now ensure that its internal package names registry files are written with unicode strings. \fI\%#2450\fP +.IP \(bu 2 +Fixed a bug causing requirements input as relative paths to be output as absolute paths or URIs. +Fixed a bug affecting normalization of \fBgit+git@host\fP uris. \fI\%#2453\fP +.IP \(bu 2 +Pipenv will now always use \fBpathlib2\fP for \fBPath\fP based filesystem interactions by default on \fBpython<3.5\fP\&. \fI\%#2454\fP +.IP \(bu 2 +Fixed a bug which prevented passing proxy PyPI indexes set with \fB\-\-pypi\-mirror\fP from being passed to pip during virtualenv creation, which could cause the creation to freeze in some cases. \fI\%#2462\fP +.IP \(bu 2 +Using the \fBpython \-m pipenv.help\fP command will now use proper encoding for the host filesystem to avoid encoding issues. \fI\%#2466\fP +.IP \(bu 2 +The new \fBjinja2\fP templates for \fBclick_completion\fP will now be included in pipenv source distributions. \fI\%#2479\fP +.IP \(bu 2 +Resolved a long\-standing issue with re\-using previously generated \fBInstallRequirement\fP objects for resolution which could cause \fBPKG\-INFO\fP file information to be deleted, raising a \fBTypeError\fP\&. \fI\%#2480\fP +.IP \(bu 2 +Resolved an issue parsing usernames from private PyPI URIs in \fBPipfiles\fP by updating \fBrequirementslib\fP\&. \fI\%#2484\fP +.UNINDENT +.SS Vendored Libraries +.INDENT 0.0 +.IP \(bu 2 +All calls to \fBpipenv shell\fP are now implemented from the ground up using \fI\%shellingham\fP, a custom library which was purpose built to handle edge cases and shell detection. \fI\%#2371\fP +.IP \(bu 2 +Updated requirementslib to fix an issue with properly quoting markers in VCS requirements. \fI\%#2419\fP +.IP \(bu 2 +Installed new vendored jinja2 templates for \fBclick\-completion\fP which were causing template errors for users with completion enabled. \fI\%#2422\fP +.IP \(bu 2 +Add patch to \fBprettytoml\fP to support Python 3.7. \fI\%#2426\fP +.IP \(bu 2 +Patched \fBprettytoml.AbstractTable._enumerate_items\fP to handle \fBStopIteration\fP errors in preparation of release of python 3.7. \fI\%#2427\fP +.IP \(bu 2 +Fixed an issue reading package names from \fBsetup.py\fP files in projects which imported utilities such as \fBversioneer\fP\&. \fI\%#2433\fP +.IP \(bu 2 +Updated \fBrequirementslib\fP to version \fB1.0.9\fP \fI\%#2453\fP +.IP \(bu 2 +Unraveled a lot of old, unnecessary patches to \fBpip\-tools\fP which were causing non\-deterministic resolution errors. \fI\%#2480\fP +.IP \(bu 2 +Resolved an issue parsing usernames from private PyPI URIs in \fBPipfiles\fP by updating \fBrequirementslib\fP\&. \fI\%#2484\fP +.UNINDENT +.SS Improved Documentation +.INDENT 0.0 +.IP \(bu 2 +Added instructions for installing using Fedora\(aqs official repositories. \fI\%#2404\fP +.UNINDENT +.SS 2018.6.25 (2018\-06\-25) +.SS Features & Improvements +.INDENT 0.0 +.IP \(bu 2 +Pipenv\-created virtualenvs will now be associated with a \fB\&.project\fP folder +(features can be implemented on top of this later or users may choose to use +\fBpipenv\-pipes\fP to take full advantage of this.) \fI\%#1861\fP +.IP \(bu 2 +Virtualenv names will now appear in prompts for most Windows users. \fI\%#2167\fP +.IP \(bu 2 +Added support for cmder shell paths with spaces. \fI\%#2168\fP +.IP \(bu 2 +Added nested JSON output to the \fBpipenv graph\fP command. \fI\%#2199\fP +.IP \(bu 2 +Dropped vendored pip 9 and vendored, patched, and migrated to pip 10. Updated +patched piptools version. \fI\%#2255\fP +.IP \(bu 2 +PyPI mirror URLs can now be set to override instances of PyPI urls by passing +the \fB\-\-pypi\-mirror\fP argument from the command line or setting the +\fBPIPENV_PYPI_MIRROR\fP environment variable. \fI\%#2281\fP +.IP \(bu 2 +Virtualenv activation lines will now avoid being written to some shell +history files. \fI\%#2287\fP +.IP \(bu 2 +Pipenv will now only search for \fBrequirements.txt\fP files when creating new +projects, and during that time only if the user doesn\(aqt specify packages to +pass in. \fI\%#2309\fP +.IP \(bu 2 +Added support for mounted drives via UNC paths. \fI\%#2331\fP +.IP \(bu 2 +Added support for Windows Subsystem for Linux bash shell detection. \fI\%#2363\fP +.IP \(bu 2 +Pipenv will now generate hashes much more quickly by resolving them in a +single pass during locking. \fI\%#2384\fP +.IP \(bu 2 +\fBpipenv run\fP will now avoid spawning additional \fBCOMSPEC\fP instances to +run commands in when possible. \fI\%#2385\fP +.IP \(bu 2 +Massive internal improvements to requirements parsing codebase, resolver, and +error messaging. \fI\%#2388\fP +.IP \(bu 2 +\fBpipenv check\fP now may take multiple of the additional argument +\fB\-\-ignore\fP which takes a parameter \fBcve_id\fP for the purpose of ignoring +specific CVEs. \fI\%#2408\fP +.UNINDENT +.SS Behavior Changes +.INDENT 0.0 +.IP \(bu 2 +Pipenv will now parse & capitalize \fBplatform_python_implementation\fP markers +.. warning:: This could cause an issue if you have an out of date \fBPipfile\fP +which lowercases the comparison value (e.g. \fBcpython\fP instead of +\fBCPython\fP). \fI\%#2123\fP +.IP \(bu 2 +Pipenv will now only search for \fBrequirements.txt\fP files when creating new +projects, and during that time only if the user doesn\(aqt specify packages to +pass in. \fI\%#2309\fP +.UNINDENT +.SS Bug Fixes +.INDENT 0.0 +.IP \(bu 2 +Massive internal improvements to requirements parsing codebase, resolver, and +error messaging. \fI\%#1962\fP, +\fI\%#2186\fP, +\fI\%#2263\fP, +\fI\%#2312\fP +.IP \(bu 2 +Pipenv will now parse & capitalize \fBplatform_python_implementation\fP +markers. \fI\%#2123\fP +.IP \(bu 2 +Fixed a bug with parsing and grouping old\-style \fBsetup.py\fP extras during +resolution \fI\%#2142\fP +.IP \(bu 2 +Fixed a bug causing pipenv graph to throw unhelpful exceptions when running +against empty or non\-existent environments. \fI\%#2161\fP +.IP \(bu 2 +Fixed a bug which caused \fB\-\-system\fP to incorrectly abort when users were in +a virtualenv. \fI\%#2181\fP +.IP \(bu 2 +Removed vendored \fBcacert.pem\fP which could cause issues for some users with +custom certificate settings. \fI\%#2193\fP +.IP \(bu 2 +Fixed a regression which led to direct invocations of \fBvirtualenv\fP, rather +than calling it by module. \fI\%#2198\fP +.IP \(bu 2 +Locking will now pin the correct VCS ref during \fBpipenv update\fP runs. +Running \fBpipenv update\fP with a new vcs ref specified in the \fBPipfile\fP +will now properly obtain, resolve, and install the specified dependency at +the specified ref. \fI\%#2209\fP +.IP \(bu 2 +\fBpipenv clean\fP will now correctly ignore comments from \fBpip freeze\fP when +cleaning the environment. \fI\%#2262\fP +.IP \(bu 2 +Resolution bugs causing packages for incompatible python versions to be +locked have been fixed. \fI\%#2267\fP +.IP \(bu 2 +Fixed a bug causing pipenv graph to fail to display sometimes. \fI\%#2268\fP +.IP \(bu 2 +Updated \fBrequirementslib\fP to fix a bug in pipfile parsing affecting +relative path conversions. \fI\%#2269\fP +.IP \(bu 2 +Windows executable discovery now leverages \fBos.pathext\fP\&. \fI\%#2298\fP +.IP \(bu 2 +Fixed a bug which caused \fB\-\-deploy \-\-system\fP to inadvertently create a +virtualenv before failing. \fI\%#2301\fP +.IP \(bu 2 +Fixed an issue which led to a failure to unquote special characters in file +and wheel paths. \fI\%#2302\fP +.IP \(bu 2 +VCS dependencies are now manually obtained only if they do not match the +requested ref. \fI\%#2304\fP +.IP \(bu 2 +Added error handling functionality to properly cope with single\-digit +\fBRequires\-Python\fP metatdata with no specifiers. \fI\%#2377\fP +.IP \(bu 2 +\fBpipenv update\fP will now always run the resolver and lock before ensuring +your dependencies are in sync with your lockfile. \fI\%#2379\fP +.IP \(bu 2 +Resolved a bug in our patched resolvers which could cause nondeterministic +resolution failures in certain conditions. Running \fBpipenv install\fP with no +arguments in a project with only a \fBPipfile\fP will now correctly lock first +for dependency resolution before installing. \fI\%#2384\fP +.IP \(bu 2 +Patched \fBpython\-dotenv\fP to ensure that environment variables always get +encoded to the filesystem encoding. \fI\%#2386\fP +.UNINDENT +.SS Improved Documentation +.INDENT 0.0 +.IP \(bu 2 +Update documentation wording to clarify Pipenv\(aqs overall role in the packaging ecosystem. \fI\%#2194\fP +.IP \(bu 2 +Added contribution documentation and guidelines. \fI\%#2205\fP +.IP \(bu 2 +Added instructions for supervisord compatibility. \fI\%#2215\fP +.IP \(bu 2 +Fixed broken links to development philosophy and contribution documentation. \fI\%#2248\fP +.UNINDENT +.SS Vendored Libraries +.INDENT 0.0 +.IP \(bu 2 +Removed vendored \fBcacert.pem\fP which could cause issues for some users with +custom certificate settings. \fI\%#2193\fP +.IP \(bu 2 +Dropped vendored pip 9 and vendored, patched, and migrated to pip 10. Updated +patched piptools version. \fI\%#2255\fP +.IP \(bu 2 +Updated \fBrequirementslib\fP to fix a bug in pipfile parsing affecting +relative path conversions. \fI\%#2269\fP +.IP \(bu 2 +Added custom shell detection library \fBshellingham\fP, a port of our changes +to \fBpew\fP\&. \fI\%#2363\fP +.IP \(bu 2 +Patched \fBpython\-dotenv\fP to ensure that environment variables always get +encoded to the filesystem encoding. \fI\%#2386\fP +.IP \(bu 2 +Updated vendored libraries. The following vendored libraries were updated: +.INDENT 2.0 +.IP \(bu 2 +distlib from version \fB0.2.6\fP to \fB0.2.7\fP\&. +.IP \(bu 2 +jinja2 from version \fB2.9.5\fP to \fB2.10\fP\&. +.IP \(bu 2 +pathlib2 from version \fB2.1.0\fP to \fB2.3.2\fP\&. +.IP \(bu 2 +parse from version \fB2.8.0\fP to \fB2.8.4\fP\&. +.IP \(bu 2 +pexpect from version \fB2.5.2\fP to \fB2.6.0\fP\&. +.IP \(bu 2 +requests from version \fB2.18.4\fP to \fB2.19.1\fP\&. +.IP \(bu 2 +idna from version \fB2.6\fP to \fB2.7\fP\&. +.IP \(bu 2 +certifi from version \fB2018.1.16\fP to \fB2018.4.16\fP\&. +.IP \(bu 2 +packaging from version \fB16.8\fP to \fB17.1\fP\&. +.IP \(bu 2 +six from version \fB1.10.0\fP to \fB1.11.0\fP\&. +.IP \(bu 2 +requirementslib from version \fB0.2.0\fP to \fB1.0.1\fP\&. +.UNINDENT +.sp +In addition, scandir was vendored and patched to avoid importing host system binaries when falling back to pathlib2. \fI\%#2368\fP +.UNINDENT .SH USER TESTIMONIALS .INDENT 0.0 .TP \fBJannis Leidel\fP, former pip maintainer— -\fIPipenv is the porcelain I always wanted to build for pip9. It fits my brain and mostly replaces virtualenvwrapper and manual pip calls for me. Use it.\fP +\fIPipenv is the porcelain I always wanted to build for pip. It fits my brain and mostly replaces virtualenvwrapper and manual pip calls for me. Use it.\fP +.TP +\fBDavid Gang\fP— +\fIThis package manager is really awesome. For the first time I know exactly what my dependencies are which I installed and what the transitive dependencies are. Combined with the fact that installs are deterministic, makes this package manager first class, like cargo\fP\&. .TP \fBJustin Myles Holmes\fP— \fIPipenv is finally an abstraction meant to engage the mind instead of merely the filesystem.\fP -.TP -\fBIsaac Sanders\fP— -\fIPipenv is literally the best thing about my day today. Thanks, Kenneth!\fP .UNINDENT .SH ☤ PIPENV FEATURES .INDENT 0.0 @@ -448,11 +1154,11 @@ Otherwise, whatever virtualenv defaults to will be the default. .SS Other Commands .INDENT 0.0 .IP \(bu 2 -\fBgraph\fP will show you a dependency graph, of your installed dependencies. +\fBgraph\fP will show you a dependency graph of your installed dependencies. .IP \(bu 2 -\fBshell\fP will spawn a shell with the virtualenv activated. +\fBshell\fP will spawn a shell with the virtualenv activated. This shell can be deactivated by using \fBexit\fP\&. .IP \(bu 2 -\fBrun\fP will run a given command from the virtualenv, with any arguments forwarded (e.g. \fB$ pipenv run python\fP). +\fBrun\fP will run a given command from the virtualenv, with any arguments forwarded (e.g. \fB$ pipenv run python\fP or \fB$ pipenv run pip freeze\fP). .IP \(bu 2 \fBcheck\fP checks for security vulnerabilities and asserts that PEP 508 requirements are being met by the current environment. .UNINDENT @@ -577,6 +1283,87 @@ pytest = "*" .fi .UNINDENT .UNINDENT +.SS ☤ General Recommendations & Version Control +.INDENT 0.0 +.IP \(bu 2 +Generally, keep both \fBPipfile\fP and \fBPipfile.lock\fP in version control. +.IP \(bu 2 +Do not keep \fBPipfile.lock\fP in version control if multiple versions of Python are being targeted. +.IP \(bu 2 +Specify your target Python version in your \fIPipfile\fP\(aqs \fB[requires]\fP section. Ideally, you should only have one target Python version, as this is a deployment tool. +.IP \(bu 2 +\fBpipenv install\fP is fully compatible with \fBpip install\fP syntax, for which the full documentation can be found \fI\%here\fP\&. +.UNINDENT +.SS ☤ Example Pipenv Workflow +.sp +Clone / create project repository: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ cd myproject +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Install from Pipfile, if there is one: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ pipenv install +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Or, add a package to your new project: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ pipenv install +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +This will create a \fBPipfile\fP if one doesn\(aqt exist. If one does exist, it will automatically be edited with the new package you provided. +.sp +Next, activate the Pipenv shell: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ pipenv shell +$ python \-\-version +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +This will spawn a new shell subprocess, which can be deactivated by using \fBexit\fP\&. +.SS ☤ Example Pipenv Upgrade Workflow +.INDENT 0.0 +.IP \(bu 2 +Find out what\(aqs changed upstream: \fB$ pipenv update \-\-outdated\fP\&. +.IP \(bu 2 +.INDENT 2.0 +.TP +.B Upgrade packages, two options: +.INDENT 7.0 +.IP a. 3 +Want to upgrade everything? Just do \fB$ pipenv update\fP\&. +.IP b. 3 +Want to upgrade packages one\-at\-a\-time? \fB$ pipenv update \fP for each outdated package. +.UNINDENT +.UNINDENT +.UNINDENT .SS ☤ Importing from requirements.txt .sp If you only have a \fBrequirements.txt\fP file available when running \fBpipenv install\fP, @@ -584,23 +1371,71 @@ pipenv will automatically import the contents of this file and create a \fBPipfi .sp You can also specify \fB$ pipenv install \-r path/to/requirements.txt\fP to import a requirements file. .sp -Note, that when importing a requirements file, they often have version numbers pinned, which you likely won\(aqt want -in your \fBPipfile\fP, so you\(aqll have to manually update your \fBPipfile\fP afterwards to reflect this. +If your requirements file has version numbers pinned, you\(aqll likely want to edit the new \fBPipfile\fP +to remove those, and let \fBpipenv\fP keep track of pinning. If you want to keep the pinned versions +in your \fBPipfile.lock\fP for now, run \fBpipenv lock \-\-keep\-outdated\fP\&. Make sure to +\fI\%upgrade\fP soon! .SS ☤ Specifying Versions of a Package .sp -To tell pipenv to install a specific version of a library, the usage is simple: +You can specify versions of a package using the \fI\%Semantic Versioning scheme\fP +(i.e. \fBmajor.minor.micro\fP). +.sp +For example, to install requests you can use: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -$ pipenv install requests==2.13.0 +$ pipenv install requests~=1.2 # equivalent to requests~=1.2.0 .ft P .fi .UNINDENT .UNINDENT .sp +Pipenv will install version \fB1.2\fP and any minor update, but not \fB2.0\fP\&. +.sp This will update your \fBPipfile\fP to reflect this requirement, automatically. +.sp +In general, Pipenv uses the same specifier format as pip. However, note that according to \fI\%PEP 440\fP , you can\(aqt use versions containing a hyphen or a plus sign. +.sp +To make inclusive or exclusive version comparisons you can use: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ pipenv install "requests>=1.4" # will install a version equal or larger than 1.4.0 +$ pipenv install "requests<=2.13" # will install a version equal or lower than 2.13.0 +$ pipenv install "requests>2.19" # will install 2.19.1 but not 2.19.0 +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +The use of \fB" "\fP around the package and version specification is highly recommended +to avoid issues with \fI\%Input and output redirection\fP +in Unix\-based operating systems. +.UNINDENT +.UNINDENT +.sp +The use of \fB~=\fP is preferred over the \fB==\fP identifier as the former prevents pipenv from updating the packages: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ pipenv install "requests~=2.2" # locks the major version of the package (this is equivalent to using ==2.*) +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +To avoid installing a specific version you can use the \fB!=\fP identifier. +.sp +For an in depth explanation of the valid identifiers and more complex use cases check \fI\%the relevant section of PEP\-440\fP\&. .SS ☤ Specifying Versions of Python .sp To create a new virtualenv, using a specific version of Python you have installed (and @@ -665,11 +1500,16 @@ python_version = "3.6" .UNINDENT .UNINDENT .sp -Note the inclusion of \fB[requires] python_version = "3.6"\fP\&. This specifies that your application requires this version +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +The inclusion of \fB[requires] python_version = "3.6"\fP specifies that your application requires this version of Python, and will be used automatically when running \fBpipenv install\fP against this \fBPipfile\fP in the future (e.g. on other machines). If this is not true, feel free to simply remove this section. +.UNINDENT +.UNINDENT .sp -If you don\(aqt specify a Python version on the command-line, either the \fB[requires]\fP \fBpython_full_version\fP or \fBpython_version\fP will be selected +If you don\(aqt specify a Python version on the command–line, either the \fB[requires]\fP \fBpython_full_version\fP or \fBpython_version\fP will be selected automatically, falling back to whatever your system\(aqs default \fBpython\fP installation is, at time of execution. .SS ☤ Editable Dependencies (e.g. \fB\-e .\fP ) .sp @@ -680,17 +1520,25 @@ the current working directory when working on packages: .sp .nf .ft C -$ pipenv install \(aq\-e .\(aq \-\-dev +$ pipenv install \-\-dev \-e . $ cat Pipfile +\&... [dev\-packages] "e1839a8" = {path = ".", editable = true} +\&... .ft P .fi .UNINDENT .UNINDENT .sp -Note that all sub\-dependencies will get added to the \fBPipfile.lock\fP as well. +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +All sub\-dependencies will get added to the \fBPipfile.lock\fP as well. Sub\-dependencies are \fBnot\fP added to the +\fBPipfile.lock\fP if you leave the \fB\-e\fP option out. +.UNINDENT +.UNINDENT .SS ☤ Environment Management with Pipenv .sp The three primary commands you\(aqll use in managing your pipenv environment are @@ -745,7 +1593,7 @@ is unique. .UNINDENT .INDENT 0.0 .IP \(bu 2 -\fB\-\-dev\fP — Install both \fBdevelop\fP and \fBdefault\fP packages from \fBPipfile.lock\fP\&. +\fB\-\-dev\fP — Install both \fBdevelop\fP and \fBdefault\fP packages from \fBPipfile\fP\&. .IP \(bu 2 \fB\-\-system\fP — Use the system \fBpip\fP command rather than the one from your virtualenv. .IP \(bu 2 @@ -758,13 +1606,16 @@ is unique. .SS $ pipenv uninstall .sp \fB$ pipenv uninstall\fP supports all of the parameters in \fI\%pipenv install\fP, -as well as one additional, \fB\-\-all\fP\&. +as well as two additional options, \fB\-\-all\fP and \fB\-\-all\-dev\fP\&. .INDENT 0.0 .INDENT 3.5 .INDENT 0.0 .IP \(bu 2 \fB\-\-all\fP — This parameter will purge all files from the virtual environment, but leave the Pipfile untouched. +.IP \(bu 2 +\fB\-\-all\-dev\fP — This parameter will remove all of the development packages from +the virtual environment, and remove them from the Pipfile. .UNINDENT .UNINDENT .UNINDENT @@ -797,22 +1648,50 @@ You should do this for your shell too, in your \fB~/.profile\fP or \fB~/.bashrc\ The shell launched in interactive mode. This means that if your shell reads its configuration from a specific file for interactive mode (e.g. bash by default looks for a \fB~/.bashrc\fP configuration file for interactive mode), then you\(aqll need to modify (or create) this file. .UNINDENT .UNINDENT +.sp +If you experience issues with \fB$ pipenv shell\fP, just check the \fBPIPENV_SHELL\fP environment variable, which \fB$ pipenv shell\fP will use if available. For detail, see configuration\-with\-environment\-variables\&. .SS ☤ A Note about VCS Dependencies .sp -Pipenv will resolve the sub-dependencies of VCS dependencies, but only if they are editable, like so: +You can install packages with pipenv from git and other version control systems using URLs formatted according to the following rule: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -[packages] -requests = {git = "https://github.com/requests/requests.git", editable=true} ++:////@#egg= .ft P .fi .UNINDENT .UNINDENT .sp -If editable is not true, sub-dependencies will not get resolved. +The only optional section is the \fB@\fP section. When using git over SSH, you may use the shorthand vcs and scheme alias \fBgit+git@:/@#\fP\&. Note that this is translated to \fBgit+ssh://git@\fP when parsed. +.sp +Note that it is \fBstrongly recommended\fP that you install any version\-controlled dependencies in editable mode, using \fBpipenv install \-e\fP, in order to ensure that dependency resolution can be performed with an up to date copy of the repository each time it is performed, and that it includes all known dependencies. +.sp +Below is an example usage which installs the git repository located at \fBhttps://github.com/requests/requests.git\fP from tag \fBv2.20.1\fP as package name \fBrequests\fP: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ pipenv install \-e git+https://github.com/requests/requests.git@v2.20.1#egg=requests +Creating a Pipfile for this project... +Installing \-e git+https://github.com/requests/requests.git@v2.20.1#egg=requests... +[...snipped...] +Adding \-e git+https://github.com/requests/requests.git@v2.20.1#egg=requests to Pipfile\(aqs [packages]... +[...] + +$ cat Pipfile +[packages] +requests = {git = "https://github.com/requests/requests.git", editable = true, ref = "v2.20.1"} +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Valid values for \fB\fP include \fBgit\fP, \fBbzr\fP, \fBsvn\fP, and \fBhg\fP\&. Valid values for \fB\fP include \fBhttp\fP, \fBhttps\fP, \fBssh\fP, and \fBfile\fP\&. In specific cases you also have access to other schemes: \fBsvn\fP may be combined with \fBsvn\fP as a scheme, and \fBbzr\fP can be combined with \fBsftp\fP and \fBlp\fP\&. +.sp +You can read more about pip\(aqs implementation of VCS support \fI\%here\fP\&. For more information about other options available when specifying VCS dependencies, please check the \fI\%Pipfile spec\fP\&. .SS ☤ Pipfile.lock Security Features .sp \fBPipfile.lock\fP takes advantage of some great new security improvements in \fBpip\fP\&. @@ -824,10 +1703,28 @@ We highly recommend approaching deployments with promoting projects from a devel environment into production. You can use \fBpipenv lock\fP to compile your dependencies on your development environment and deploy the compiled \fBPipfile.lock\fP to all of your production environments for reproducible builds. +.sp +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +If you\(aqd like a \fBrequirements.txt\fP output of the lockfile, run \fB$ pipenv lock \-r\fP\&. +This will include all hashes, however (which is great!). To get a \fBrequirements.txt\fP +without hashes, use \fB$ pipenv run pip freeze\fP\&. +.UNINDENT +.UNINDENT .SS Advanced Usage of Pipenv [image] .sp This document covers some of Pipenv\(aqs more glorious and advanced features. +.SS ☤ Caveats +.INDENT 0.0 +.IP \(bu 2 +Dependencies of wheels provided in a \fBPipfile\fP will not be captured by \fB$ pipenv lock\fP\&. +.IP \(bu 2 +There are some known issues with using private indexes, related to hashing. We\(aqre actively working to solve this problem. You may have great luck with this, however. +.IP \(bu 2 +Installation is intended to be as deterministic as possible —\ use the \fB\-\-sequential\fP flag to increase this, if experiencing issues. +.UNINDENT .SS ☤ Specifying Package Indexes .sp If you\(aqd like a specific package to be installed with a specific package index, you can do the following: @@ -858,6 +1755,50 @@ records = "*" .UNINDENT .sp Very fancy. +.SS ☤ Using a PyPI Mirror +.sp +If you\(aqd like to override the default PyPI index urls with the url for a PyPI mirror, you can use the following: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ pipenv install \-\-pypi\-mirror + +$ pipenv update \-\-pypi\-mirror + +$ pipenv sync \-\-pypi\-mirror + +$ pipenv lock \-\-pypi\-mirror + +$ pipenv uninstall \-\-pypi\-mirror +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Alternatively, you can set the \fBPIPENV_PYPI_MIRROR\fP environment variable. +.SS ☤ Injecting credentials into Pipfiles via environment variables +.sp +Pipenv will expand environment variables (if defined) in your Pipfile. Quite +useful if you need to authenticate to a private PyPI: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +[[source]] +url = "https://$USERNAME:${PASSWORD}@mypypi.example.com/simple" +verify_ssl = true +name = "pypi" +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Luckily \- pipenv will hash your Pipfile \fIbefore\fP expanding environment +variables (and, helpfully, will substitute the environment variables again when +you install from the lock file \- so no need to commit any secrets! Woo!) .SS ☤ Specifying Basically Anything .sp If you\(aqd like to specify that a specific package only be installed on certain systems, @@ -876,7 +1817,7 @@ name = "pypi" [packages] requests = "*" -pywinusb = {version = "*", os_name = "== \(aqwindows\(aq"} +pywinusb = {version = "*", sys_platform = "== \(aqwin32\(aq"} .ft P .fi .UNINDENT @@ -902,9 +1843,45 @@ unittest2 = {version = ">=1.0,<3.0", markers="python_version < \(aq2.7.9\(aq or .UNINDENT .sp Magic. Pure, unadulterated magic. -.SS ☤ Deploying System Dependencies +.SS ☤ Using pipenv for Deployments .sp -You can tell Pipenv to install things into its parent system with the \fB\-\-system\fP flag: +You may want to use \fBpipenv\fP as part of a deployment process. +.sp +You can enforce that your \fBPipfile.lock\fP is up to date using the \fB\-\-deploy\fP flag: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ pipenv install \-\-deploy +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +This will fail a build if the \fBPipfile.lock\fP is out–of–date, instead of generating a new one. +.sp +Or you can install packages exactly as specified in \fBPipfile.lock\fP using the \fBsync\fP command: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ pipenv sync +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +\fBpipenv install \-\-ignore\-pipfile\fP is nearly equivalent to \fBpipenv sync\fP, but \fBpipenv sync\fP will \fInever\fP attempt to re\-lock your dependencies as it is considered an atomic operation. \fBpipenv install\fP by default does attempt to re\-lock unless using the \fB\-\-deploy\fP flag. +.UNINDENT +.UNINDENT +.SS Deploying System Dependencies +.sp +You can tell Pipenv to install a Pipfile\(aqs contents into its parent system with the \fB\-\-system\fP flag: .INDENT 0.0 .INDENT 3.5 .sp @@ -916,24 +1893,35 @@ $ pipenv install \-\-system .UNINDENT .UNINDENT .sp -This is useful for Docker containers, and deployment infrastructure (e.g. Heroku does this). +This is useful for managing the system Python, and deployment infrastructure (e.g. Heroku does this). +.SS ☤ Pipenv and Other Python Distributions .sp -Also useful for deployment is the \fB\-\-deploy\fP flag: +To use Pipenv with a third\-party Python distribution (e.g. Anaconda), you simply provide the path to the Python binary: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -$ pipenv install \-\-system \-\-deploy +$ pipenv install \-\-python=/path/to/python .ft P .fi .UNINDENT .UNINDENT .sp -This will fail a build if the \fBPipfile.lock\fP is out-of-date, instead of generating a new one. +Anaconda uses Conda to manage packages. To reuse Conda–installed Python packages, use the \fB\-\-site\-packages\fP flag: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ pipenv \-\-python=/path/to/python \-\-site\-packages +.ft P +.fi +.UNINDENT +.UNINDENT .SS ☤ Generating a \fBrequirements.txt\fP .sp -You can convert a \fBPipfile\fP and \fBPipenv.lock\fP into a \fBrequirements.txt\fP file very easily, and get all the benefits of hashes, extras, and other goodies we have included. +You can convert a \fBPipfile\fP and \fBPipfile.lock\fP into a \fBrequirements.txt\fP file very easily, and get all the benefits of extras and other goodies we have included. .sp Let\(aqs take this \fBPipfile\fP: .INDENT 0.0 @@ -959,11 +1947,42 @@ And generate a \fBrequirements.txt\fP out of it: .nf .ft C $ pipenv lock \-r -chardet==3.0.4 \-\-hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 \-\-hash=sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae -requests==2.18.4 \-\-hash=sha256:6a1b267aa90cac58ac3a765d067950e7dbbf75b1da07e895d1f594193a40a38b \-\-hash=sha256:9c443e7324ba5b85070c4a818ade28bfabedf16ea10206da1132edaa6dda237e -certifi==2017.7.27.1 \-\-hash=sha256:54a07c09c586b0e4c619f02a5e94e36619da8e2b053e20f594348c0611803704 \-\-hash=sha256:40523d2efb60523e113b44602298f0960e900388cf3bb6043f645cf57ea9e3f5 -idna==2.6 \-\-hash=sha256:8c7309c718f94b3a625cb648ace320157ad16ff131ae0af362c9f21b80ef6ec4 \-\-hash=sha256:2c6a5de3089009e3da7c5dde64a141dbc8551d5b7f6cf4ed7c2568d0cc520a8f -urllib3==1.22 \-\-hash=sha256:06330f386d6e4b195fbfc736b297f58c5a892e4440e54d294d7004e3a9bbea1b \-\-hash=sha256:cc44da8e1145637334317feebd728bd869a35285b93cbb4cca2577da7e62db4f +chardet==3.0.4 +requests==2.18.4 +certifi==2017.7.27.1 +idna==2.6 +urllib3==1.22 +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +If you wish to generate a \fBrequirements.txt\fP with only the development requirements you can do that too! Let\(aqs take the following \fBPipfile\fP: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +[[source]] +url = "https://pypi.python.org/simple" +verify_ssl = true + +[dev\-packages] +pytest = {version="*"} +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +And generate a \fBrequirements.txt\fP out of it: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ pipenv lock \-r \-\-dev +py==1.4.34 +pytest==3.2.3 .ft P .fi .UNINDENT @@ -1025,26 +2044,50 @@ hardened for production use and should be used only as a development aid. .UNINDENT .sp ✨🍰✨ -.SS ☤ Code Style Checking .sp -Pipenv has \fI\%Flake 8\fP built into it. You can check the style of your code like so, without installing anything: +\fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 +In order to enable this functionality while maintaining its permissive +copyright license, \fIpipenv\fP embeds an API client key for the backend +Safety API operated by pyup.io rather than including a full copy of the +CC\-BY\-NC\-SA licensed Safety\-DB database. This embedded client key is +shared across all \fIpipenv check\fP users, and hence will be subject to +API access throttling based on overall usage rather than individual +client usage. .sp -.nf -.ft C -$ cat t.py -import requests - -$ pipenv check \-\-style t.py -t.py:1:1: F401 \(aqrequests\(aq imported but unused -t.py:1:16: W292 no newline at end of file -.ft P -.fi +You can also use your own safety API key by setting the +environment variable \fBPIPENV_PYUP_API_KEY\fP\&. .UNINDENT .UNINDENT +.SS ☤ Community Integrations .sp -Super useful :) +There are a range of community\-maintained plugins and extensions available for a range of editors and IDEs, as well as +different products which integrate with Pipenv projects: +.INDENT 0.0 +.IP \(bu 2 +\fI\%Heroku\fP (Cloud Hosting) +.IP \(bu 2 +\fI\%Platform.sh\fP (Cloud Hosting) +.IP \(bu 2 +\fI\%PyUp\fP (Security Notification) +.IP \(bu 2 +\fI\%Emacs\fP (Editor Integration) +.IP \(bu 2 +\fI\%Fish Shell\fP (Automatic \fB$ pipenv shell\fP!) +.IP \(bu 2 +\fI\%VS Code\fP (Editor Integration) +.IP \(bu 2 +\fI\%PyCharm\fP (Editor Integration) +.UNINDENT +.sp +Works in progress: +.INDENT 0.0 +.IP \(bu 2 +\fI\%Sublime Text\fP (Editor Integration) +.IP \(bu 2 +Mysterious upcoming Google Cloud product (Cloud Hosting) +.UNINDENT .SS ☤ Open a Module in Your Editor .sp Pipenv allows you to open any Python module that is installed (including ones in your codebase), with the \fB$ pipenv open\fP command: @@ -1070,7 +2113,7 @@ This allows you to easily read the code you\(aqre consuming, instead of looking \fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -The standard \fBEDITOR\fP environment variable is used for this. If you\(aqre using Sublime Text, for example, you\(aqll want to \fBexport EDITOR=subl\fP (once you\(aqve installed the command\-line utility). +The standard \fBEDITOR\fP environment variable is used for this. If you\(aqre using VS Code, for example, you\(aqll want to \fBexport EDITOR=code\fP (if you\(aqre on macOS you will want to \fI\%install the command\fP on to your \fBPATH\fP first). .UNINDENT .UNINDENT .SS ☤ Automatic Python Installation @@ -1157,37 +2200,297 @@ $ PIPENV_DOTENV_LOCATION=/path/to/.env pipenv shell .fi .UNINDENT .UNINDENT +.sp +To prevent pipenv from loading the \fB\&.env\fP file, set the \fBPIPENV_DONT_LOAD_ENV\fP environment variable: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ PIPENV_DONT_LOAD_ENV=1 pipenv shell +.ft P +.fi +.UNINDENT +.UNINDENT +.SS ☤ Custom Script Shortcuts +.sp +Pipenv supports creating custom shortcuts in the (optional) \fB[scripts]\fP section of your Pipfile. +.sp +You can then run \fBpipenv run \fP in your terminal to run the command in the +context of your pipenv virtual environment even if you have not activated the pipenv shell first. +.sp +For example, in your Pipfile: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +[scripts] +printspam = "python \-c \e"print(\(aqI am a silly example, no one would need to do this\(aq)\e"" +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +And then in your terminal: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ pipenv run printspam +I am a silly example, no one would need to do this +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Commands that expect arguments will also work. +For example: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +[scripts] +echospam = "echo I am really a very silly example" + +$ pipenv run echospam "indeed" +I am really a very silly example indeed +.ft P +.fi +.UNINDENT +.UNINDENT +.SS ☤ Support for Environment Variables +.sp +Pipenv supports the usage of environment variables in values. For example: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +[[source]] +url = "https://${PYPI_USERNAME}:${PYPI_PASSWORD}@my_private_repo.example.com/simple" +verify_ssl = true +name = "pypi" + +[dev\-packages] + +[packages] +requests = {version="*", index="home"} +maya = {version="*", index="pypi"} +records = "*" +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Environment variables may be specified as \fB${MY_ENVAR}\fP or \fB$MY_ENVAR\fP\&. +On Windows, \fB%MY_ENVAR%\fP is supported in addition to \fB${MY_ENVAR}\fP or \fB$MY_ENVAR\fP\&. .SS ☤ Configuration With Environment Variables .sp -\fBpipenv\fP comes with a handful of options that can be enabled via shell environment +Pipenv comes with a handful of options that can be enabled via shell environment variables. To activate them, simply create the variable in your shell and pipenv will detect it. .INDENT 0.0 -.INDENT 3.5 +.TP +.B pipenv.environments.PIPENV_CACHE_DIR = \(aq/Users/fming/Library/Caches/pipenv\(aq +Location for Pipenv to store it\(aqs package cache. +.sp +Default is to use appdir\(aqs user cache directory. +.UNINDENT .INDENT 0.0 -.IP \(bu 2 -\fBPIPENV_DEFAULT_PYTHON_VERSION\fP — Use this version of Python when creating new virtual environments, by default (e.g. \fB3.6\fP). -.IP \(bu 2 -\fBPIPENV_SHELL_FANCY\fP — Always use fancy mode when invoking \fBpipenv shell\fP\&. -.IP \(bu 2 -\fBPIPENV_VENV_IN_PROJECT\fP — If set, use \fB\&.venv\fP in your project directory -instead of the global location. -.IP \(bu 2 -\fBPIPENV_COLORBLIND\fP — Disable terminal colors, for some reason. -.IP \(bu 2 -\fBPIPENV_NOSPIN\fP — Disable terminal spinner, for cleaner logs. Automatically set in CI environments. -.IP \(bu 2 -\fBPIPENV_MAX_DEPTH\fP — Set to an integer for the maximum number of directories to recursively -search for a Pipfile. -.IP \(bu 2 -\fBPIPENV_TIMEOUT\fP — Set to an integer for the max number of seconds Pipenv will -wait for virtualenv creation to complete. Defaults to 120 seconds. -.IP \(bu 2 -\fBPIPENV_IGNORE_VIRTUALENVS\fP — Set to disable automatically using an activated virtualenv over -the current project\(aqs own virtual environment. +.TP +.B pipenv.environments.PIPENV_COLORBLIND = False +If set, disable terminal colors. +.sp +Some people don\(aqt like colors in their terminals, for some reason. Default is +to show colors. .UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_DEFAULT_PYTHON_VERSION = None +Use this Python version when creating new virtual environments by default. +.sp +This can be set to a version string, e.g. \fB3.6\fP, or a path. Default is to use +whatever Python Pipenv is installed under (i.e. \fBsys.executable\fP). Command +line flags (e.g. \fB\-\-python\fP, \fB\-\-three\fP, and \fB\-\-two\fP) are prioritized over +this configuration. .UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_DONT_LOAD_ENV = False +If set, Pipenv does not load the \fB\&.env\fP file. +.sp +Default is to load \fB\&.env\fP for \fBrun\fP and \fBshell\fP commands. .UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_DONT_USE_PYENV = False +If set, Pipenv does not attempt to install Python with pyenv. +.sp +Default is to install Python automatically via pyenv when needed, if possible. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_DOTENV_LOCATION = None +If set, Pipenv loads the \fB\&.env\fP file at the specified location. +.sp +Default is to load \fB\&.env\fP from the project root, if found. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_EMULATOR = \(aq\(aq +If set, the terminal emulator\(aqs name for \fBpipenv shell\fP to use. +.sp +Default is to detect emulators automatically. This should be set if your +emulator, e.g. Cmder, cannot be detected correctly. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_HIDE_EMOJIS = False +Disable emojis in output. +.sp +Default is to show emojis. This is automatically set on Windows. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_IGNORE_VIRTUALENVS = False +If set, Pipenv will always assign a virtual environment for this project. +.sp +By default, Pipenv tries to detect whether it is run inside a virtual +environment, and reuses it if possible. This is usually the desired behavior, +and enables the user to use any user\-built environments with Pipenv. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_INSTALL_TIMEOUT = 900 +Max number of seconds to wait for package installation. +.sp +Defaults to 900 (15 minutes), a very long arbitrary time. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_MAX_DEPTH = 4 +Maximum number of directories to recursively search for a Pipfile. +.sp +Default is 3. See also \fBPIPENV_NO_INHERIT\fP\&. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_MAX_RETRIES = 0 +Specify how many retries Pipenv should attempt for network requests. +.sp +Default is 0. Automatically set to 1 on CI environments for robust testing. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_MAX_ROUNDS = 16 +Tells Pipenv how many rounds of resolving to do for Pip\-Tools. +.sp +Default is 16, an arbitrary number that works most of the time. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_MAX_SUBPROCESS = 16 +How many subprocesses should Pipenv use when installing. +.sp +Default is 16, an arbitrary number that seems to work. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_NOSPIN = False +If set, disable terminal spinner. +.sp +This can make the logs cleaner. Automatically set on Windows, and in CI +environments. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_NO_INHERIT = False +Tell Pipenv not to inherit parent directories. +.sp +This is useful for deployment to avoid using the wrong current directory. +Overwrites \fBPIPENV_MAX_DEPTH\fP\&. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_PIPFILE = None +If set, this specifies a custom Pipfile location. +.sp +When running pipenv from a location other than the same directory where the +Pipfile is located, instruct pipenv to find the Pipfile in the location +specified by this environment variable. +.sp +Default is to find Pipfile automatically in the current and parent directories. +See also \fBPIPENV_MAX_DEPTH\fP\&. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_PYPI_MIRROR = None +If set, tells pipenv to override PyPI index urls with a mirror. +.sp +Default is to not mirror PyPI, i.e. use the real one, pypi.org. The +\fB\-\-pypi\-mirror\fP command line flag overwrites this. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_SHELL = \(aq/bin/zsh\(aq +An absolute path to the preferred shell for \fBpipenv shell\fP\&. +.sp +Default is to detect automatically what shell is currently in use. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_SHELL_FANCY = False +If set, always use fancy mode when invoking \fBpipenv shell\fP\&. +.sp +Default is to use the compatibility shell if possible. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_SKIP_LOCK = False +If set, Pipenv won\(aqt lock dependencies automatically. +.sp +This might be desirable if a project has large number of dependencies, +because locking is an inherently slow operation. +.sp +Default is to lock dependencies and update \fBPipfile.lock\fP on each run. +.sp +NOTE: This only affects the \fBinstall\fP and \fBuninstall\fP commands. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_SPINNER = \(aqdots\(aq +Sets the default spinner type. +.sp +Spinners are identitcal to the node.js spinners and can be found at +\fI\%https://github.com/sindresorhus/cli\-spinners\fP +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_TIMEOUT = 120 +Max number of seconds Pipenv will wait for virtualenv creation to complete. +.sp +Default is 120 seconds, an arbitrary number that seems to work. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_VENV_IN_PROJECT = False +If set, creates \fB\&.venv\fP in your project directory. +.sp +Default is to create new virtual environments in a global location. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_YES = False +If set, Pipenv automatically assumes "yes" at all prompts. +.sp +Default is to prompt the user for an answer if the current command line session +if interactive. +.UNINDENT +.sp +If you\(aqd like to set these environment variables on a per\-project basis, I recommend utilizing the fantastic \fI\%direnv\fP project, in order to do so. .sp Also note that \fI\%pip itself supports environment variables\fP, if you need additional customization. .sp @@ -1204,8 +2507,9 @@ $ PIP_INSTALL_OPTION="\-\- \-DCMAKE_BUILD_TYPE=Release" pipenv install \-e . .UNINDENT .SS ☤ Custom Virtual Environment Location .sp -Pipenv will automatically honor the \fBWORKON_HOME\fP environment -variable, if you have it set —\ so you can tell pipenv to store your virtual environments wherever you want, e.g.: +Pipenv automatically honors the \fBWORKON_HOME\fP environment variable, if you +have it set —\ so you can tell pipenv to store your virtual environments +wherever you want, e.g.: .INDENT 0.0 .INDENT 3.5 .sp @@ -1241,7 +2545,7 @@ python: \- "3.4" \- "3.5" \- "3.6" - \- "3.7dev" + \- "3.7\-dev" # command to install dependencies install: "make" @@ -1291,7 +2595,6 @@ commands= [testenv:flake8\-py3] basepython = python3.4 commands= - {[testenv]deps} pipenv install \-\-dev pipenv run flake8 \-\-version pipenv run flake8 setup.py docs project test @@ -1299,6 +2602,22 @@ commands= .fi .UNINDENT .UNINDENT +.sp +Pipenv will automatically use the virtualenv provided by \fBtox\fP\&. If \fBpipenv install \-\-dev\fP installs e.g. \fBpytest\fP, then installed command \fBpy.test\fP will be present in given virtualenv and can be called directly by \fBpy.test tests\fP instead of \fBpipenv run py.test tests\fP\&. +.sp +You might also want to add \fB\-\-ignore\-pipfile\fP to \fBpipenv install\fP, as to +not accidentally modify the lock\-file on each test run. This causes Pipenv +to ignore changes to the \fBPipfile\fP and (more importantly) prevents it from +adding the current environment to \fBPipfile.lock\fP\&. This might be important as +the current environment (i.e. the virtualenv provisioned by tox) will usually +contain the current project (which may or may not be desired) and additional +dependencies from \fBtox\fP\(aqs \fBdeps\fP directive. The initial provisioning may +alternatively be disabled by adding \fBskip_install = True\fP to tox.ini. +.sp +This method requires you to be explicit about updating the lock\-file, which is +probably a good idea in any case. +.sp +A 3rd party plugin, \fI\%tox\-pipenv\fP is also available to use Pipenv natively with tox. .SS ☤ Shell Completion .sp To enable completion in fish, add this to your config: @@ -1313,6 +2632,18 @@ eval (pipenv \-\-completion) .UNINDENT .UNINDENT .sp +Alternatively, with bash or zsh, add this to your config: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +eval "$(pipenv \-\-completion)" +.ft P +.fi +.UNINDENT +.UNINDENT +.sp Magic shell completions are now enabled! .sp ✨🍰✨ @@ -1348,7 +2679,1213 @@ $ PIP_IGNORE_INSTALLED=1 pipenv install \-\-dev .fi .UNINDENT .UNINDENT +.SS ☤ Pipfile vs setup.py +.sp +There is a subtle but very important distinction to be made between \fBapplications\fP and \fBlibraries\fP\&. This is a very common source of confusion in the Python community. +.sp +Libraries provide reusable functionality to other libraries and applications (let\(aqs use the umbrella term \fBprojects\fP here). They are required to work alongside other libraries, all with their own set of subdependencies. They define \fBabstract dependencies\fP\&. To avoid version conflicts in subdependencies of different libraries within a project, libraries should never ever pin dependency versions. Although they may specify lower or (less frequently) upper bounds, if they rely on some specific feature/fix/bug. Library dependencies are specified via \fBinstall_requires\fP in \fBsetup.py\fP\&. +.sp +Libraries are ultimately meant to be used in some \fBapplication\fP\&. Applications are different in that they usually are not depended on by other projects. They are meant to be deployed into some specific environment and only then should the exact versions of all their dependencies and subdependencies be made concrete. To make this process easier is currently the main goal of Pipenv. +.sp +To summarize: +.INDENT 0.0 +.IP \(bu 2 +For libraries, define \fBabstract dependencies\fP via \fBinstall_requires\fP in \fBsetup.py\fP\&. The decision of which version exactly to be installed and where to obtain that dependency is not yours to make! +.IP \(bu 2 +For applications, define \fBdependencies and where to get them\fP in the \fIPipfile\fP and use this file to update the set of \fBconcrete dependencies\fP in \fBPipfile.lock\fP\&. This file defines a specific idempotent environment that is known to work for your project. The \fBPipfile.lock\fP is your source of truth. The \fBPipfile\fP is a convenience for you to create that lock\-file, in that it allows you to still remain somewhat vague about the exact version of a dependency to be used. Pipenv is there to help you define a working conflict\-free set of specific dependency\-versions, which would otherwise be a very tedious task. +.IP \(bu 2 +Of course, \fBPipfile\fP and Pipenv are still useful for library developers, as they can be used to define a development or test environment. +.IP \(bu 2 +And, of course, there are projects for which the distinction between library and application isn\(aqt that clear. In that case, use \fBinstall_requires\fP alongside Pipenv and \fBPipfile\fP\&. +.UNINDENT +.sp +You can also do this: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ pipenv install \-e . +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +This will tell Pipenv to lock all your \fBsetup.py\fP–declared dependencies. +.SS ☤ Changing Pipenv\(aqs Cache Location +.sp +You can force Pipenv to use a different cache location by setting the environment variable \fBPIPENV_CACHE_DIR\fP to the location you wish. This is useful in the same situations that you would change \fBPIP_CACHE_DIR\fP to a different directory. +.SS ☤ Changing Default Python Versions +.sp +By default, Pipenv will initialize a project using whatever version of python the python3 is. Besides starting a project with the \fB\-\-three\fP or \fB\-\-two\fP flags, you can also use \fBPIPENV_DEFAULT_PYTHON_VERSION\fP to specify what version to use when starting a project when \fB\-\-three\fP or \fB\-\-two\fP aren\(aqt used. +.SS Frequently Encountered Pipenv Problems +.sp +Pipenv is constantly being improved by volunteers, but is still a very young +project with limited resources, and has some quirks that needs to be dealt +with. We need everyone’s help (including yours!). +.sp +Here are some common questions people have using Pipenv. Please take a look +below and see if they resolve your problem. +.sp +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +\fBMake sure you’re running the newest Pipenv version first!\fP +.UNINDENT +.UNINDENT +.SS ☤ Your dependencies could not be resolved +.sp +Make sure your dependencies actually \fIdo\fP resolve. If you’re confident they +are, you may need to clear your resolver cache. Run the following command: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +pipenv lock \-\-clear +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +and try again. +.sp +If this does not work, try manually deleting the whole cache directory. It is +usually one of the following locations: +.INDENT 0.0 +.IP \(bu 2 +\fB~/Library/Caches/pipenv\fP (macOS) +.IP \(bu 2 +\fB%LOCALAPPDATA%\epipenv\epipenv\eCache\fP (Windows) +.IP \(bu 2 +\fB~/.cache/pipenv\fP (other operating systems) +.UNINDENT +.sp +Pipenv does not install prereleases (i.e. a version with an alpha/beta/etc. +suffix, such as \fI1.0b1\fP) by default. You will need to pass the \fB\-\-pre\fP flag +in your command, or set +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +[pipenv] +allow_prereleases = true +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +in your Pipfile. +.SS ☤ No module named +.sp +This is usually a result of mixing Pipenv with system packages. We \fIstrongly\fP +recommend installing Pipenv in an isolated environment. Uninstall all existing +Pipenv installations, and see installing\-pipenv to choose one of the +recommended way to install Pipenv instead. +.SS ☤ My pyenv\-installed Python is not found +.sp +Make sure you have \fBPYENV_ROOT\fP set correctly. Pipenv only supports CPython +distributions, with version name like \fB3.6.4\fP or similar. +.SS ☤ Pipenv does not respect pyenv’s global and local Python versions +.sp +Pipenv by default uses the Python it is installed against to create the +virtualenv. You can set the \fB\-\-python\fP option, or +\fB$PYENV_ROOT/shims/python\fP to let it consult pyenv when choosing the +interpreter. See specifying_versions for more information. +.sp +If you want Pipenv to automatically “do the right thing”, you can set the +environment variable \fBPIPENV_PYTHON\fP to \fB$PYENV_ROOT/shims/python\fP\&. This +will make Pipenv use pyenv’s active Python version to create virtual +environments by default. +.SS ☤ ValueError: unknown locale: UTF\-8 +.sp +macOS has a bug in its locale detection that prevents us from detecting your +shell encoding correctly. This can also be an issue on other systems if the +locale variables do not specify an encoding. +.sp +The workaround is to set the following two environment variables to a standard +localization format: +.INDENT 0.0 +.IP \(bu 2 +\fBLC_ALL\fP +.IP \(bu 2 +\fBLANG\fP +.UNINDENT +.sp +For Bash, for example, you can add the following to your \fB~/.bash_profile\fP: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +export LC_ALL=\(aqen_US.UTF\-8\(aq +export LANG=\(aqen_US.UTF\-8\(aq +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +For Zsh, the file to edit is \fB~/.zshrc\fP\&. +.sp +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +You can change both the \fBen_US\fP and \fBUTF\-8\fP part to the +language/locale and encoding you use. +.UNINDENT +.UNINDENT +.SS ☤ /bin/pip: No such file or directory +.sp +This may be related to your locale setting. See \fI\%☤ ValueError: unknown locale: UTF\-8\fP +for a possible solution. +.SS ☤ \fBshell\fP does not show the virtualenv’s name in prompt +.sp +This is intentional. You can do it yourself with either shell plugins, or +clever \fBPS1\fP configuration. If you really want it back, use +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +pipenv shell \-c +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +instead (not available on Windows). +.SS ☤ Pipenv does not respect dependencies in setup.py +.sp +No, it does not, intentionally. Pipfile and setup.py serve different purposes, +and should not consider each other by default. See pipfile\-vs\-setuppy +for more information. +.SS ☤ Using \fBpipenv run\fP in Supervisor program +.sp +When you configure a supervisor program\(aqs \fBcommand\fP with \fBpipenv run ...\fP, you +need to set locale enviroment variables properly to make it work. +.sp +Add this line under \fB[supervisord]\fP section in \fB/etc/supervisor/supervisord.conf\fP: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +[supervisord] +environment=LC_ALL=\(aqen_US.UTF\-8\(aq,LANG=\(aqen_US.UTF\-8\(aq +.ft P +.fi +.UNINDENT +.UNINDENT +.SS ☤ An exception is raised during \fBLocking dependencies…\fP +.sp +Run \fBpipenv lock \-\-clear\fP and try again. The lock sequence caches results +to speed up subsequent runs. The cache may contain faulty results if a bug +causes the format to corrupt, even after the bug is fixed. \fB\-\-clear\fP flushes +the cache, and therefore removes the bad results. +.SH CONTRIBUTION GUIDES +.SS Development Philosophy +.sp +Pipenv is an open but opinionated tool, created by an open but opinionated developer. +.SS Management Style +.sp +\fI\%Kenneth Reitz\fP is the BDFL. He has final say in any decision related to the Pipenv project. Kenneth is responsible for the direction and form of the library, as well as its presentation. In addition to making decisions based on technical merit, he is responsible for making decisions based on the development philosophy of Pipenv. +.sp +\fI\%Dan Ryan\fP, \fI\%Tzu\-ping Chung\fP, and \fI\%Nate Prewitt\fP are the core contributors. +They are responsible for triaging bug reports, reviewing pull requests and ensuring that Kenneth is kept up to speed with developments around the library. +The day\-to\-day managing of the project is done by the core contributors. They are responsible for making judgements about whether or not a feature request is +likely to be accepted by Kenneth. +.SS Values +.INDENT 0.0 +.IP \(bu 2 +Simplicity is always better than functionality. +.IP \(bu 2 +Listen to everyone, then disregard it. +.IP \(bu 2 +The API is all that matters. Everything else is secondary. +.IP \(bu 2 +Fit the 90% use\-case. Ignore the nay\-sayers. +.UNINDENT +.SS Contributing to Pipenv +.sp +If you\(aqre reading this, you\(aqre probably interested in contributing to Pipenv. +Thank you very much! Open source projects live\-and\-die based on the support +they receive from others, and the fact that you\(aqre even considering +contributing to the Pipenv project is \fIvery\fP generous of you. +.sp +This document lays out guidelines and advice for contributing to this project. +If you\(aqre thinking of contributing, please start by reading this document and +getting a feel for how contributing to this project works. If you have any +questions, feel free to reach out to either \fI\%Dan Ryan\fP, \fI\%Tzu\-ping Chung\fP, +or \fI\%Nate Prewitt\fP, the primary maintainers. +.sp +The guide is split into sections based on the type of contribution you\(aqre +thinking of making, with a section that covers general guidelines for all +contributors. +.SS Be Cordial +.INDENT 0.0 +.INDENT 3.5 +\fBBe cordial or be on your way\fP\&. \fI—Kenneth Reitz\fP +.UNINDENT +.UNINDENT +.sp +Pipenv has one very important rule governing all forms of contribution, +including reporting bugs or requesting features. This golden rule is +"\fI\%be cordial or be on your way\fP". +.sp +\fBAll contributions are welcome\fP, as long as +everyone involved is treated with respect. +.SS Get Early Feedback +.sp +If you are contributing, do not feel the need to sit on your contribution until +it is perfectly polished and complete. It helps everyone involved for you to +seek feedback as early as you possibly can. Submitting an early, unfinished +version of your contribution for feedback in no way prejudices your chances of +getting that contribution accepted, and can save you from putting a lot of work +into a contribution that is not suitable for the project. +.SS Contribution Suitability +.sp +Our project maintainers have the last word on whether or not a contribution is +suitable for Pipenv. All contributions will be considered carefully, but from +time to time, contributions will be rejected because they do not suit the +current goals or needs of the project. +.sp +If your contribution is rejected, don\(aqt despair! As long as you followed these +guidelines, you will have a much better chance of getting your next +contribution accepted. +.SS Code Contributions +.SS Steps for Submitting Code +.sp +When contributing code, you\(aqll want to follow this checklist: +.INDENT 0.0 +.IP 1. 3 +Fork the repository on GitHub. +.IP 2. 3 +\fI\%Run the tests\fP to confirm they all pass on your system. If they don\(aqt, you\(aqll +need to investigate why they fail. If you\(aqre unable to diagnose this +yourself, raise it as a bug report by following the guidelines in this +document: \fI\%Bug Reports\fP\&. +.IP 3. 3 +Write tests that demonstrate your bug or feature. Ensure that they fail. +.IP 4. 3 +Make your change. +.IP 5. 3 +Run the entire test suite again, confirming that all tests pass \fIincluding +the ones you just added\fP\&. +.IP 6. 3 +Send a GitHub Pull Request to the main repository\(aqs \fBmaster\fP branch. +GitHub Pull Requests are the expected method of code collaboration on this +project. +.UNINDENT +.sp +The following sub\-sections go into more detail on some of the points above. +.SS Code Review +.sp +Contributions will not be merged until they\(aqve been code reviewed. You should +implement any code review feedback unless you strongly object to it. In the +event that you object to the code review feedback, you should make your case +clearly and calmly. If, after doing so, the feedback is judged to still apply, +you must either apply the feedback or withdraw your contribution. +.SS Documentation Contributions +.sp +Documentation improvements are always welcome! The documentation files live in +the \fBdocs/\fP directory of the codebase. They\(aqre written in +\fI\%reStructuredText\fP, and use \fI\%Sphinx\fP to generate the full suite of +documentation. +.sp +When contributing documentation, please do your best to follow the style of the +documentation files. This means a soft\-limit of 79 characters wide in your text +files and a semi\-formal, yet friendly and approachable, prose style. +.sp +When presenting Python code, use single\-quoted strings (\fB\(aqhello\(aq\fP instead of +\fB"hello"\fP). +.SS Bug Reports +.sp +Bug reports are hugely important! Before you raise one, though, please check +through the \fI\%GitHub issues\fP, \fBboth open and closed\fP, to confirm that the bug +hasn\(aqt been reported before. Duplicate bug reports are a huge drain on the time +of other contributors, and should be avoided as much as possible. +.SS Run the tests +.sp +Three ways of running the tests are as follows: +.INDENT 0.0 +.IP 1. 3 +\fBmake test\fP (which uses \fBdocker\fP) +.IP 2. 3 +\fB\&./run\-tests.sh\fP or \fBrun\-tests.bat\fP +.IP 3. 3 +Using pipenv: +.INDENT 3.0 +.INDENT 3.5 +.sp +.nf +.ft C +pipenv install \-\-dev +pipenv run pytest +.ft P +.fi +.UNINDENT +.UNINDENT +.UNINDENT +.sp +For the last two, it is important that your environment is setup correctly, and +this may take some work, for example, on a specific Mac installation, the following +steps may be needed: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +# Make sure the tests can access github +if [ "$SSH_AGENT_PID" = "" ] +then + eval \(gassh\-agent\(ga + ssh\-add +fi + +# Use unix like utilities, installed with brew, +# e.g. brew install coreutils +for d in /usr/local/opt/*/libexec/gnubin /usr/local/opt/python/libexec/bin +do + [[ ":$PATH:" != *":$d:"* ]] && PATH="$d:${PATH}" +done + +export PATH + +# PIP_FIND_LINKS currently breaks test_uninstall.py +unset PIP_FIND_LINKS +.ft P +.fi +.UNINDENT +.UNINDENT .SH ☤ PIPENV USAGE +.SS pipenv +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +pipenv [OPTIONS] COMMAND [ARGS]... +.ft P +.fi +.UNINDENT +.UNINDENT +Options.INDENT 0.0 +.TP +.B \-\-where +Output project home information. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-venv +Output virtualenv information. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-py +Output Python interpreter information. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-envs +Output Environment Variable options. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-rm +Remove the virtualenv. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-bare +Minimal output. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-completion +Output completion (to be eval\(aqd). +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-man +Display manpage. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-support +Output diagnostic information for use in GitHub issues. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-site\-packages +Enable site\-packages for the virtualenv. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-python +Specify which version of Python virtualenv should use. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-three, \-\-two +Use Python 3/2 when creating virtualenv. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-clear +Clears caches (pipenv, pip, and pip\-tools). +.UNINDENT +.INDENT 0.0 +.TP +.B \-v, \-\-verbose +Verbose mode. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-pypi\-mirror +Specify a PyPI mirror. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-version +Show the version and exit. +.UNINDENT +.SS check +.sp +Checks for security vulnerabilities and against PEP 508 markers provided in Pipfile. +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +pipenv check [OPTIONS] [ARGS]... +.ft P +.fi +.UNINDENT +.UNINDENT +Options.INDENT 0.0 +.TP +.B \-\-unused +Given a code path, show potentially unused dependencies. +.UNINDENT +.INDENT 0.0 +.TP +.B \-i, \-\-ignore +Ignore specified vulnerability during safety checks. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-python +Specify which version of Python virtualenv should use. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-three, \-\-two +Use Python 3/2 when creating virtualenv. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-clear +Clears caches (pipenv, pip, and pip\-tools). +.UNINDENT +.INDENT 0.0 +.TP +.B \-v, \-\-verbose +Verbose mode. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-pypi\-mirror +Specify a PyPI mirror. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-system +System pip management. +.UNINDENT +Arguments.INDENT 0.0 +.TP +.B ARGS +Optional argument(s) +.UNINDENT +.SS clean +.sp +Uninstalls all packages not specified in Pipfile.lock. +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +pipenv clean [OPTIONS] +.ft P +.fi +.UNINDENT +.UNINDENT +Options.INDENT 0.0 +.TP +.B \-\-bare +Minimal output. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-dry\-run +Just output unneeded packages. +.UNINDENT +.INDENT 0.0 +.TP +.B \-v, \-\-verbose +Verbose mode. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-three, \-\-two +Use Python 3/2 when creating virtualenv. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-python +Specify which version of Python virtualenv should use. +.UNINDENT +.SS graph +.sp +Displays currently\-installed dependency graph information. +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +pipenv graph [OPTIONS] +.ft P +.fi +.UNINDENT +.UNINDENT +Options.INDENT 0.0 +.TP +.B \-\-bare +Minimal output. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-json +Output JSON. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-json\-tree +Output JSON in nested tree. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-reverse +Reversed dependency graph. +.UNINDENT +.SS install +.sp +Installs provided packages and adds them to Pipfile, or (if no packages are given), installs all packages from Pipfile. +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +pipenv install [OPTIONS] [PACKAGES]... +.ft P +.fi +.UNINDENT +.UNINDENT +Options.INDENT 0.0 +.TP +.B \-\-system +System pip management. +.UNINDENT +.INDENT 0.0 +.TP +.B \-c, \-\-code +Import from codebase. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-deploy +Abort if the Pipfile.lock is out\-of\-date, or Python version is wrong. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-skip\-lock +Skip locking mechanisms and use the Pipfile instead during operation. +.UNINDENT +.INDENT 0.0 +.TP +.B \-e, \-\-editable +An editable python package URL or path, often to a VCS repo. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-ignore\-pipfile +Ignore Pipfile when installing, using the Pipfile.lock. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-selective\-upgrade +Update specified packages. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-pre +Allow pre\-releases. +.UNINDENT +.INDENT 0.0 +.TP +.B \-r, \-\-requirements +Import a requirements.txt file. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-extra\-index\-url +URLs to the extra PyPI compatible indexes to query for package lookups. +.UNINDENT +.INDENT 0.0 +.TP +.B \-i, \-\-index +Target PyPI\-compatible package index url. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-sequential +Install dependencies one\-at\-a\-time, instead of concurrently. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-keep\-outdated +Keep out\-dated dependencies from being updated in Pipfile.lock. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-pre +Allow pre\-releases. +.UNINDENT +.INDENT 0.0 +.TP +.B \-d, \-\-dev +Install both develop and default packages. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-python +Specify which version of Python virtualenv should use. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-three, \-\-two +Use Python 3/2 when creating virtualenv. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-clear +Clears caches (pipenv, pip, and pip\-tools). +.UNINDENT +.INDENT 0.0 +.TP +.B \-v, \-\-verbose +Verbose mode. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-pypi\-mirror +Specify a PyPI mirror. +.UNINDENT +Arguments.INDENT 0.0 +.TP +.B PACKAGES +Optional argument(s) +.UNINDENT +Environment variables.INDENT 0.0 +.TP +.B PIPENV_SKIP_LOCK +Provide a default for \fI\%\-\-skip\-lock\fP +.UNINDENT +.INDENT 0.0 +.TP +.B PIP_EXTRA_INDEX_URL +Provide a default for \fI\%\-\-extra\-index\-url\fP +.UNINDENT +.INDENT 0.0 +.TP +.B PIP_INDEX_URL +Provide a default for \fI\%\-i\fP +.UNINDENT +.SS lock +.sp +Generates Pipfile.lock. +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +pipenv lock [OPTIONS] +.ft P +.fi +.UNINDENT +.UNINDENT +Options.INDENT 0.0 +.TP +.B \-r, \-\-requirements +Generate output in requirements.txt format. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-keep\-outdated +Keep out\-dated dependencies from being updated in Pipfile.lock. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-pre +Allow pre\-releases. +.UNINDENT +.INDENT 0.0 +.TP +.B \-d, \-\-dev +Install both develop and default packages. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-python +Specify which version of Python virtualenv should use. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-three, \-\-two +Use Python 3/2 when creating virtualenv. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-clear +Clears caches (pipenv, pip, and pip\-tools). +.UNINDENT +.INDENT 0.0 +.TP +.B \-v, \-\-verbose +Verbose mode. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-pypi\-mirror +Specify a PyPI mirror. +.UNINDENT +.SS open +.sp +View a given module in your editor. +.sp +This uses the EDITOR environment variable. You can temporarily override it, +for example: +.INDENT 0.0 +.INDENT 3.5 +EDITOR=atom pipenv open requests +.UNINDENT +.UNINDENT +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +pipenv open [OPTIONS] MODULE +.ft P +.fi +.UNINDENT +.UNINDENT +Options.INDENT 0.0 +.TP +.B \-\-python +Specify which version of Python virtualenv should use. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-three, \-\-two +Use Python 3/2 when creating virtualenv. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-clear +Clears caches (pipenv, pip, and pip\-tools). +.UNINDENT +.INDENT 0.0 +.TP +.B \-v, \-\-verbose +Verbose mode. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-pypi\-mirror +Specify a PyPI mirror. +.UNINDENT +Arguments.INDENT 0.0 +.TP +.B MODULE +Required argument +.UNINDENT +.SS run +.sp +Spawns a command installed into the virtualenv. +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +pipenv run [OPTIONS] COMMAND [ARGS]... +.ft P +.fi +.UNINDENT +.UNINDENT +Options.INDENT 0.0 +.TP +.B \-\-python +Specify which version of Python virtualenv should use. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-three, \-\-two +Use Python 3/2 when creating virtualenv. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-clear +Clears caches (pipenv, pip, and pip\-tools). +.UNINDENT +.INDENT 0.0 +.TP +.B \-v, \-\-verbose +Verbose mode. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-pypi\-mirror +Specify a PyPI mirror. +.UNINDENT +Arguments.INDENT 0.0 +.TP +.B COMMAND +Required argument +.UNINDENT +.INDENT 0.0 +.TP +.B ARGS +Optional argument(s) +.UNINDENT +.SS shell +.sp +Spawns a shell within the virtualenv. +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +pipenv shell [OPTIONS] [SHELL_ARGS]... +.ft P +.fi +.UNINDENT +.UNINDENT +Options.INDENT 0.0 +.TP +.B \-\-fancy +Run in shell in fancy mode (for elegantly configured shells). +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-anyway +Always spawn a subshell, even if one is already spawned. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-pypi\-mirror +Specify a PyPI mirror. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-three, \-\-two +Use Python 3/2 when creating virtualenv. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-python +Specify which version of Python virtualenv should use. +.UNINDENT +Arguments.INDENT 0.0 +.TP +.B SHELL_ARGS +Optional argument(s) +.UNINDENT +.SS sync +.sp +Installs all packages specified in Pipfile.lock. +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +pipenv sync [OPTIONS] +.ft P +.fi +.UNINDENT +.UNINDENT +Options.INDENT 0.0 +.TP +.B \-\-bare +Minimal output. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-sequential +Install dependencies one\-at\-a\-time, instead of concurrently. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-keep\-outdated +Keep out\-dated dependencies from being updated in Pipfile.lock. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-pre +Allow pre\-releases. +.UNINDENT +.INDENT 0.0 +.TP +.B \-d, \-\-dev +Install both develop and default packages. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-python +Specify which version of Python virtualenv should use. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-three, \-\-two +Use Python 3/2 when creating virtualenv. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-clear +Clears caches (pipenv, pip, and pip\-tools). +.UNINDENT +.INDENT 0.0 +.TP +.B \-v, \-\-verbose +Verbose mode. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-pypi\-mirror +Specify a PyPI mirror. +.UNINDENT +.SS uninstall +.sp +Un\-installs a provided package and removes it from Pipfile. +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +pipenv uninstall [OPTIONS] [PACKAGES]... +.ft P +.fi +.UNINDENT +.UNINDENT +Options.INDENT 0.0 +.TP +.B \-\-skip\-lock, \-\-lock +Lock afterwards. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-all\-dev +Un\-install all package from [dev\-packages]. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-all +Purge all package(s) from virtualenv. Does not edit Pipfile. +.UNINDENT +.INDENT 0.0 +.TP +.B \-e, \-\-editable +An editable python package URL or path, often to a VCS repo. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-skip\-lock +Skip locking mechanisms and use the Pipfile instead during operation. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-keep\-outdated +Keep out\-dated dependencies from being updated in Pipfile.lock. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-pre +Allow pre\-releases. +.UNINDENT +.INDENT 0.0 +.TP +.B \-d, \-\-dev +Install both develop and default packages. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-python +Specify which version of Python virtualenv should use. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-three, \-\-two +Use Python 3/2 when creating virtualenv. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-clear +Clears caches (pipenv, pip, and pip\-tools). +.UNINDENT +.INDENT 0.0 +.TP +.B \-v, \-\-verbose +Verbose mode. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-pypi\-mirror +Specify a PyPI mirror. +.UNINDENT +Arguments.INDENT 0.0 +.TP +.B PACKAGES +Optional argument(s) +.UNINDENT +Environment variables.INDENT 0.0 +.TP +.B PIPENV_SKIP_LOCK +Provide a default for \fI\%\-\-skip\-lock\fP +.UNINDENT +.SS update +.sp +Runs lock, then sync. +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +pipenv update [OPTIONS] [PACKAGES]... +.ft P +.fi +.UNINDENT +.UNINDENT +Options.INDENT 0.0 +.TP +.B \-\-bare +Minimal output. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-outdated +List out\-of\-date dependencies. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-dry\-run +List out\-of\-date dependencies. +.UNINDENT +.INDENT 0.0 +.TP +.B \-e, \-\-editable +An editable python package URL or path, often to a VCS repo. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-ignore\-pipfile +Ignore Pipfile when installing, using the Pipfile.lock. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-selective\-upgrade +Update specified packages. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-pre +Allow pre\-releases. +.UNINDENT +.INDENT 0.0 +.TP +.B \-r, \-\-requirements +Import a requirements.txt file. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-extra\-index\-url +URLs to the extra PyPI compatible indexes to query for package lookups. +.UNINDENT +.INDENT 0.0 +.TP +.B \-i, \-\-index +Target PyPI\-compatible package index url. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-sequential +Install dependencies one\-at\-a\-time, instead of concurrently. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-keep\-outdated +Keep out\-dated dependencies from being updated in Pipfile.lock. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-pre +Allow pre\-releases. +.UNINDENT +.INDENT 0.0 +.TP +.B \-d, \-\-dev +Install both develop and default packages. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-python +Specify which version of Python virtualenv should use. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-three, \-\-two +Use Python 3/2 when creating virtualenv. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-clear +Clears caches (pipenv, pip, and pip\-tools). +.UNINDENT +.INDENT 0.0 +.TP +.B \-v, \-\-verbose +Verbose mode. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-pypi\-mirror +Specify a PyPI mirror. +.UNINDENT +Arguments.INDENT 0.0 +.TP +.B PACKAGES +Optional argument(s) +.UNINDENT +Environment variables.INDENT 0.0 +.TP +.B PIP_EXTRA_INDEX_URL +Provide a default for \fI\%\-\-extra\-index\-url\fP +.UNINDENT +.INDENT 0.0 +.TP +.B PIP_INDEX_URL +Provide a default for \fI\%\-i\fP +.UNINDENT .INDENT 0.0 .IP \(bu 2 genindex diff --git a/pipenv/project.py b/pipenv/project.py index c2dc9668..0067348c 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -4,18 +4,21 @@ import json import os import re import sys +import glob import base64 import fnmatch import hashlib -import contoml from first import first from cached_property import cached_property +import operator import pipfile import pipfile.api import six import vistir import toml +import tomlkit +from .environment import Environment from .cmdparse import Script from .utils import ( pep423_name, @@ -23,8 +26,10 @@ from .utils import ( find_requirements, is_editable, cleanup_toml, + convert_toml_outline_tables, is_installable_file, is_valid_url, + get_url_name, normalize_drive, python_version, safe_expandvars, @@ -32,6 +37,7 @@ from .utils import ( get_workon_home, is_virtual_environment, looks_like_dir, + get_canonical_names ) from .environments import ( PIPENV_MAX_DEPTH, @@ -42,21 +48,24 @@ from .environments import ( PIPENV_PYTHON, PIPENV_DEFAULT_PYTHON_VERSION, ) -from requirementslib.utils import is_vcs def _normalized(p): if p is None: return None loc = vistir.compat.Path(p) - if loc.is_absolute(): - return normalize_drive(str(loc)) - else: + if not loc.is_absolute(): try: loc = loc.resolve() except OSError: loc = loc.absolute() - return normalize_drive(str(loc)) + # Recase the path properly on Windows. From https://stackoverflow.com/a/35229734/5043728 + if os.name == 'nt': + matches = glob.glob(re.sub(r'([^:/\\])(?=[/\\]|$)', r'[\1]', str(loc))) + path_str = matches and matches[0] or str(loc) + else: + path_str = str(loc) + return normalize_drive(path_str) DEFAULT_NEWLINES = u"\n" @@ -68,7 +77,7 @@ class _LockFileEncoder(json.JSONEncoder): This adds a few characteristics to the encoder: * The JSON is always prettified with indents and spaces. - * PrettyTOML's container elements are seamlessly encodable. + * TOMLKit's container elements are seamlessly encodable. * The output is always UTF-8-encoded text, never binary, even on Python 2. """ @@ -78,10 +87,8 @@ class _LockFileEncoder(json.JSONEncoder): ) def default(self, obj): - from prettytoml.elements.common import ContainerElement, TokenElement - - if isinstance(obj, (ContainerElement, TokenElement)): - return obj.primitive_value + if isinstance(obj, vistir.compat.Path): + obj = obj.as_posix() return super(_LockFileEncoder, self).default(obj) def encode(self, obj): @@ -144,7 +151,11 @@ class Project(object): self._lockfile_newlines = DEFAULT_NEWLINES self._requirements_location = None self._original_dir = os.path.abspath(os.curdir) - self.which = which + self._environment = None + self._which = which + self._build_system = { + "requires": ["setuptools", "wheel"] + } self.python_version = python_version # Hack to skip this during pipenv run, or -r. if ("run" not in sys.argv) and chdir: @@ -162,6 +173,7 @@ class Project(object): def _build_package_list(self, package_section): """Returns a list of packages for pip-tools to consume.""" + from pipenv.vendor.requirementslib.utils import is_vcs ps = {} # TODO: Separate the logic for showing packages from the filters for supplying pip-tools for k, v in self.parsed_pipfile.get(package_section, {}).items(): @@ -266,18 +278,33 @@ class Project(object): return False def get_location_for_virtualenv(self): - if self.is_venv_in_project(): - return os.path.join(self.project_directory, ".venv") + # If there's no project yet, set location based on config. + if not self.project_directory: + if self.is_venv_in_project(): + return os.path.abspath(".venv") + return str(get_workon_home().joinpath(self.virtualenv_name)) - name = self.virtualenv_name - if self.project_directory: - venv_path = os.path.join(self.project_directory, ".venv") - if os.path.exists(venv_path) and not os.path.isdir(".venv"): - with io.open(venv_path, "r") as f: - name = f.read().strip() - # Assume file's contents is a path if it contains slashes. - if looks_like_dir(name): - return vistir.compat.Path(name).absolute().as_posix() + dot_venv = os.path.join(self.project_directory, ".venv") + + # If there's no .venv in project root, set location based on config. + if not os.path.exists(dot_venv): + if self.is_venv_in_project(): + return dot_venv + return str(get_workon_home().joinpath(self.virtualenv_name)) + + # If .venv in project root is a directory, use it. + if os.path.isdir(dot_venv): + return dot_venv + + # Now we assume .venv in project root is a file. Use its content. + with io.open(dot_venv) as f: + name = f.read().strip() + + # If content looks like a path, use it as a relative path. + # Otherwise use directory named after content in WORKON_HOME. + if looks_like_dir(name): + path = vistir.compat.Path(self.project_directory, name) + return path.absolute().as_posix() return str(get_workon_home().joinpath(name)) @property @@ -287,40 +314,49 @@ class Project(object): import pkg_resources return pkg_resources.WorkingSet(sys_path) - def find_egg(self, egg_dist): - import site - from distutils import sysconfig as distutils_sysconfig - site_packages = distutils_sysconfig.get_python_lib() - search_filename = "{0}.egg-link".format(egg_dist.project_name) - try: - user_site = site.getusersitepackages() - except AttributeError: - user_site = site.USER_SITE - search_locations = [site_packages, user_site] - for site_directory in search_locations: - egg = os.path.join(site_directory, search_filename) - if os.path.isfile(egg): - return egg + @property + def installed_packages(self): + return self.environment.get_installed_packages() - def locate_dist(self, dist): - location = self.find_egg(dist) - if not location: - return dist.location + @property + def installed_package_names(self): + return get_canonical_names([pkg.key for pkg in self.installed_packages]) - def dist_is_in_project(self, dist): - prefix = _normalized(self.env_paths["prefix"]) - location = self.locate_dist(dist) - if not location: - return False - return _normalized(location).startswith(prefix) + @property + def lockfile_package_names(self): + dev_keys = get_canonical_names(self.lockfile_content["develop"].keys()) + default_keys = get_canonical_names(self.lockfile_content["default"].keys()) + return { + "dev": dev_keys, + "default": default_keys, + "combined": dev_keys | default_keys + } - def get_installed_packages(self): - workingset = self.working_set - if self.virtualenv_exists: - packages = [pkg for pkg in workingset if self.dist_is_in_project(pkg)] - else: - packages = [pkg for pkg in packages] - return packages + @property + def pipfile_package_names(self): + dev_keys = get_canonical_names(self.dev_packages.keys()) + default_keys = get_canonical_names(self.packages.keys()) + return { + "dev": dev_keys, + "default": default_keys, + "combined": dev_keys | default_keys + } + + @property + def environment(self): + if not self._environment: + prefix = self.get_location_for_virtualenv() + is_venv = prefix == sys.prefix + sources = self.sources if self.sources else [DEFAULT_SOURCE,] + self._environment = Environment( + prefix=prefix, is_venv=is_venv, sources=sources, pipfile=self.parsed_pipfile, + project=self + ) + self._environment.add_dist("pipenv") + return self._environment + + def get_outdated_packages(self): + return self.environment.get_outdated_packages(pre=self.pipfile.get("pre", False)) @classmethod def _sanitize(cls, name): @@ -358,7 +394,7 @@ class Project(object): # In-project venv # "Proper" path casing (on non-case-sensitive filesystems). if ( - fnmatch.fnmatch("A", "a") + not fnmatch.fnmatch("A", "a") or self.is_venv_in_project() or get_workon_home().joinpath(venv_name).exists() ): @@ -484,32 +520,33 @@ class Project(object): _pipfile_cache.clear() def _parse_pipfile(self, contents): - # If any outline tables are present... - if ("[packages." in contents) or ("[dev-packages." in contents): - data = toml.loads(contents) - # Convert all outline tables to inline tables. - for section in ("packages", "dev-packages"): - for package in data.get(section, {}): - # Convert things to inline tables — fancy :) - if hasattr(data[section][package], "keys"): - _data = data[section][package] - data[section][package] = toml.TomlDecoder().get_empty_inline_table() - data[section][package].update(_data) - toml_encoder = toml.TomlEncoder(preserve=True) + try: + return tomlkit.parse(contents) + except Exception: # We lose comments here, but it's for the best.) - try: - return contoml.loads(toml.dumps(data, encoder=toml_encoder)) - - except RuntimeError: - return toml.loads(toml.dumps(data, encoder=toml_encoder)) - - else: # Fallback to toml parser, for large files. - try: - return contoml.loads(contents) + return toml.loads(contents) - except Exception: - return toml.loads(contents) + def _read_pyproject(self): + pyproject = self.path_to("pyproject.toml") + if os.path.exists(pyproject): + self._pyproject = toml.load(pyproject) + build_system = self._pyproject.get("build-system", None) + if not os.path.exists(self.path_to("setup.py")): + if not build_system or not build_system.get("requires"): + build_system = { + "requires": ["setuptools>=38.2.5", "wheel"], + "build-backend": "setuptools.build_meta", + } + self._build_system = build_system + + @property + def build_requires(self): + return self._build_system.get("requires", []) + + @property + def build_backend(self): + return self._build_system.get("build-backend", None) @property def settings(self): @@ -556,6 +593,12 @@ class Project(object): lockfile[section][norm_key] = lock_section.pop(key) return lockfile + @property + def _pipfile(self): + from .vendor.requirementslib.models.pipfile import Pipfile as ReqLibPipfile + pf = ReqLibPipfile.load(self.pipfile_location) + return pf + @property def lockfile_location(self): return "{0}.lock".format(self.pipfile_location) @@ -570,17 +613,22 @@ class Project(object): def _get_editable_packages(self, dev=False): section = "dev-packages" if dev else "packages" + # section = "{0}-editable".format(section) packages = { k: v + # for k, v in self._pipfile[section].items() for k, v in self.parsed_pipfile.get(section, {}).items() - if is_editable(v) + if is_editable(k) or is_editable(v) } return packages def _get_vcs_packages(self, dev=False): + from pipenv.vendor.requirementslib.utils import is_vcs section = "dev-packages" if dev else "packages" + # section = "{0}-vcs".format(section) packages = { k: v + # for k, v in self._pipfile[section].items() for k, v in self.parsed_pipfile.get(section, {}).items() if is_vcs(v) or is_vcs(k) } @@ -638,8 +686,9 @@ class Project(object): def create_pipfile(self, python=None): """Creates the Pipfile, filled with juicy defaults.""" - from .patched.notpip._internal import ConfigOptionParser - from .patched.notpip._internal.cmdoptions import make_option_group, index_group + from .vendor.pip_shims.shims import ( + ConfigOptionParser, make_option_group, index_group + ) config_parser = ConfigOptionParser(name=self.name) config_parser.add_option_group(make_option_group(index_group, config_parser)) @@ -649,7 +698,7 @@ class Project(object): .lstrip("\n") .split("\n") ) - sources = [DEFAULT_SOURCE] + sources = [DEFAULT_SOURCE,] for i, index in enumerate(indexes): if not index: continue @@ -676,23 +725,93 @@ class Project(object): version = python_version(required_python) or PIPENV_DEFAULT_PYTHON_VERSION if version and len(version) >= 3: data[u"requires"] = {"python_version": version[: len("2.7")]} - self.write_toml(data, "Pipfile") + self.write_toml(data) + + @classmethod + def populate_source(cls, source): + """Derive missing values of source from the existing fields.""" + # Only URL pararemter is mandatory, let the KeyError be thrown. + if "name" not in source: + source["name"] = get_url_name(source["url"]) + if "verify_ssl" not in source: + source["verify_ssl"] = "https://" in source["url"] + if not isinstance(source["verify_ssl"], bool): + source["verify_ssl"] = source["verify_ssl"].lower() == "true" + return source + + def get_or_create_lockfile(self): + from pipenv.vendor.requirementslib.models.lockfile import Lockfile as Req_Lockfile + lockfile = None + if self.lockfile_exists: + try: + lockfile = Req_Lockfile.load(self.lockfile_location) + except OSError: + lockfile = Req_Lockfile.from_data(self.lockfile_location, self.lockfile_content) + else: + lockfile = Req_Lockfile.from_data(path=self.lockfile_location, data=self._lockfile, meta_from_project=False) + if lockfile._lockfile is not None: + return lockfile + if self.lockfile_exists and self.lockfile_content: + lockfile_dict = self.lockfile_content.copy() + sources = lockfile_dict.get("_meta", {}).get("sources", []) + if not sources: + sources = self.pipfile_sources + elif not isinstance(sources, list): + sources = [sources,] + lockfile_dict["_meta"]["sources"] = [ + self.populate_source(s) for s in sources + ] + _created_lockfile = Req_Lockfile.from_data( + path=self.lockfile_location, data=lockfile_dict, meta_from_project=False + ) + lockfile._lockfile = lockfile.projectfile.model = _created_lockfile + return lockfile + elif self.pipfile_exists: + lockfile_dict = { + "default": self._lockfile["default"].copy(), + "develop": self._lockfile["develop"].copy() + } + lockfile_dict.update({"_meta": self.get_lockfile_meta()}) + _created_lockfile = Req_Lockfile.from_data( + path=self.lockfile_location, data=lockfile_dict, meta_from_project=False + ) + lockfile._lockfile = _created_lockfile + return lockfile + + def get_lockfile_meta(self): + from .vendor.plette.lockfiles import PIPFILE_SPEC_CURRENT + sources = self.lockfile_content.get("_meta", {}).get("sources", []) + if not sources: + sources = self.pipfile_sources + elif not isinstance(sources, list): + sources = [sources,] + return { + "hash": {"sha256": self.calculate_pipfile_hash()}, + "pipfile-spec": PIPFILE_SPEC_CURRENT, + "sources": sources, + "requires": self.parsed_pipfile.get("requires", {}) + } def write_toml(self, data, path=None): """Writes the given data structure out as TOML.""" if path is None: path = self.pipfile_location + data = convert_toml_outline_tables(data) try: - formatted_data = contoml.dumps(data).rstrip() + formatted_data = tomlkit.dumps(data).rstrip() except Exception: + document = tomlkit.document() for section in ("packages", "dev-packages"): + document[section] = tomlkit.container.Table() + # Convert things to inline tables — fancy :) for package in data.get(section, {}): - # Convert things to inline tables — fancy :) if hasattr(data[section][package], "keys"): - _data = data[section][package] - data[section][package] = toml.TomlDecoder().get_empty_inline_table() - data[section][package].update(_data) - formatted_data = toml.dumps(data).rstrip() + table = tomlkit.inline_table() + table.update(data[section][package]) + document[section][package] = table + else: + document[section][package] = tomlkit.string(data[section][package]) + formatted_data = tomlkit.dumps(document).rstrip() if ( vistir.compat.Path(path).absolute() @@ -736,7 +855,7 @@ class Project(object): @property def sources(self): if self.lockfile_exists and hasattr(self.lockfile_content, "keys"): - meta_ = self.lockfile_content["_meta"] + meta_ = self.lockfile_content.get("_meta", {}) sources_ = meta_.get("sources") if sources_: return sources_ @@ -795,6 +914,22 @@ class Project(object): del p[key][name] self.write_toml(p) + def remove_packages_from_pipfile(self, packages): + parsed = self.parsed_pipfile + packages = set([pep423_name(pkg) for pkg in packages]) + for section in ("dev-packages", "packages"): + pipfile_section = parsed.get(section, {}) + pipfile_packages = set([ + pep423_name(pkg_name) for pkg_name in pipfile_section.keys() + ]) + to_remove = packages & pipfile_packages + # The normal toml parser can't handle deleting packages with preceding newlines + is_dev = section == "dev-packages" + for pkg in to_remove: + pkg_name = self.get_package_name_in_pipfile(pkg, dev=is_dev) + del parsed[section][pkg_name] + self.write_toml(parsed) + def add_package_to_pipfile(self, package, dev=False): from .vendor.requirementslib import Requirement @@ -923,33 +1058,27 @@ class Project(object): # Return whether or not values have been changed. return changed_values - @property - def _pyversion(self): - include_dir = vistir.compat.Path(self.virtualenv_location) / "include" - python_path = next((x for x in include_dir.iterdir() if x.name.startswith("python")), None) - if python_path: - python_version = python_path.name.replace("python", "") - py_version_short, abiflags = python_version[:3], python_version[3:] - return {"py_version_short": py_version_short, "abiflags": abiflags} - return {} + @cached_property + def finders(self): + from .vendor.pythonfinder import Finder + scripts_dirname = "Scripts" if os.name == "nt" else "bin" + scripts_dir = os.path.join(self.virtualenv_location, scripts_dirname) + finders = [ + Finder(path=scripts_dir, global_search=gs, system=False) + for gs in (False, True) + ] + return finders @property - def env_paths(self): - import sysconfig - location = self.virtualenv_location if self.virtualenv_location else sys.prefix - prefix = vistir.compat.Path(location).as_posix() - scheme = sysconfig._get_default_scheme() - config = { - "base": prefix, - "installed_base": prefix, - "platbase": prefix, - "installed_platbase": prefix - } - config.update(self._pyversion) - paths = { - k: v.format(**config) - for k, v in sysconfig._INSTALL_SCHEMES[scheme].items() - } - if "prefix" not in paths: - paths["prefix"] = prefix - return paths + def finder(self): + return next(iter(self.finders), None) + + def which(self, search, as_path=True): + find = operator.methodcaller("which", search) + result = next(iter(filter(None, (find(finder) for finder in self.finders))), None) + if not result: + result = self._which(search) + else: + if as_path: + result = str(result.path) + return result diff --git a/pipenv/resolver.py b/pipenv/resolver.py index 6526d990..e87f3243 100644 --- a/pipenv/resolver.py +++ b/pipenv/resolver.py @@ -3,57 +3,52 @@ import sys import json import logging -os.environ["PIP_PYTHON_PATH"] = sys.executable +os.environ["PIP_PYTHON_PATH"] = str(sys.executable) def _patch_path(): + import site pipenv_libdir = os.path.dirname(os.path.abspath(__file__)) + pipenv_site_dir = os.path.dirname(pipenv_libdir) + site.addsitedir(pipenv_site_dir) for _dir in ("vendor", "patched"): sys.path.insert(0, os.path.join(pipenv_libdir, _dir)) - site_packages_dir = os.path.dirname(pipenv_libdir) - if site_packages_dir not in sys.path: - sys.path.append(site_packages_dir) + + +def get_parser(): + from argparse import ArgumentParser + parser = ArgumentParser("pipenv-resolver") + parser.add_argument("--pre", action="store_true", default=False) + parser.add_argument("--clear", action="store_true", default=False) + parser.add_argument("--verbose", "-v", action="count", default=False) + parser.add_argument("--debug", action="store_true", default=False) + parser.add_argument("--system", action="store_true", default=False) + parser.add_argument("--requirements-dir", metavar="requirements_dir", action="store", + default=os.environ.get("PIPENV_REQ_DIR")) + parser.add_argument("packages", nargs="*") + return parser def which(*args, **kwargs): return sys.executable -def main(): - do_pre = "--pre" in " ".join(sys.argv) - do_clear = "--clear" in " ".join(sys.argv) - is_verbose = "--verbose" in " ".join(sys.argv) - is_debug = "--debug" in " ".join(sys.argv) - system = "--system" in " ".join(sys.argv) - new_sys_argv = [] - for v in sys.argv: - if v.startswith("--"): - continue - - else: - new_sys_argv.append(v) - sys.argv = new_sys_argv - - os.environ["PIP_PYTHON_VERSION"] = ".".join([str(s) for s in sys.version_info[:3]]) - os.environ["PIP_PYTHON_PATH"] = sys.executable - - verbosity = int(os.environ.get("PIPENV_VERBOSITY", 0)) - if is_debug: - verbosity = max(verbosity, 2) - elif is_verbose: - verbosity = max(verbosity, 1) - if verbosity > 1: # Shit's getting real at this point. +def handle_parsed_args(parsed): + if parsed.debug: + parsed.verbose = max(parsed.verbose, 2) + if parsed.verbose > 1: logging.getLogger("notpip").setLevel(logging.DEBUG) - elif verbosity > 0: + elif parsed.verbose > 0: logging.getLogger("notpip").setLevel(logging.INFO) - if "PIPENV_PACKAGES" in os.environ: - packages = os.environ["PIPENV_PACKAGES"].strip().split("\n") - else: - packages = sys.argv[1:] - for i, package in enumerate(packages): - if package.startswith("--"): - del packages[i] + parsed.packages += os.environ.get("PIPENV_PACKAGES", "").strip().split("\n") + return parsed + + +def _main(pre, clear, verbose, system, requirements_dir, packages): + os.environ["PIP_PYTHON_VERSION"] = ".".join([str(s) for s in sys.version_info[:3]]) + os.environ["PIP_PYTHON_PATH"] = str(sys.executable) + from pipenv.utils import create_mirror_source, resolve_deps, replace_pypi_sources pypi_mirror_source = ( @@ -62,7 +57,7 @@ def main(): else None ) - def resolve(packages, pre, project, sources, clear, system): + def resolve(packages, pre, project, sources, clear, system, requirements_dir=None): return resolve_deps( packages, which, @@ -71,23 +66,23 @@ def main(): sources=sources, clear=clear, allow_global=system, + req_dir=requirements_dir ) from pipenv.core import project - sources = ( replace_pypi_sources(project.pipfile_sources, pypi_mirror_source) if pypi_mirror_source else project.pipfile_sources ) - print("using sources: %s" % sources) results = resolve( packages, - pre=do_pre, + pre=pre, project=project, sources=sources, - clear=do_clear, + clear=clear, system=system, + requirements_dir=requirements_dir, ) print("RESULTS:") if results: @@ -96,6 +91,36 @@ def main(): print(json.dumps([])) +def main(): + _patch_path() + import warnings + from pipenv.vendor.vistir.compat import ResourceWarning + warnings.simplefilter("ignore", category=ResourceWarning) + import io + import six + if six.PY3: + import atexit + stdout_wrapper = io.TextIOWrapper(sys.stdout.buffer, encoding='utf8') + atexit.register(stdout_wrapper.close) + stderr_wrapper = io.TextIOWrapper(sys.stderr.buffer, encoding='utf8') + atexit.register(stderr_wrapper.close) + sys.stdout = stdout_wrapper + sys.stderr = stderr_wrapper + else: + from pipenv._compat import force_encoding + force_encoding() + os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = str("1") + os.environ["PYTHONIOENCODING"] = str("utf-8") + parser = get_parser() + parsed, remaining = parser.parse_known_args() + # sys.argv = remaining + parsed = handle_parsed_args(parsed) + _main(parsed.pre, parsed.clear, parsed.verbose, parsed.system, + parsed.requirements_dir, parsed.packages) + + if __name__ == "__main__": _patch_path() + from pipenv.vendor import colorama + colorama.init() main() diff --git a/pipenv/utils.py b/pipenv/utils.py index b965e46d..9f62e2b2 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -1,21 +1,30 @@ # -*- coding: utf-8 -*- +import contextlib import errno import logging import os import re import shutil +import stat import sys +import toml +import tomlkit +import warnings import crayons import parse import six -import stat -import warnings from click import echo as click_echo from first import first from vistir.misc import fs_str +six.add_move(six.MovedAttribute("Mapping", "collections", "collections.abc")) +six.add_move(six.MovedAttribute("Sequence", "collections", "collections.abc")) +from six.moves import Mapping, Sequence + +from vistir.compat import ResourceWarning + try: from weakref import finalize except ImportError: @@ -38,14 +47,8 @@ from contextlib import contextmanager from . import environments from .pep508checker import lookup -six.add_move(six.MovedAttribute("Mapping", "collections", "collections.abc")) from six.moves.urllib.parse import urlparse -from six.moves import Mapping - -if six.PY2: - - class ResourceWarning(Warning): - pass +from urllib3 import util as urllib3_util specifiers = [k for k in lookup.keys()] @@ -93,6 +96,24 @@ def cleanup_toml(tml): return toml +def convert_toml_outline_tables(parsed): + """Converts all outline tables to inline tables.""" + if isinstance(parsed, tomlkit.container.Container): + empty_inline_table = tomlkit.inline_table + else: + empty_inline_table = toml.TomlDecoder().get_empty_inline_table + for section in ("packages", "dev-packages"): + table_data = parsed.get(section, {}) + for package, value in table_data.items(): + if hasattr(value, "keys") and not isinstance( + value, (tomlkit.items.InlineTable, toml.decoder.InlineTableDict) + ): + table = empty_inline_table() + table.update(value) + table_data[package] = table + return parsed + + def parse_python_version(output): """Parse a Python version output returned by `python --version`. @@ -127,20 +148,14 @@ def parse_python_version(output): def python_version(path_to_python): - import delegator + from .vendor.pythonfinder.utils import get_python_version if not path_to_python: return None try: - c = delegator.run([path_to_python, "--version"], block=False) + version = get_python_version(path_to_python) except Exception: return None - c.block() - version = parse_python_version(c.out.strip() or c.err.strip()) - try: - version = u"{major}.{minor}.{micro}".format(**version) - except TypeError: - return None return version @@ -194,7 +209,7 @@ def prepare_pip_source_args(sources, pip_args=None): # Trust the host if it's not verified. if not sources[0].get("verify_ssl", True): pip_args.extend( - ["--trusted-host", urlparse(sources[0]["url"]).hostname] + ["--trusted-host", urllib3_util.parse_url(sources[0]["url"]).host] ) # Add additional sources as extra indexes. if len(sources) > 1: @@ -203,11 +218,236 @@ def prepare_pip_source_args(sources, pip_args=None): # Trust the host if it's not verified. if not source.get("verify_ssl", True): pip_args.extend( - ["--trusted-host", urlparse(source["url"]).hostname] + ["--trusted-host", urllib3_util.parse_url(source["url"]).host] ) return pip_args +def get_resolver_metadata(deps, index_lookup, markers_lookup, project, sources): + from .vendor.requirementslib.models.requirements import Requirement + constraints = [] + for dep in deps: + if not dep: + continue + url = None + indexes, trusted_hosts, remainder = parse_indexes(dep) + if indexes: + url = indexes[0] + dep = " ".join(remainder) + req = Requirement.from_line(dep) + constraints.append(req.constraint_line) + + if url: + index_lookup[req.name] = project.get_source(url=url).get("name") + # strip the marker and re-add it later after resolution + # but we will need a fallback in case resolution fails + # eg pypiwin32 + if req.markers: + markers_lookup[req.name] = req.markers.replace('"', "'") + return constraints + + +class Resolver(object): + def __init__(self, constraints, req_dir, project, sources, clear=False, pre=False): + from pipenv.patched.piptools import logging as piptools_logging + if environments.is_verbose(): + logging.log.verbose = True + piptools_logging.log.verbose = True + self.initial_constraints = constraints + self.req_dir = req_dir + self.project = project + self.sources = sources + self.resolved_tree = set() + self.hashes = {} + self.clear = clear + self.pre = pre + self.results = None + self._pip_args = None + self._constraints = None + self._parsed_constraints = None + self._resolver = None + self._repository = None + self._session = None + self._constraint_file = None + self._pip_options = None + self._pip_command = None + self._retry_attempts = 0 + + def __repr__(self): + return ( + "".format(self=self) + ) + + def _get_pip_command(self): + from pip_shims.shims import Command + + class PipCommand(Command): + """Needed for pip-tools.""" + + name = "PipCommand" + + from pipenv.patched.piptools.scripts.compile import get_pip_command + return get_pip_command() + + @property + def pip_command(self): + if self._pip_command is None: + self._pip_command = self._get_pip_command() + return self._pip_command + + def prepare_pip_args(self): + pip_args = [] + if self.sources: + pip_args = prepare_pip_source_args(self.sources, pip_args) + return pip_args + + @property + def pip_args(self): + if self._pip_args is None: + self._pip_args = self.prepare_pip_args() + return self._pip_args + + def prepare_constraint_file(self): + from pipenv.vendor.vistir.path import create_tracked_tempfile + constraints_file = create_tracked_tempfile( + mode="w", + prefix="pipenv-", + suffix="-constraints.txt", + dir=self.req_dir, + delete=False, + ) + if self.sources: + requirementstxt_sources = " ".join(self.pip_args) if self.pip_args else "" + requirementstxt_sources = requirementstxt_sources.replace(" --", "\n--") + constraints_file.write(u"{0}\n".format(requirementstxt_sources)) + constraints = self.initial_constraints + constraints_file.write(u"\n".join([c for c in constraints])) + constraints_file.close() + return constraints_file.name + + @property + def constraint_file(self): + if self._constraint_file is None: + self._constraint_file = self.prepare_constraint_file() + return self._constraint_file + + @property + def pip_options(self): + if self._pip_options is None: + pip_options, _ = self.pip_command.parser.parse_args(self.pip_args) + pip_options.cache_dir = environments.PIPENV_CACHE_DIR + self._pip_options = pip_options + if environments.is_verbose(): + click_echo( + crayons.blue("Using pip: {0}".format(" ".join(self.pip_args))), err=True + ) + return self._pip_options + + @property + def session(self): + if self._session is None: + self._session = self.pip_command._build_session(self.pip_options) + return self._session + + @property + def repository(self): + if self._repository is None: + from pipenv.patched.piptools.repositories.pypi import PyPIRepository + self._repository = PyPIRepository( + pip_options=self.pip_options, use_json=False, session=self.session + ) + return self._repository + + @property + def constraints(self): + if self._constraints is None: + from pip_shims.shims import parse_requirements + self._constraints = parse_requirements( + self.constraint_file, finder=self.repository.finder, session=self.session, + options=self.pip_options + ) + return self._constraints + + @property + def parsed_constraints(self): + if self._parsed_constraints is None: + self._parsed_constraints = [c for c in self.constraints] + return self._parsed_constraints + + def get_resolver(self, clear=False, pre=False): + from pipenv.patched.piptools.resolver import Resolver + self._resolver = Resolver( + constraints=self.parsed_constraints, repository=self.repository, + clear_caches=clear, prereleases=pre, + ) + + @property + def resolver(self): + if self._resolver is None: + self.get_resolver(clear=self.clear, pre=self.pre) + return self._resolver + + def resolve(self): + from pipenv.vendor.pip_shims.shims import DistributionNotFound + from pipenv.vendor.requests.exceptions import HTTPError + from pipenv.patched.piptools.exceptions import NoCandidateFound + from pipenv.patched.piptools.cache import CorruptCacheError + from .exceptions import CacheError, ResolutionFailure + try: + results = self.resolver.resolve(max_rounds=environments.PIPENV_MAX_ROUNDS) + except CorruptCacheError as e: + if environments.PIPENV_IS_CI or self.clear: + if self._retry_attempts < 3: + self.get_resolver(clear=True, pre=self.pre) + self._retry_attempts += 1 + self.resolve() + else: + raise CacheError(e.path) + except (NoCandidateFound, DistributionNotFound, HTTPError) as e: + raise ResolutionFailure(message=str(e)) + else: + self.results = results + self.resolved_tree.update(results) + return self.resolved_tree + + def resolve_hashes(self): + def _should_include_hash(ireq): + from pipenv.vendor.vistir.compat import Path, to_native_string + from pipenv.vendor.vistir.path import url_to_path + + # We can only hash artifacts. + try: + if not ireq.link.is_artifact: + return False + except AttributeError: + return False + + # But we don't want normal pypi artifcats since the normal resolver + # handles those + if is_pypi_url(ireq.link.url): + return False + + # We also don't want to try to hash directories as this will fail + # as these are editable deps and are not hashable. + if (ireq.link.scheme == "file" and + Path(to_native_string(url_to_path(ireq.link.url))).is_dir()): + return False + return True + + if self.results is not None: + resolved_hashes = self.resolver.resolve_hashes(self.results) + for ireq, ireq_hashes in resolved_hashes.items(): + if ireq not in self.hashes: + if _should_include_hash(ireq): + self.hashes[ireq] = [ + self.resolver.repository._hash_cache.get_hash(ireq.link) + ] + else: + self.hashes[ireq] = ireq_hashes + return self.hashes + + def actually_resolve_deps( deps, index_lookup, @@ -218,122 +458,102 @@ def actually_resolve_deps( pre, req_dir=None, ): - from .patched.notpip._internal import basecommand - from .patched.notpip._internal.req import parse_requirements - from .patched.notpip._internal.exceptions import DistributionNotFound - from .patched.notpip._vendor.requests.exceptions import HTTPError - from pipenv.patched.piptools.resolver import Resolver - from pipenv.patched.piptools.repositories.pypi import PyPIRepository - from pipenv.patched.piptools.scripts.compile import get_pip_command - from pipenv.patched.piptools import logging as piptools_logging - from pipenv.patched.piptools.exceptions import NoCandidateFound - from .vendor.requirementslib.models.requirements import Requirement - from ._compat import TemporaryDirectory, NamedTemporaryFile + from pipenv.vendor.vistir.path import create_tracked_tempdir - class PipCommand(basecommand.Command): - """Needed for pip-tools.""" - - name = "PipCommand" - - constraints = [] - cleanup_req_dir = False if not req_dir: - req_dir = TemporaryDirectory(suffix="-requirements", prefix="pipenv-") - cleanup_req_dir = True - for dep in deps: - if not dep: - continue - url = None - indexes, trusted_hosts, remainder = parse_indexes(dep) - if indexes: - url = indexes[0] - dep = " ".join(remainder) - req = Requirement.from_line(dep) - - # extra_constraints = [] - - if url: - index_lookup[req.name] = project.get_source(url=url).get("name") - # strip the marker and re-add it later after resolution - # but we will need a fallback in case resolution fails - # eg pypiwin32 - if req.markers: - markers_lookup[req.name] = req.markers.replace('"', "'") - constraints.append(req.constraint_line) - - pip_command = get_pip_command() - constraints_file = None - pip_args = [] - if sources: - pip_args = prepare_pip_source_args(sources, pip_args) - if environments.is_verbose(): - print("Using pip: {0}".format(" ".join(pip_args))) - with NamedTemporaryFile( - mode="w", - prefix="pipenv-", - suffix="-constraints.txt", - dir=req_dir.name, - delete=False, - ) as f: - if sources: - requirementstxt_sources = " ".join(pip_args) if pip_args else "" - requirementstxt_sources = requirementstxt_sources.replace(" --", "\n--") - f.write(u"{0}\n".format(requirementstxt_sources)) - f.write(u"\n".join([_constraint for _constraint in constraints])) - constraints_file = f.name - pip_options, _ = pip_command.parser.parse_args(pip_args) - pip_options.cache_dir = environments.PIPENV_CACHE_DIR - session = pip_command._build_session(pip_options) - pypi = PyPIRepository(pip_options=pip_options, use_json=False, session=session) - constraints = parse_requirements( - constraints_file, finder=pypi.finder, session=pypi.session, options=pip_options + req_dir = create_tracked_tempdir(suffix="-requirements", prefix="pipenv-") + constraints = get_resolver_metadata( + deps, index_lookup, markers_lookup, project, sources, ) - constraints = [c for c in constraints] - if environments.is_verbose(): - logging.log.verbose = True - piptools_logging.log.verbose = True - resolved_tree = set() - resolver = Resolver( - constraints=constraints, repository=pypi, clear_caches=clear, prereleases=pre - ) - # pre-resolve instead of iterating to avoid asking pypi for hashes of editable packages - hashes = None - try: - results = resolver.resolve(max_rounds=environments.PIPENV_MAX_ROUNDS) - hashes = resolver.resolve_hashes(results) - resolved_tree.update(results) - except (NoCandidateFound, DistributionNotFound, HTTPError) as e: - click_echo( - "{0}: Your dependencies could not be resolved. You likely have a " - "mismatch in your sub-dependencies.\n " - "First try clearing your dependency cache with {1}, then try the original command again.\n " - "Alternatively, you can use {2} to bypass this mechanism, then run " - "{3} to inspect the situation.\n " - "Hint: try {4} if it is a pre-release dependency." - "".format( - crayons.red("Warning", bold=True), - crayons.red("$ pipenv lock --clear"), - crayons.red("$ pipenv install --skip-lock"), - crayons.red("$ pipenv graph"), - crayons.red("$ pipenv lock --pre"), - ), - err=True, - ) - click_echo(crayons.blue(str(e)), err=True) - if "no version found at all" in str(e): - click_echo( - crayons.blue( - "Please check your version specifier and version number. See PEP440 for more information." - ) - ) - if cleanup_req_dir: - req_dir.cleanup() - raise RuntimeError - if cleanup_req_dir: - req_dir.cleanup() + resolver = Resolver(constraints, req_dir, project, sources, clear=clear, pre=pre) + resolved_tree = resolver.resolve() + hashes = resolver.resolve_hashes() + return (resolved_tree, hashes, markers_lookup, resolver) +@contextlib.contextmanager +def create_spinner(text, nospin=None, spinner_name=None): + import vistir.spin + if not spinner_name: + spinner_name = environments.PIPENV_SPINNER + if nospin is None: + nospin = environments.PIPENV_NOSPIN + with vistir.spin.create_spinner( + spinner_name=spinner_name, + start_text=vistir.compat.fs_str(text), + nospin=nospin + ) as sp: + yield sp + + +def resolve(cmd, sp): + from .vendor import delegator + from .cmdparse import Script + from .vendor.pexpect.exceptions import EOF, TIMEOUT + from .vendor.vistir.compat import to_native_string + EOF.__module__ = "pexpect.exceptions" + from ._compat import decode_output + c = delegator.run(Script.parse(cmd).cmdify(), block=False, env=os.environ.copy()) + _out = decode_output("") + result = None + out = to_native_string("") + while True: + try: + result = c.expect(u"\n", timeout=environments.PIPENV_TIMEOUT) + except (EOF, TIMEOUT): + pass + if result is None: + break + _out = c.subprocess.before + if _out is not None: + _out = decode_output("{0}".format(_out)) + out += _out + sp.text = to_native_string("{0}".format(_out[:100])) + if environments.is_verbose(): + if _out is not None: + sp._hide_cursor() + sp.write(_out.rstrip()) + sp._show_cursor() + c.block() + if c.return_code != 0: + sp.red.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format( + "Locking Failed!" + )) + click_echo(c.out.strip(), err=True) + click_echo(c.err.strip(), err=True) + sys.exit(c.return_code) + return c + + +def get_locked_dep(dep, pipfile_section): + entry = None + cleaner_kwargs = { + "is_top_level": False, + "pipfile_entry": None + } + if isinstance(dep, Mapping) and dep.get("name", ""): + name_options = [dep["name"], pep423_name(dep["name"])] + name = next(iter(k for k in name_options if k in pipfile_section), None) + entry = pipfile_section[name] if name else None + + if entry: + cleaner_kwargs.update({"is_top_level": True, "pipfile_entry": entry}) + lockfile_entry = clean_resolved_dep(dep, **cleaner_kwargs) + return lockfile_entry + + +def prepare_lockfile(results, pipfile, lockfile): + from .vendor.requirementslib.utils import is_vcs + for dep in results: + # Merge in any relevant information from the pipfile entry, including + # markers, normalized names, URL info, etc that we may have dropped during lock + if not is_vcs(dep): + lockfile_entry = get_locked_dep(dep, pipfile) + lockfile.update(lockfile_entry) + return lockfile + + def venv_resolve_deps( deps, which, @@ -342,45 +562,93 @@ def venv_resolve_deps( clear=False, allow_global=False, pypi_mirror=None, + dev=False, + pipfile=None, + lockfile=None ): from .vendor.vistir.misc import fs_str - from .vendor import delegator + from .vendor.vistir.compat import Path, to_native_string, JSONDecodeError + from .vendor.vistir.path import create_tracked_tempdir from . import resolver import json - if not deps: - return [] - resolver = escape_grouped_arguments(resolver.__file__.rstrip("co")) - cmd = "{0} {1} {2} {3} {4}".format( - escape_grouped_arguments(which("python", allow_global=allow_global)), - resolver, - "--pre" if pre else "", - "--clear" if clear else "", - "--system" if allow_global else "", - ) + vcs_deps = [] + vcs_lockfile = {} + results = [] + pipfile_section = "dev_packages" if dev else "packages" + lockfile_section = "develop" if dev else "default" + vcs_section = "vcs_{0}".format(pipfile_section) + vcs_deps = getattr(project, vcs_section, []) + if not deps and not vcs_deps: + return {} + + if not pipfile: + pipfile = getattr(project, pipfile_section, None) + if not lockfile: + lockfile = project._lockfile + req_dir = create_tracked_tempdir(prefix="pipenv", suffix="requirements") + if vcs_deps: + with create_spinner(text=fs_str("Pinning VCS Packages...")) as sp: + vcs_reqs, vcs_lockfile = get_vcs_deps( + project, + which=which, + clear=clear, + pre=pre, + allow_global=allow_global, + dev=dev, + ) + vcs_deps = [req.as_line() for req in vcs_reqs if req.editable] + cmd = [ + which("python", allow_global=allow_global), + Path(resolver.__file__.rstrip("co")).as_posix() + ] + if pre: + cmd.append("--pre") + if clear: + cmd.append("--clear") + if allow_global: + cmd.append("--system") with temp_environ(): os.environ = {fs_str(k): fs_str(val) for k, val in os.environ.items()} os.environ["PIPENV_PACKAGES"] = str("\n".join(deps)) if pypi_mirror: os.environ["PIPENV_PYPI_MIRROR"] = str(pypi_mirror) os.environ["PIPENV_VERBOSITY"] = str(environments.PIPENV_VERBOSITY) - c = delegator.run(cmd, block=True) - try: - assert c.return_code == 0 - except AssertionError: - if environments.is_verbose(): - click_echo(c.out, err=True) - click_echo(c.err, err=True) - else: - click_echo(c.err[(int(len(c.err) / 2) - 1):], err=True) - sys.exit(c.return_code) + os.environ["PIPENV_REQ_DIR"] = fs_str(req_dir) + os.environ["PIP_NO_INPUT"] = fs_str("1") + with create_spinner(text=fs_str("Locking...")) as sp: + c = resolve(cmd, sp) + results = c.out + if vcs_deps: + with temp_environ(): + os.environ["PIPENV_PACKAGES"] = str("\n".join(vcs_deps)) + sp.text = to_native_string("Locking VCS Dependencies...") + vcs_c = resolve(cmd, sp) + vcs_results, vcs_err = vcs_c.out, vcs_c.err + else: + vcs_results, vcs_err = "", "" + sp.green.ok(environments.PIPENV_SPINNER_OK_TEXT.format("Success!")) + outputs = [results, vcs_results] if environments.is_verbose(): - click_echo(c.out.split("RESULTS:")[0], err=True) + for output in outputs: + click_echo(output.split("RESULTS:")[0], err=True) try: - return json.loads(c.out.split("RESULTS:")[1].strip()) + results = json.loads(results.split("RESULTS:")[1].strip()) + if vcs_results: + # For vcs dependencies, treat the initial pass at locking (i.e. checkout) + # as the pipfile entry because it gets us an actual ref to use + vcs_results = json.loads(vcs_results.split("RESULTS:")[1].strip()) + vcs_lockfile = prepare_lockfile(vcs_results, vcs_lockfile.copy(), vcs_lockfile) + else: + vcs_results = [] - except IndexError: + except (IndexError, JSONDecodeError): + for out, err in [(c.out, c.err), (vcs_results, vcs_err)]: + click_echo(out.strip(), err=True) + click_echo(err.strip(), err=True) raise RuntimeError("There was a problem with locking.") + lockfile[lockfile_section] = prepare_lockfile(results, pipfile, lockfile[lockfile_section]) + lockfile[lockfile_section].update(vcs_lockfile) def resolve_deps( @@ -392,23 +660,28 @@ def resolve_deps( clear=False, pre=False, allow_global=False, + req_dir=None ): """Given a list of dependencies, return a resolved list of dependencies, using pip-tools -- and their hashes, using the warehouse API / pip. """ - from .patched.notpip._vendor.requests.exceptions import ConnectionError + from .vendor.requests.exceptions import ConnectionError from .vendor.requirementslib.models.requirements import Requirement - from ._compat import TemporaryDirectory index_lookup = {} markers_lookup = {} python_path = which("python", allow_global=allow_global) + if not os.environ.get("PIP_SRC"): + os.environ["PIP_SRC"] = project.virtualenv_src_location backup_python_path = sys.executable results = [] if not deps: return results # First (proper) attempt: - req_dir = TemporaryDirectory(prefix="pipenv-", suffix="-requirements") + req_dir = req_dir if req_dir else os.environ.get("req_dir", None) + if not req_dir: + from .vendor.vistir.path import create_tracked_tempdir + req_dir = create_tracked_tempdir(prefix="pipenv-", suffix="-requirements") with HackedPythonVersion(python_version=python, python_path=python_path): try: resolved_tree, hashes, markers_lookup, resolver = actually_resolve_deps( @@ -444,7 +717,6 @@ def resolve_deps( req_dir=req_dir, ) except RuntimeError: - req_dir.cleanup() sys.exit(1) for result in resolved_tree: if not result.editable: @@ -478,7 +750,7 @@ def resolve_deps( click_echo( "{0}: Error generating hash for {1}".format( crayons.red("Warning", bold=True), name - ) + ), err=True ) # # Collect un-collectable hashes (should work with devpi). # try: @@ -503,17 +775,9 @@ def resolve_deps( entry.update({"markers": markers_lookup.get(result.name)}) entry = translate_markers(entry) results.append(entry) - req_dir.cleanup() return results -def multi_split(s, split): - """Splits on multiple given separators.""" - for r in split: - s = s.replace(r, "|") - return [i for i in s.split("|") if len(i) > 0] - - def is_star(val): return isinstance(val, six.string_types) and val == "*" @@ -526,7 +790,6 @@ def is_pinned(val): def convert_deps_to_pip(deps, project=None, r=True, include_index=True): """"Converts a Pipfile-formatted dependency to a pip-formatted one.""" - from ._compat import NamedTemporaryFile from .vendor.requirementslib.models.requirements import Requirement dependencies = [] @@ -541,7 +804,8 @@ def convert_deps_to_pip(deps, project=None, r=True, include_index=True): return dependencies # Write requirements.txt to tmp directory. - f = NamedTemporaryFile(suffix="-requirements.txt", delete=False) + from .vendor.vistir.path import create_tracked_tempfile + f = create_tracked_tempfile(suffix="-requirements.txt", delete=False) f.write("\n".join(dependencies).encode("utf-8")) f.close() return f.name @@ -584,22 +848,6 @@ def is_required_version(version, specified_version): return True -def strip_ssh_from_git_uri(uri): - """Return git+ssh:// formatted URI to git+git@ format""" - if isinstance(uri, six.string_types): - uri = uri.replace("git+ssh://", "git+") - return uri - - -def clean_git_uri(uri): - """Cleans VCS uris from pip format""" - if isinstance(uri, six.string_types): - # Add scheme for parsing purposes, this is also what pip does - if uri.startswith("git+") and "://" not in uri: - uri = uri.replace("git+", "git+ssh://") - return uri - - def is_editable(pipfile_entry): if hasattr(pipfile_entry, "get"): return pipfile_entry.get("editable", False) and any( @@ -610,9 +858,8 @@ def is_editable(pipfile_entry): def is_installable_file(path): """Determine if a path can potentially be installed""" - from .patched.notpip._internal.utils.misc import is_installable_dir + from .vendor.pip_shims.shims import is_installable_dir, is_archive_file from .patched.notpip._internal.utils.packaging import specifiers - from .patched.notpip._internal.download import is_archive_file from ._compat import Path if hasattr(path, "keys") and any( @@ -664,7 +911,7 @@ def is_file(package): def pep440_version(version): """Normalize version to PEP 440 standards""" - from .patched.notpip._internal.index import parse_version + from .vendor.pip_shims.shims import parse_version # Use pip built-in version parser. return str(parse_version(version)) @@ -743,94 +990,6 @@ def split_section(input_file, section_suffix, test_function): return input_file -def split_file(file_dict): - """Split VCS and editable dependencies out from file.""" - from .vendor.requirementslib.utils import is_vcs - sections = { - "vcs": is_vcs, - "editable": lambda x: hasattr(x, "keys") and x.get("editable"), - } - for k, func in sections.items(): - file_dict = split_section(file_dict, k, func) - return file_dict - - -def merge_deps( - file_dict, - project, - dev=False, - requirements=False, - ignore_hashes=False, - blocking=False, - only=False, -): - """ - Given a file_dict, merges dependencies and converts them to pip dependency lists. - :param dict file_dict: The result of calling :func:`pipenv.utils.split_file` - :param :class:`pipenv.project.Project` project: Pipenv project - :param bool dev=False: Flag indicating whether dev dependencies are to be installed - :param bool requirements=False: Flag indicating whether to use a requirements file - :param bool ignore_hashes=False: - :param bool blocking=False: - :param bool only=False: - :return: Pip-converted 3-tuples of [deps, requirements_deps] - """ - deps = [] - requirements_deps = [] - for section in list(file_dict.keys()): - # Turn develop-vcs into ['develop', 'vcs'] - section_name, suffix = ( - section.rsplit("-", 1) - if "-" in section and not section == "dev-packages" - else (section, None) - ) - if not file_dict[section] or section_name not in ( - "dev-packages", - "packages", - "default", - "develop", - ): - continue - - is_dev = section_name in ("dev-packages", "develop") - if is_dev and not dev: - continue - - if ignore_hashes: - for k, v in file_dict[section]: - if "hash" in v: - del v["hash"] - # Block and ignore hashes for all suffixed sections (vcs/editable) - no_hashes = True if suffix else ignore_hashes - block = True if suffix else blocking - include_index = True if not suffix else False - converted = convert_deps_to_pip( - file_dict[section], project, r=False, include_index=include_index - ) - deps.extend((d, no_hashes, block) for d in converted) - if dev and is_dev and requirements: - requirements_deps.extend((d, no_hashes, block) for d in converted) - return deps, requirements_deps - - -def recase_file(file_dict): - """Recase file before writing to output.""" - if "packages" in file_dict or "dev-packages" in file_dict: - sections = ("packages", "dev-packages") - elif "default" in file_dict or "develop" in file_dict: - sections = ("default", "develop") - for section in sections: - file_section = file_dict.get(section, {}) - # Try to properly case each key if we can. - for key in list(file_section.keys()): - try: - cased_key = proper_case(key) - except IOError: - cased_key = key - file_section[cased_key] = file_section.pop(key) - return file_dict - - def get_windows_path(*args): """Sanitize a path for windows environments @@ -863,6 +1022,29 @@ def path_to_url(path): return Path(normalize_drive(os.path.abspath(path))).as_uri() +def normalize_path(path): + return os.path.expandvars(os.path.expanduser( + os.path.normcase(os.path.normpath(os.path.abspath(str(path)))) + )) + + +def get_url_name(url): + if not isinstance(url, six.string_types): + return + return urllib3_util.parse_url(url).host + + +def get_canonical_names(packages): + """Canonicalize a list of packages and return a set of canonical names""" + from .vendor.packaging.utils import canonicalize_name + + if not isinstance(packages, Sequence): + if not isinstance(packages, six.string_types): + return packages + packages = [packages,] + return set([canonicalize_name(pkg) for pkg in packages if pkg]) + + def walk_up(bottom): """Mimic os.walk, but walk 'up' instead of down the directory tree. From: https://gist.github.com/zdavkeos/1098474 @@ -1054,54 +1236,6 @@ def escape_cmd(cmd): return cmd -@contextmanager -def atomic_open_for_write(target, binary=False, newline=None, encoding=None): - """Atomically open `target` for writing. - - This is based on Lektor's `atomic_open()` utility, but simplified a lot - to handle only writing, and skip many multi-process/thread edge cases - handled by Werkzeug. - - How this works: - - * Create a temp file (in the same directory of the actual target), and - yield for surrounding code to write to it. - * If some thing goes wrong, try to remove the temp file. The actual target - is not touched whatsoever. - * If everything goes well, close the temp file, and replace the actual - target with this new file. - """ - from ._compat import NamedTemporaryFile - - mode = "w+b" if binary else "w" - f = NamedTemporaryFile( - dir=os.path.dirname(target), - prefix=".__atomic-write", - mode=mode, - encoding=encoding, - newline=newline, - delete=False, - ) - # set permissions to 0644 - os.chmod(f.name, stat.S_IWUSR | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) - try: - yield f - except BaseException: - f.close() - try: - os.remove(f.name) - except OSError: - pass - raise - else: - f.close() - try: - os.remove(target) # This is needed on Windows. - except OSError: - pass - os.rename(f.name, target) # No os.replace() on Python 2. - - def safe_expandvars(value): """Call os.path.expandvars if value is a string, otherwise do nothing. """ @@ -1110,16 +1244,8 @@ def safe_expandvars(value): return value -def extract_uri_from_vcs_dep(dep): - valid_keys = VCS_LIST + ("uri", "file") - if hasattr(dep, "keys"): - return first(dep[k] for k in valid_keys if k in dep) or None - return None - - def get_vcs_deps( project, - pip_freeze=None, which=None, clear=False, pre=False, @@ -1127,8 +1253,6 @@ def get_vcs_deps( dev=False, pypi_mirror=None, ): - from ._compat import TemporaryDirectory, Path - import atexit from .vendor.requirementslib.models.requirements import Requirement section = "vcs_dev_packages" if dev else "vcs_packages" @@ -1138,24 +1262,19 @@ def get_vcs_deps( packages = getattr(project, section) except AttributeError: return [], [] - if os.environ.get("PIP_SRC"): - src_dir = Path( - os.environ.get("PIP_SRC", os.path.join(project.virtualenv_location, "src")) - ) - src_dir.mkdir(mode=0o775, exist_ok=True) - else: - src_dir = TemporaryDirectory(prefix="pipenv-lock-dir") - atexit.register(src_dir.cleanup) for pkg_name, pkg_pipfile in packages.items(): requirement = Requirement.from_pipfile(pkg_name, pkg_pipfile) name = requirement.normalized_name commit_hash = None if requirement.is_vcs: - with locked_repository(requirement) as repo: - commit_hash = repo.get_commit_hash() - lockfile[name] = requirement.pipfile_entry[1] - lockfile[name]['ref'] = commit_hash - reqs.append(requirement) + try: + with locked_repository(requirement) as repo: + commit_hash = repo.get_commit_hash() + lockfile[name] = requirement.pipfile_entry[1] + lockfile[name]['ref'] = commit_hash + reqs.append(requirement) + except OSError: + continue return reqs, lockfile @@ -1172,7 +1291,7 @@ def translate_markers(pipfile_entry): """ if not isinstance(pipfile_entry, Mapping): raise TypeError("Entry is not a pipfile formatted mapping.") - from notpip._vendor.distlib.markers import DEFAULT_CONTEXT as marker_context + from .vendor.distlib.markers import DEFAULT_CONTEXT as marker_context from .vendor.packaging.markers import Marker from .vendor.vistir.misc import dedup @@ -1191,8 +1310,10 @@ def translate_markers(pipfile_entry): marker_set.add(str(Marker("{0}{1}".format(m, entry)))) new_pipfile.pop(m) if marker_set: - new_pipfile["markers"] = str(Marker(" or ".join(["{0}".format(s) if " and " in s else s - for s in sorted(dedup(marker_set))]))).replace('"', "'") + new_pipfile["markers"] = str(Marker(" or ".join( + "{0}".format(s) if " and " in s else s + for s in sorted(dedup(marker_set)) + ))).replace('"', "'") return new_pipfile @@ -1200,9 +1321,6 @@ def clean_resolved_dep(dep, is_top_level=False, pipfile_entry=None): name = pep423_name(dep["name"]) # We use this to determine if there are any markers on top level packages # So we can make sure those win out during resolution if the packages reoccur - dep_keys = ( - [k for k in getattr(pipfile_entry, "keys", list)()] if is_top_level else [] - ) lockfile = {"version": "=={0}".format(dep["version"])} for key in ["hashes", "index", "extras"]: if key in dep: @@ -1280,12 +1398,11 @@ def is_virtual_environment(path): @contextmanager def locked_repository(requirement): from .vendor.vistir.path import create_tracked_tempdir - from .vendor.vistir.misc import fs_str - src_dir = create_tracked_tempdir(prefix="pipenv-src") if not requirement.is_vcs: return original_base = os.environ.pop("PIP_SHIMS_BASE_MODULE", None) os.environ["PIP_SHIMS_BASE_MODULE"] = fs_str("pipenv.patched.notpip") + src_dir = create_tracked_tempdir(prefix="pipenv-", suffix="-src") try: with requirement.req.locked_vcs_repo(src_dir=src_dir) as repo: yield repo @@ -1318,10 +1435,14 @@ def looks_like_dir(path): def parse_indexes(line): from argparse import ArgumentParser parser = ArgumentParser("indexes") - parser.add_argument("--index", "-i", "--index-url", metavar="index_url", - action="store", nargs="?",) - parser.add_argument("--extra-index-url", "--extra-index", metavar="extra_indexes", - action="append") + parser.add_argument( + "--index", "-i", "--index-url", + metavar="index_url", action="store", nargs="?", + ) + parser.add_argument( + "--extra-index-url", "--extra-index", + metavar="extra_indexes",action="append", + ) parser.add_argument("--trusted-host", metavar="trusted_hosts", action="append") args, remainder = parser.parse_known_args(line.split()) index = [] if not args.index else [args.index,] @@ -1329,3 +1450,17 @@ def parse_indexes(line): indexes = index + extra_indexes trusted_hosts = args.trusted_host if args.trusted_host else [] return indexes, trusted_hosts, remainder + + +@contextmanager +def sys_version(version_tuple): + """ + Set a temporary sys.version_info tuple + + :param version_tuple: a fake sys.version_info tuple + """ + + old_version = sys.version_info + sys.version_info = version_tuple + yield + sys.version_info = old_version diff --git a/pipenv/vendor/backports/__init__.py b/pipenv/vendor/backports/__init__.py index 3cd30963..0c64b4c1 100644 --- a/pipenv/vendor/backports/__init__.py +++ b/pipenv/vendor/backports/__init__.py @@ -1,7 +1,5 @@ -# See https://pypi.python.org/pypi/backports - -from pkgutil import extend_path -__path__ = extend_path(__path__, __name__) +__path__ = __import__('pkgutil').extend_path(__path__, __name__) from . import weakref from . import enum from . import shutil_get_terminal_size +from . import functools_lru_cache diff --git a/pipenv/vendor/backports/functools_lru_cache.LICENSE b/pipenv/vendor/backports/functools_lru_cache.LICENSE new file mode 100644 index 00000000..5e795a61 --- /dev/null +++ b/pipenv/vendor/backports/functools_lru_cache.LICENSE @@ -0,0 +1,7 @@ +Copyright Jason R. Coombs + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/pipenv/vendor/backports/functools_lru_cache.py b/pipenv/vendor/backports/functools_lru_cache.py new file mode 100644 index 00000000..707c6c76 --- /dev/null +++ b/pipenv/vendor/backports/functools_lru_cache.py @@ -0,0 +1,184 @@ +from __future__ import absolute_import + +import functools +from collections import namedtuple +from threading import RLock + +_CacheInfo = namedtuple("CacheInfo", ["hits", "misses", "maxsize", "currsize"]) + + +@functools.wraps(functools.update_wrapper) +def update_wrapper(wrapper, + wrapped, + assigned = functools.WRAPPER_ASSIGNMENTS, + updated = functools.WRAPPER_UPDATES): + """ + Patch two bugs in functools.update_wrapper. + """ + # workaround for http://bugs.python.org/issue3445 + assigned = tuple(attr for attr in assigned if hasattr(wrapped, attr)) + wrapper = functools.update_wrapper(wrapper, wrapped, assigned, updated) + # workaround for https://bugs.python.org/issue17482 + wrapper.__wrapped__ = wrapped + return wrapper + + +class _HashedSeq(list): + __slots__ = 'hashvalue' + + def __init__(self, tup, hash=hash): + self[:] = tup + self.hashvalue = hash(tup) + + def __hash__(self): + return self.hashvalue + + +def _make_key(args, kwds, typed, + kwd_mark=(object(),), + fasttypes=set([int, str, frozenset, type(None)]), + sorted=sorted, tuple=tuple, type=type, len=len): + 'Make a cache key from optionally typed positional and keyword arguments' + key = args + if kwds: + sorted_items = sorted(kwds.items()) + key += kwd_mark + for item in sorted_items: + key += item + if typed: + key += tuple(type(v) for v in args) + if kwds: + key += tuple(type(v) for k, v in sorted_items) + elif len(key) == 1 and type(key[0]) in fasttypes: + return key[0] + return _HashedSeq(key) + + +def lru_cache(maxsize=100, typed=False): + """Least-recently-used cache decorator. + + If *maxsize* is set to None, the LRU features are disabled and the cache + can grow without bound. + + If *typed* is True, arguments of different types will be cached separately. + For example, f(3.0) and f(3) will be treated as distinct calls with + distinct results. + + Arguments to the cached function must be hashable. + + View the cache statistics named tuple (hits, misses, maxsize, currsize) with + f.cache_info(). Clear the cache and statistics with f.cache_clear(). + Access the underlying function with f.__wrapped__. + + See: http://en.wikipedia.org/wiki/Cache_algorithms#Least_Recently_Used + + """ + + # Users should only access the lru_cache through its public API: + # cache_info, cache_clear, and f.__wrapped__ + # The internals of the lru_cache are encapsulated for thread safety and + # to allow the implementation to change (including a possible C version). + + def decorating_function(user_function): + + cache = dict() + stats = [0, 0] # make statistics updateable non-locally + HITS, MISSES = 0, 1 # names for the stats fields + make_key = _make_key + cache_get = cache.get # bound method to lookup key or return None + _len = len # localize the global len() function + lock = RLock() # because linkedlist updates aren't threadsafe + root = [] # root of the circular doubly linked list + root[:] = [root, root, None, None] # initialize by pointing to self + nonlocal_root = [root] # make updateable non-locally + PREV, NEXT, KEY, RESULT = 0, 1, 2, 3 # names for the link fields + + if maxsize == 0: + + def wrapper(*args, **kwds): + # no caching, just do a statistics update after a successful call + result = user_function(*args, **kwds) + stats[MISSES] += 1 + return result + + elif maxsize is None: + + def wrapper(*args, **kwds): + # simple caching without ordering or size limit + key = make_key(args, kwds, typed) + result = cache_get(key, root) # root used here as a unique not-found sentinel + if result is not root: + stats[HITS] += 1 + return result + result = user_function(*args, **kwds) + cache[key] = result + stats[MISSES] += 1 + return result + + else: + + def wrapper(*args, **kwds): + # size limited caching that tracks accesses by recency + key = make_key(args, kwds, typed) if kwds or typed else args + with lock: + link = cache_get(key) + if link is not None: + # record recent use of the key by moving it to the front of the list + root, = nonlocal_root + link_prev, link_next, key, result = link + link_prev[NEXT] = link_next + link_next[PREV] = link_prev + last = root[PREV] + last[NEXT] = root[PREV] = link + link[PREV] = last + link[NEXT] = root + stats[HITS] += 1 + return result + result = user_function(*args, **kwds) + with lock: + root, = nonlocal_root + if key in cache: + # getting here means that this same key was added to the + # cache while the lock was released. since the link + # update is already done, we need only return the + # computed result and update the count of misses. + pass + elif _len(cache) >= maxsize: + # use the old root to store the new key and result + oldroot = root + oldroot[KEY] = key + oldroot[RESULT] = result + # empty the oldest link and make it the new root + root = nonlocal_root[0] = oldroot[NEXT] + oldkey = root[KEY] + root[KEY] = root[RESULT] = None + # now update the cache dictionary for the new links + del cache[oldkey] + cache[key] = oldroot + else: + # put result in a new link at the front of the list + last = root[PREV] + link = [last, root, key, result] + last[NEXT] = root[PREV] = cache[key] = link + stats[MISSES] += 1 + return result + + def cache_info(): + """Report cache statistics""" + with lock: + return _CacheInfo(stats[HITS], stats[MISSES], maxsize, len(cache)) + + def cache_clear(): + """Clear the cache and cache statistics""" + with lock: + cache.clear() + root = nonlocal_root[0] + root[:] = [root, root, None, None] + stats[:] = [0, 0] + + wrapper.__wrapped__ = user_function + wrapper.cache_info = cache_info + wrapper.cache_clear = cache_clear + return update_wrapper(wrapper, user_function) + + return decorating_function diff --git a/pipenv/vendor/backports/shutil_get_terminal_size/__init__.py b/pipenv/vendor/backports/shutil_get_terminal_size/__init__.py index fa12816e..cfcbdf66 100644 --- a/pipenv/vendor/backports/shutil_get_terminal_size/__init__.py +++ b/pipenv/vendor/backports/shutil_get_terminal_size/__init__.py @@ -1,9 +1,11 @@ -__title__ = "shutil_backports" -__version__ = "0.1.0" +"""A backport of the get_terminal_size function from Python 3.3's shutil.""" + +__title__ = "backports.shutil_get_terminal_size" +__version__ = "1.0.0" __license__ = "MIT" __author__ = "Christopher Rosell" __copyright__ = "Copyright 2014 Christopher Rosell" __all__ = ["get_terminal_size"] -from .get_terminal_size import * +from .get_terminal_size import get_terminal_size diff --git a/pipenv/vendor/backports/shutil_get_terminal_size/get_terminal_size.py b/pipenv/vendor/backports/shutil_get_terminal_size/get_terminal_size.py index f1336e58..28c96da8 100644 --- a/pipenv/vendor/backports/shutil_get_terminal_size/get_terminal_size.py +++ b/pipenv/vendor/backports/shutil_get_terminal_size/get_terminal_size.py @@ -39,7 +39,7 @@ try: except Exception: pass - return columns, lines + return terminal_size(columns, lines) except ImportError: import fcntl @@ -52,7 +52,7 @@ except ImportError: except Exception: columns = lines = 0 - return columns, lines + return terminal_size(columns, lines) def get_terminal_size(fallback=(80, 24)): @@ -74,7 +74,7 @@ def get_terminal_size(fallback=(80, 24)): The value returned is a named tuple of type os.terminal_size. """ - # Attempt to use the environment first + # Try the environment first try: columns = int(os.environ["COLUMNS"]) except (KeyError, ValueError): @@ -88,13 +88,14 @@ def get_terminal_size(fallback=(80, 24)): # Only query if necessary if columns <= 0 or lines <= 0: try: - columns, lines = _get_terminal_size(sys.__stdout__.fileno()) + size = _get_terminal_size(sys.__stdout__.fileno()) except (NameError, OSError): - pass + size = terminal_size(*fallback) - # Use fallback as last resort - if columns <= 0 and lines <= 0: - columns, lines = fallback + if columns <= 0: + columns = size.columns + if lines <= 0: + lines = size.lines return terminal_size(columns, lines) diff --git a/pipenv/vendor/certifi/__init__.py b/pipenv/vendor/certifi/__init__.py index aa329fbb..50f2e130 100644 --- a/pipenv/vendor/certifi/__init__.py +++ b/pipenv/vendor/certifi/__init__.py @@ -1,3 +1,3 @@ from .core import where, old_where -__version__ = "2018.08.24" +__version__ = "2018.10.15" diff --git a/pipenv/vendor/certifi/cacert.pem b/pipenv/vendor/certifi/cacert.pem index 85de024e..e75d85b3 100644 --- a/pipenv/vendor/certifi/cacert.pem +++ b/pipenv/vendor/certifi/cacert.pem @@ -326,36 +326,6 @@ OCiNUW7dFGdTbHFcJoRNdVq2fmBWqU2t+5sel/MN2dKXVHfaPRK34B7vCAas+YWH QMAJKOSLakhT2+zNVVXxxvjpoixMptEmX36vWkzaH6byHCx+rgIW0lbQL1dTR+iS -----END CERTIFICATE----- -# Issuer: CN=Visa eCommerce Root O=VISA OU=Visa International Service Association -# Subject: CN=Visa eCommerce Root O=VISA OU=Visa International Service Association -# Label: "Visa eCommerce Root" -# Serial: 25952180776285836048024890241505565794 -# MD5 Fingerprint: fc:11:b8:d8:08:93:30:00:6d:23:f9:7e:eb:52:1e:02 -# SHA1 Fingerprint: 70:17:9b:86:8c:00:a4:fa:60:91:52:22:3f:9f:3e:32:bd:e0:05:62 -# SHA256 Fingerprint: 69:fa:c9:bd:55:fb:0a:c7:8d:53:bb:ee:5c:f1:d5:97:98:9f:d0:aa:ab:20:a2:51:51:bd:f1:73:3e:e7:d1:22 ------BEGIN CERTIFICATE----- -MIIDojCCAoqgAwIBAgIQE4Y1TR0/BvLB+WUF1ZAcYjANBgkqhkiG9w0BAQUFADBr -MQswCQYDVQQGEwJVUzENMAsGA1UEChMEVklTQTEvMC0GA1UECxMmVmlzYSBJbnRl -cm5hdGlvbmFsIFNlcnZpY2UgQXNzb2NpYXRpb24xHDAaBgNVBAMTE1Zpc2EgZUNv -bW1lcmNlIFJvb3QwHhcNMDIwNjI2MDIxODM2WhcNMjIwNjI0MDAxNjEyWjBrMQsw -CQYDVQQGEwJVUzENMAsGA1UEChMEVklTQTEvMC0GA1UECxMmVmlzYSBJbnRlcm5h -dGlvbmFsIFNlcnZpY2UgQXNzb2NpYXRpb24xHDAaBgNVBAMTE1Zpc2EgZUNvbW1l -cmNlIFJvb3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCvV95WHm6h -2mCxlCfLF9sHP4CFT8icttD0b0/Pmdjh28JIXDqsOTPHH2qLJj0rNfVIsZHBAk4E -lpF7sDPwsRROEW+1QK8bRaVK7362rPKgH1g/EkZgPI2h4H3PVz4zHvtH8aoVlwdV -ZqW1LS7YgFmypw23RuwhY/81q6UCzyr0TP579ZRdhE2o8mCP2w4lPJ9zcc+U30rq -299yOIzzlr3xF7zSujtFWsan9sYXiwGd/BmoKoMWuDpI/k4+oKsGGelT84ATB+0t -vz8KPFUgOSwsAGl0lUq8ILKpeeUYiZGo3BxN77t+Nwtd/jmliFKMAGzsGHxBvfaL -dXe6YJ2E5/4tAgMBAAGjQjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQD -AgEGMB0GA1UdDgQWBBQVOIMPPyw/cDMezUb+B4wg4NfDtzANBgkqhkiG9w0BAQUF -AAOCAQEAX/FBfXxcCLkr4NWSR/pnXKUTwwMhmytMiUbPWU3J/qVAtmPN3XEolWcR -zCSs00Rsca4BIGsDoo8Ytyk6feUWYFN4PMCvFYP3j1IzJL1kk5fui/fbGKhtcbP3 -LBfQdCVp9/5rPJS+TUtBjE7ic9DjkCJzQ83z7+pzzkWKsKZJ/0x9nXGIxHYdkFsd -7v3M9+79YKWxehZx0RbQfBI8bGmX265fOZpwLwU8GUYEmSA20GBuYQa7FkKMcPcw -++DbZqMAAb3mLNqRX6BGi01qnD093QVG/na/oAo85ADmJ7f/hC3euiInlhBx6yLt -398znM/jra6O1I7mT1GvFpLgXPYHDw== ------END CERTIFICATE----- - # Issuer: CN=AAA Certificate Services O=Comodo CA Limited # Subject: CN=AAA Certificate Services O=Comodo CA Limited # Label: "Comodo AAA Services root" diff --git a/pipenv/vendor/click_completion/__init__.py b/pipenv/vendor/click_completion/__init__.py index b849ae23..e30cc0e3 100644 --- a/pipenv/vendor/click_completion/__init__.py +++ b/pipenv/vendor/click_completion/__init__.py @@ -19,7 +19,7 @@ from click_completion.core import completion_configuration, get_code, install, s from click_completion.lib import get_auto_shell from click_completion.patch import patch as _patch -__version__ = '0.4.1' +__version__ = '0.5.0' _initialized = False diff --git a/pipenv/vendor/click_completion/lib.py b/pipenv/vendor/click_completion/lib.py index cd53bc03..167ecfd8 100644 --- a/pipenv/vendor/click_completion/lib.py +++ b/pipenv/vendor/click_completion/lib.py @@ -59,7 +59,7 @@ def double_quote(s): return '"' + s.replace('"', '"\'"\'"') + '"' -def resolve_ctx(cli, prog_name, args): +def resolve_ctx(cli, prog_name, args, resilient_parsing=True): """ Parameters @@ -76,13 +76,18 @@ def resolve_ctx(cli, prog_name, args): click.core.Context A new context corresponding to the current command """ - ctx = cli.make_context(prog_name, list(args), resilient_parsing=True) + ctx = cli.make_context(prog_name, list(args), resilient_parsing=resilient_parsing) while ctx.args + ctx.protected_args and isinstance(ctx.command, MultiCommand): a = ctx.protected_args + ctx.args cmd = ctx.command.get_command(ctx, a[0]) if cmd is None: return None - ctx = cmd.make_context(a[0], a[1:], parent=ctx, resilient_parsing=True) + if hasattr(cmd, "no_args_is_help"): + no_args_is_help = cmd.no_args_is_help + cmd.no_args_is_help = False + ctx = cmd.make_context(a[0], a[1:], parent=ctx, resilient_parsing=resilient_parsing) + if hasattr(cmd, "no_args_is_help"): + cmd.no_args_is_help = no_args_is_help return ctx diff --git a/pipenv/vendor/cursor/LICENSE b/pipenv/vendor/cursor/LICENSE new file mode 100644 index 00000000..00023c80 --- /dev/null +++ b/pipenv/vendor/cursor/LICENSE @@ -0,0 +1,5 @@ +This work is licensed under the Creative Commons +Attribution-ShareAlike 2.5 International License. To view a copy of +this license, visit http://creativecommons.org/licenses/by-sa/2.5/ or +send a letter to Creative Commons, PO Box 1866, Mountain View, +CA 94042, USA. diff --git a/pipenv/vendor/cursor/__init__.py b/pipenv/vendor/cursor/__init__.py new file mode 100644 index 00000000..76a4f671 --- /dev/null +++ b/pipenv/vendor/cursor/__init__.py @@ -0,0 +1,4 @@ +from .cursor import hide, show, HiddenCursor + +__all__ = ["hide", "show", "HiddenCursor"] + diff --git a/pipenv/vendor/cursor/cursor.py b/pipenv/vendor/cursor/cursor.py new file mode 100644 index 00000000..e4407c02 --- /dev/null +++ b/pipenv/vendor/cursor/cursor.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python2 +# -*- coding: utf-8 -*- + +## Author: James Spencer: http://stackoverflow.com/users/1375885/james-spencer +## Packager: Gijs TImmers: https://github.com/GijsTimmers + +## Based on James Spencer's answer on StackOverflow: +## http://stackoverflow.com/questions/5174810/how-to-turn-off-blinking-cursor-in-command-window + +## Licence: CC-BY-SA-2.5 +## http://creativecommons.org/licenses/by-sa/2.5/ + +## This work is licensed under the Creative Commons +## Attribution-ShareAlike 2.5 International License. To view a copy of +## this license, visit http://creativecommons.org/licenses/by-sa/2.5/ or +## send a letter to Creative Commons, PO Box 1866, Mountain View, +## CA 94042, USA. + +import sys +import os + +if os.name == 'nt': + import ctypes + + class _CursorInfo(ctypes.Structure): + _fields_ = [("size", ctypes.c_int), + ("visible", ctypes.c_byte)] + +def hide(stream=sys.stdout): + if os.name == 'nt': + ci = _CursorInfo() + handle = ctypes.windll.kernel32.GetStdHandle(-11) + ctypes.windll.kernel32.GetConsoleCursorInfo(handle, ctypes.byref(ci)) + ci.visible = False + ctypes.windll.kernel32.SetConsoleCursorInfo(handle, ctypes.byref(ci)) + elif os.name == 'posix': + stream.write("\033[?25l") + stream.flush() + +def show(stream=sys.stdout): + if os.name == 'nt': + ci = _CursorInfo() + handle = ctypes.windll.kernel32.GetStdHandle(-11) + ctypes.windll.kernel32.GetConsoleCursorInfo(handle, ctypes.byref(ci)) + ci.visible = True + ctypes.windll.kernel32.SetConsoleCursorInfo(handle, ctypes.byref(ci)) + elif os.name == 'posix': + stream.write("\033[?25h") + stream.flush() + +class HiddenCursor(object): + def __init__(self, stream=sys.stdout): + self._stream = stream + def __enter__(self): + hide(stream=self._stream) + def __exit__(self, type, value, traceback): + show(stream=self._stream) \ No newline at end of file diff --git a/pipenv/vendor/delegator.py b/pipenv/vendor/delegator.py index d15aeb97..367f52b0 100644 --- a/pipenv/vendor/delegator.py +++ b/pipenv/vendor/delegator.py @@ -7,6 +7,8 @@ import locale import errno from pexpect.popen_spawn import PopenSpawn +import pexpect +pexpect.EOF.__module__ = "pexpect.exceptions" # Include `unicode` in STR_TYPES for Python 2.X try: @@ -110,8 +112,11 @@ class Command(object): if self.subprocess.before: result += self.subprocess.before - if self.subprocess.after: - result += self.subprocess.after + if self.subprocess.after and self.subprocess.after not in (pexpect.EOF, pexpect.TIMEOUT): + try: + result += self.subprocess.after + except (pexpect.EOF, pexpect.TIMEOUT): + pass result += self.subprocess.read() return result @@ -178,6 +183,7 @@ class Command(object): # Use subprocess. if self.blocking: popen_kwargs = self._default_popen_kwargs.copy() + del popen_kwargs["stdin"] popen_kwargs["universal_newlines"] = not binary if cwd: popen_kwargs["cwd"] = cwd @@ -205,7 +211,10 @@ class Command(object): if self.blocking: raise RuntimeError("expect can only be used on non-blocking commands.") - self.subprocess.expect(pattern=pattern, timeout=timeout) + try: + self.subprocess.expect(pattern=pattern, timeout=timeout) + except pexpect.EOF: + pass def send(self, s, end=os.linesep, signal=False): """Sends the given string or signal to std_in.""" @@ -234,14 +243,25 @@ class Command(object): """Blocks until process is complete.""" if self._uses_subprocess: # consume stdout and stderr - try: - stdout, stderr = self.subprocess.communicate() - self.__out = stdout - self.__err = stderr - except ValueError: - pass # Don't read from finished subprocesses. + if self.blocking: + try: + stdout, stderr = self.subprocess.communicate() + self.__out = stdout + self.__err = stderr + except ValueError: + pass # Don't read from finished subprocesses. + else: + self.subprocess.stdin.close() + self.std_out.close() + self.std_err.close() + self.subprocess.wait() else: - self.subprocess.wait() + self.subprocess.sendeof() + try: + self.subprocess.wait() + finally: + if self.subprocess.proc.stdout: + self.subprocess.proc.stdout.close() def pipe(self, command, timeout=None, cwd=None): """Runs the current command and passes its output to the next @@ -263,7 +283,6 @@ class Command(object): c.run(block=False, cwd=cwd) if data: c.send(data) - c.subprocess.sendeof() c.block() return c diff --git a/pipenv/vendor/modutil.LICENSE b/pipenv/vendor/modutil.LICENSE deleted file mode 100644 index f680f071..00000000 --- a/pipenv/vendor/modutil.LICENSE +++ /dev/null @@ -1,29 +0,0 @@ -BSD 3-Clause License - -Copyright (c) 2018, Brett Cannon -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -* Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/pipenv/vendor/parse.py b/pipenv/vendor/parse.py index ba58decb..7f9f0786 100644 --- a/pipenv/vendor/parse.py +++ b/pipenv/vendor/parse.py @@ -186,6 +186,19 @@ And messing about with alignment: Note that the "center" alignment does not test to make sure the value is centered - it just strips leading and trailing whitespace. +Width and precision may be used to restrict the size of matched text +from the input. Width specifies a minimum size and precision specifies +a maximum. For example: + +>>> parse('{:.2}{:.2}', 'look') # specifying precision + +>>> parse('{:4}{:4}', 'look at that') # specifying width + +>>> parse('{:4}{:.4}', 'look at that') # specifying both + +>>> parse('{:2d}{:2d}', '0440') # parsing two contiguous numbers + + Some notes for the date and time types: - the presence of the time part is optional (including ISO 8601, starting @@ -329,6 +342,9 @@ the pattern, the actual match represents the shortest successful match for **Version history (in brief)**: +- 1.9.0 We now honor precision and width specifiers when parsing numbers + and strings, allowing parsing of concatenated elements of fixed width + (thanks Julia Signell) - 1.8.4 Add LICENSE file at request of packagers. Correct handling of AM/PM to follow most common interpretation. Correct parsing of hexadecimal that looks like a binary prefix. @@ -389,7 +405,7 @@ See the end of the source file for the license of use. ''' from __future__ import absolute_import -__version__ = '1.8.4' +__version__ = '1.9.0' # yes, I now have two problems import re @@ -977,7 +993,11 @@ class Parser(object): self._group_index += 2 self._type_conversions[group] = lambda s, m: float(s) elif type == 'd': - s = r'\d+|0[xX][0-9a-fA-F]+|0[bB][01]+|0[oO][0-7]+' + if format.get('width'): + width = '{1,%s}' % int(format['width']) + else: + width = '+' + s = '\\d{w}|0[xX][0-9a-fA-F]{w}|0[bB][01]{w}|0[oO][0-7]{w}'.format(w=width) self._type_conversions[group] = int_convert(10) elif type == 'ti': s = r'(\d{4}-\d\d-\d\d)((\s+|T)%s)?(Z|\s*[-+]\d\d:?\d\d)?' % \ @@ -1038,6 +1058,13 @@ class Parser(object): elif type: s = r'\%s+' % type + elif format.get('precision'): + if format.get('width'): + s = '.{%s,%s}?' % (format['width'], format['precision']) + else: + s = '.{1,%s}?' % format['precision'] + elif format.get('width'): + s = '.{%s,}?' % format['width'] else: s = '.+?' @@ -1053,8 +1080,6 @@ class Parser(object): if not fill: fill = '0' s = '%s*' % fill + s - elif format['zero']: - s = '0*' + s # allow numbers to be prefixed with a sign s = r'[-+ ]?' + s diff --git a/pipenv/vendor/passa/cli/options.py b/pipenv/vendor/passa/cli/options.py index da89a3b1..f8ba1fe7 100644 --- a/pipenv/vendor/passa/cli/options.py +++ b/pipenv/vendor/passa/cli/options.py @@ -46,9 +46,12 @@ class Option(object): class ArgumentGroup(object): - def __init__(self, name, parser=None, is_mutually_exclusive=False, required=None, options=[]): + def __init__( + self, name, parser=None, + is_mutually_exclusive=False, + required=None, options=None): self.name = name - self.options = options + self.options = options or [] self.parser = parser self.required = required self.is_mutually_exclusive = is_mutually_exclusive diff --git a/pipenv/vendor/passa/internals/dependencies.py b/pipenv/vendor/passa/internals/dependencies.py index 53b19b17..358cc33b 100644 --- a/pipenv/vendor/passa/internals/dependencies.py +++ b/pipenv/vendor/passa/internals/dependencies.py @@ -154,6 +154,7 @@ def _get_dependencies_from_json(ireq, sources): return dependencies except Exception as e: print("unable to read dependencies via {0} ({1})".format(url, e)) + session.close() return diff --git a/pipenv/vendor/pathlib2.LICENSE.rst b/pipenv/vendor/pathlib2.LICENSE.rst deleted file mode 100644 index ddb51b8a..00000000 --- a/pipenv/vendor/pathlib2.LICENSE.rst +++ /dev/null @@ -1,23 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2014 Matthias C. M. Troffaes -Copyright (c) 2012-2014 Antoine Pitrou and contributors - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - diff --git a/pipenv/vendor/pip_shims/__init__.py b/pipenv/vendor/pip_shims/__init__.py index 70fb0d58..22bf130f 100644 --- a/pipenv/vendor/pip_shims/__init__.py +++ b/pipenv/vendor/pip_shims/__init__.py @@ -3,7 +3,7 @@ from __future__ import absolute_import import sys -__version__ = '0.3.1' +__version__ = '0.3.2' from . import shims diff --git a/pipenv/vendor/pip_shims/shims.py b/pipenv/vendor/pip_shims/shims.py index 7b81a608..34de6906 100644 --- a/pipenv/vendor/pip_shims/shims.py +++ b/pipenv/vendor/pip_shims/shims.py @@ -43,6 +43,9 @@ class _shims(object): "pip_shims.utils": utils } self.pip_version = getattr(self._modules["pip"], "__version__") + version_types = ["post", "pre", "dev", "rc"] + if any(post in self.pip_version.rsplit(".")[-1] for post in version_types): + self.pip_version, _, _ = self.pip_version.rpartition(".") self.parsed_pip_version = self._parse(self.pip_version) self._contextmanagers = ("RequirementTracker",) self._moves = { @@ -140,6 +143,7 @@ class _shims(object): ("wheel.WheelCache", "7", "9.0.3") ), "WheelBuilder": ("wheel.WheelBuilder", "7.0.0", "9999"), + "PyPI": ("models.index.PyPI", "7.0.0", "9999"), } def _ensure_methods(self, cls, classname, *methods): diff --git a/pipenv/vendor/pythonfinder/__init__.py b/pipenv/vendor/pythonfinder/__init__.py index 672724b4..fd9ec1be 100644 --- a/pipenv/vendor/pythonfinder/__init__.py +++ b/pipenv/vendor/pythonfinder/__init__.py @@ -1,6 +1,6 @@ from __future__ import print_function, absolute_import -__version__ = '1.1.2' +__version__ = '1.1.9' # Add NullHandler to "pythonfinder" logger, because Python2's default root # logger has no handler and warnings like this would be reported: diff --git a/pipenv/vendor/pythonfinder/cli.py b/pipenv/vendor/pythonfinder/cli.py index b5aa7da3..221cb2fd 100644 --- a/pipenv/vendor/pythonfinder/cli.py +++ b/pipenv/vendor/pythonfinder/cli.py @@ -17,7 +17,7 @@ from .pythonfinder import Finder @click.option( "--version", is_flag=True, default=False, help="Display PythonFinder version." ) -@click.option("--ignore-unsupported/--no-unsupported", is_flag=True, default=True, help="Ignore unsupported python versions.") +@click.option("--ignore-unsupported/--no-unsupported", is_flag=True, default=True, envvar="PYTHONFINDER_IGNORE_UNSUPPORTED", help="Ignore unsupported python versions.") @click.version_option(prog_name='pyfinder', version=__version__) @click.pass_context def cli(ctx, find=False, which=False, findall=False, version=False, ignore_unsupported=True): @@ -30,40 +30,57 @@ def cli(ctx, find=False, which=False, findall=False, version=False, ignore_unsup sys.exit(0) finder = Finder(ignore_unsupported=ignore_unsupported) if findall: - versions = finder.find_all_python_versions() + versions = [v for v in finder.find_all_python_versions()] if versions: click.secho("Found python at the following locations:", fg="green") for v in versions: py = v.py_version + comes_from = getattr(py, "comes_from", None) + if comes_from is not None: + comes_from_path = getattr(comes_from, "path", v.path) + else: + comes_from_path = v.path click.secho( - "Python: {py.version!s} ({py.architecture!s}) @ {py.comes_from.path!s}".format( - py=py + "{py.name!s}: {py.version!s} ({py.architecture!s}) @ {comes_from!s}".format( + py=py, comes_from=comes_from_path ), fg="yellow", ) + sys.exit(0) else: click.secho( "ERROR: No valid python versions found! Check your path and try again.", fg="red", ) if find: - if any([find.startswith("{0}".format(n)) for n in range(10)]): - found = finder.find_python_version(find.strip()) - else: - found = finder.system_path.python_executables + click.secho("Searching for python: {0!s}".format(find.strip()), fg="yellow") + found = finder.find_python_version(find.strip()) if found: - click.echo("Found Python Version: {0}".format(found), color="white") + py = found.py_version + comes_from = getattr(py, "comes_from", None) + if comes_from is not None: + comes_from_path = getattr(comes_from, "path", found.path) + else: + comes_from_path = found.path + arch = getattr(py, "architecture", None) + click.secho("Found python at the following locations:", fg="green") + click.secho( + "{py.name!s}: {py.version!s} ({py.architecture!s}) @ {comes_from!s}".format( + py=py, comes_from=comes_from_path + ), + fg="yellow", + ) sys.exit(0) else: - click.echo("Failed to find matching executable...") + click.secho("Failed to find matching executable...", fg="yellow") sys.exit(1) elif which: found = finder.system_path.which(which.strip()) if found: - click.echo("Found Executable: {0}".format(found), color="white") + click.secho("Found Executable: {0}".format(found), fg="white") sys.exit(0) else: - click.echo("Failed to find matching executable...") + click.secho("Failed to find matching executable...", fg="yellow") sys.exit(1) else: click.echo("Please provide a command", color="red") diff --git a/pipenv/vendor/pythonfinder/environment.py b/pipenv/vendor/pythonfinder/environment.py index 2cdb5fd9..ec4a760f 100644 --- a/pipenv/vendor/pythonfinder/environment.py +++ b/pipenv/vendor/pythonfinder/environment.py @@ -7,7 +7,13 @@ import sys PYENV_INSTALLED = bool(os.environ.get("PYENV_SHELL")) or bool( os.environ.get("PYENV_ROOT") ) -PYENV_ROOT = os.path.expandvars(os.environ.get("PYENV_ROOT", "~/.pyenv")) +ASDF_INSTALLED = bool(os.environ.get("ASDF_DATA_DIR")) +PYENV_ROOT = os.path.expanduser( + os.path.expandvars(os.environ.get("PYENV_ROOT", "~/.pyenv")) +) +ASDF_DATA_DIR = os.path.expanduser( + os.path.expandvars(os.environ.get("ASDF_DATA_DIR", "~/.asdf")) +) IS_64BIT_OS = None SYSTEM_ARCH = platform.architecture()[0] @@ -15,3 +21,6 @@ if sys.maxsize > 2 ** 32: IS_64BIT_OS = platform.machine() == "AMD64" else: IS_64BIT_OS = False + + +IGNORE_UNSUPPORTED = bool(os.environ.get("PYTHONFINDER_IGNORE_UNSUPPORTED", False)) diff --git a/pipenv/vendor/pythonfinder/models/mixins.py b/pipenv/vendor/pythonfinder/models/mixins.py index 8cbd45df..7d406548 100644 --- a/pipenv/vendor/pythonfinder/models/mixins.py +++ b/pipenv/vendor/pythonfinder/models/mixins.py @@ -2,12 +2,14 @@ from __future__ import absolute_import, unicode_literals import abc +import attr import operator import six -from ..utils import KNOWN_EXTS, unnest +from ..utils import ensure_path, KNOWN_EXTS, unnest +@attr.s class BasePath(object): def which(self, name): """Search in this path for an executable. @@ -33,7 +35,14 @@ class BasePath(object): return found def find_all_python_versions( - self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None + self, + major=None, + minor=None, + patch=None, + pre=None, + dev=None, + arch=None, + name=None, ): """Search for a specific python version on the path. Return all copies @@ -44,6 +53,7 @@ class BasePath(object): :param bool pre: Search for prereleases (default None) - prioritize releases if None :param bool dev: Search for devreleases (default None) - prioritize releases if None :param str arch: Architecture to include, e.g. '64bit', defaults to None + :param str name: The name of a python version, e.g. ``anaconda3-5.3.0`` :return: A list of :class:`~pythonfinder.models.PathEntry` instances matching the version requested. :rtype: List[:class:`~pythonfinder.models.PathEntry`] """ @@ -52,7 +62,14 @@ class BasePath(object): "find_all_python_versions" if self.is_dir else "find_python_version" ) sub_finder = operator.methodcaller( - call_method, major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch + call_method, + major=major, + minor=minor, + patch=patch, + pre=pre, + dev=dev, + arch=arch, + name=name, ) if not self.is_dir: return sub_finder(self) @@ -61,7 +78,14 @@ class BasePath(object): return [c for c in sorted(path_filter, key=version_sort, reverse=True)] def find_python_version( - self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None + self, + major=None, + minor=None, + patch=None, + pre=None, + dev=None, + arch=None, + name=None, ): """Search or self for the specified Python version and return the first match. @@ -72,6 +96,7 @@ class BasePath(object): :param bool pre: Search for prereleases (default None) - prioritize releases if None :param bool dev: Search for devreleases (default None) - prioritize releases if None :param str arch: Architecture to include, e.g. '64bit', defaults to None + :param str name: The name of a python version, e.g. ``anaconda3-5.3.0`` :returns: A :class:`~pythonfinder.models.PathEntry` instance matching the version requested. """ @@ -83,12 +108,13 @@ class BasePath(object): pre=pre, dev=dev, arch=arch, + name=name, ) is_py = operator.attrgetter("is_python") py_version = operator.attrgetter("as_python") if not self.is_dir: - if self.is_python and self.as_python and version_matcher(self.as_python): - return self + if self.is_python and self.as_python and version_matcher(self.py_version): + return attr.evolve(self) return finder = ( (child, child.as_python) diff --git a/pipenv/vendor/pythonfinder/models/path.py b/pipenv/vendor/pythonfinder/models/path.py index c90d9be3..7fae3312 100644 --- a/pipenv/vendor/pythonfinder/models/path.py +++ b/pipenv/vendor/pythonfinder/models/path.py @@ -10,21 +10,34 @@ from collections import defaultdict from itertools import chain import attr +import six from cached_property import cached_property from vistir.compat import Path, fs_str from .mixins import BasePath -from ..environment import PYENV_INSTALLED, PYENV_ROOT +from ..environment import PYENV_INSTALLED, PYENV_ROOT, ASDF_INSTALLED, ASDF_DATA_DIR from ..exceptions import InvalidPythonVersion from ..utils import ( - ensure_path, filter_pythons, looks_like_python, optional_instance_of, - path_is_known_executable, unnest + ensure_path, + filter_pythons, + looks_like_python, + optional_instance_of, + path_is_known_executable, + unnest, + normalize_path, + parse_pyenv_version_order, + parse_asdf_version_order ) from .python import PythonVersion +ASDF_SHIM_PATH = normalize_path(os.path.join(ASDF_DATA_DIR, "shims")) +PYENV_SHIM_PATH = normalize_path(os.path.join(PYENV_ROOT, "shims")) +SHIM_PATHS = [ASDF_SHIM_PATH, PYENV_SHIM_PATH] + + @attr.s class SystemPath(object): global_search = attr.ib(default=True) @@ -35,6 +48,7 @@ class SystemPath(object): python_version_dict = attr.ib(default=attr.Factory(defaultdict)) only_python = attr.ib(default=False) pyenv_finder = attr.ib(default=None, validator=optional_instance_of("PyenvPath")) + asdf_finder = attr.ib(default=None) system = attr.ib(default=False) _version_dict = attr.ib(default=attr.Factory(defaultdict)) ignore_unsupported = attr.ib(default=False) @@ -75,7 +89,7 @@ class SystemPath(object): if entry not in self._version_dict[version]: self._version_dict[version].append(entry) continue - if isinstance(entry, VersionPath): + if type(entry).__name__ == "VersionPath": for path in entry.paths.values(): if path not in self._version_dict[version] and path.is_python: self._version_dict[version].append(path) @@ -100,6 +114,8 @@ class SystemPath(object): self._setup_windows() if PYENV_INSTALLED: self._setup_pyenv() + if ASDF_INSTALLED: + self._setup_asdf() venv = os.environ.get("VIRTUAL_ENV") if os.name == "nt": bin_dir = "Scripts" @@ -119,28 +135,76 @@ class SystemPath(object): path=syspath_bin, is_root=True, only_python=False ) - def _setup_pyenv(self): - from .pyenv import PyenvFinder - - last_pyenv = next( - (p for p in reversed(self.path_order) if PYENV_ROOT.lower() in p.lower()), - None, + def _get_last_instance(self, path): + reversed_paths = reversed(self.path_order) + paths = [normalize_path(p) for p in reversed_paths] + normalized_target = normalize_path(path) + last_instance = next( + iter(p for p in paths if normalized_target in p), None ) try: - pyenv_index = self.path_order.index(last_pyenv) + path_index = self.path_order.index(last_instance) except ValueError: return - self.pyenv_finder = PyenvFinder.create(root=PYENV_ROOT, ignore_unsupported=self.ignore_unsupported) - # paths = (v.paths.values() for v in self.pyenv_finder.versions.values()) - root_paths = ( - p for path in self.pyenv_finder.expanded_paths for p in path if p.is_root - ) - before_path = self.path_order[: pyenv_index + 1] - after_path = self.path_order[pyenv_index + 2 :] + return path_index + + def _slice_in_paths(self, start_idx, paths): + before_path = self.path_order[: start_idx + 1] + after_path = self.path_order[start_idx + 2 :] self.path_order = ( - before_path + [p.path.as_posix() for p in root_paths] + after_path + before_path + [p.as_posix() for p in paths] + after_path ) - self.paths.update({p.path: p for p in root_paths}) + + def _remove_path(self, path): + path_copy = [p for p in reversed(self.path_order[:])] + new_order = [] + target = normalize_path(path) + path_map = { + normalize_path(pth): pth + for pth in self.paths.keys() + } + if target in path_map: + del self.paths[path_map.get(target)] + for current_path in path_copy: + normalized = normalize_path(current_path) + if normalized != target: + new_order.append(normalized) + new_order = [p for p in reversed(new_order)] + self.path_order = new_order + + def _setup_asdf(self): + from .python import PythonFinder + self.asdf_finder = PythonFinder.create( + root=ASDF_DATA_DIR, ignore_unsupported=True, + sort_function=parse_asdf_version_order, version_glob_path="installs/python/*") + asdf_index = self._get_last_instance(ASDF_DATA_DIR) + if not asdf_index: + # we are in a virtualenv without global pyenv on the path, so we should + # not write pyenv to the path here + return + root_paths = [p for p in self.asdf_finder.roots] + self._slice_in_paths(asdf_index, root_paths) + self.paths.update(self.asdf_finder.roots) + self._remove_path(normalize_path(os.path.join(ASDF_DATA_DIR, "shims"))) + self._register_finder("asdf", self.asdf_finder) + + def _setup_pyenv(self): + from .python import PythonFinder + + self.pyenv_finder = PythonFinder.create( + root=PYENV_ROOT, sort_function=parse_pyenv_version_order, + version_glob_path="versions/*", ignore_unsupported=self.ignore_unsupported + ) + pyenv_index = self._get_last_instance(PYENV_ROOT) + if not pyenv_index: + # we are in a virtualenv without global pyenv on the path, so we should + # not write pyenv to the path here + return + root_paths = [p for p in self.pyenv_finder.roots] + self._slice_in_paths(pyenv_index, root_paths) + + self.paths.update(self.pyenv_finder.roots) + self._remove_path(os.path.join(PYENV_ROOT, "shims")) self._register_finder("pyenv", self.pyenv_finder) def _setup_windows(self): @@ -155,7 +219,9 @@ class SystemPath(object): def get_path(self, path): path = ensure_path(path) - _path = self.paths.get(path.as_posix()) + _path = self.paths.get(path) + if not _path: + _path = self.paths.get(path.as_posix()) if not _path and path.as_posix() in self.path_order: _path = PathEntry.create( path=path.absolute(), is_root=True, only_python=self.only_python @@ -163,6 +229,14 @@ class SystemPath(object): self.paths[path.as_posix()] = _path return _path + def _get_paths(self): + return (self.get_path(k) for k in self.path_order) + + @cached_property + def path_entries(self): + paths = self._get_paths() + return paths + def find_all(self, executable): """Search the path for an executable. Return all copies. @@ -171,8 +245,8 @@ class SystemPath(object): :returns: List[PathEntry] """ sub_which = operator.methodcaller("which", name=executable) - filtered = filter(None, (sub_which(self.get_path(k)) for k in self.path_order)) - return [f for f in filtered] + filtered = (sub_which(self.get_path(k)) for k in self.path_order) + return list(filtered) def which(self, executable): """Search for an executable on the path. @@ -182,11 +256,39 @@ class SystemPath(object): :returns: :class:`~pythonfinder.models.PathEntry` object. """ sub_which = operator.methodcaller("which", name=executable) - filtered = filter(None, (sub_which(self.get_path(k)) for k in self.path_order)) - return next((f for f in filtered), None) + filtered = (sub_which(self.get_path(k)) for k in self.path_order) + return next(iter(f for f in filtered if f is not None), None) + + def _filter_paths(self, finder): + return ( + pth for pth in unnest(finder(p) for p in self.path_entries if p is not None) + if pth is not None + ) + + def _get_all_pythons(self, finder): + paths = {p.path.as_posix(): p for p in self._filter_paths(finder)} + paths.update(self.python_executables) + return (p for p in paths.values() if p is not None) + + def get_pythons(self, finder): + sort_key = operator.attrgetter("as_python.version_sort") + return ( + k for k in sorted( + (p for p in self._filter_paths(finder) if p.is_python), + key=sort_key, + reverse=True + ) if k is not None + ) def find_all_python_versions( - self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None + self, + major=None, + minor=None, + patch=None, + pre=None, + dev=None, + arch=None, + name=None, ): """Search for a specific python version on the path. Return all copies @@ -197,32 +299,46 @@ class SystemPath(object): :param bool pre: Search for prereleases (default None) - prioritize releases if None :param bool dev: Search for devreleases (default None) - prioritize releases if None :param str arch: Architecture to include, e.g. '64bit', defaults to None + :param str name: The name of a python version, e.g. ``anaconda3-5.3.0`` :return: A list of :class:`~pythonfinder.models.PathEntry` instances matching the version requested. :rtype: List[:class:`~pythonfinder.models.PathEntry`] """ sub_finder = operator.methodcaller( "find_all_python_versions", - major, + major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch, + name=name, ) + alternate_sub_finder = None + if major and not (minor or patch or pre or dev or arch or name): + alternate_sub_finder = operator.methodcaller( + "find_all_python_versions", + major=None, + name=major + ) if os.name == "nt" and self.windows_finder: windows_finder_version = sub_finder(self.windows_finder) if windows_finder_version: return windows_finder_version - paths = (self.get_path(k) for k in self.path_order) - path_filter = filter( - None, unnest((sub_finder(p) for p in paths if p is not None)) - ) - version_sort = operator.attrgetter("as_python.version_sort") - return [c for c in sorted(path_filter, key=version_sort, reverse=True)] + values = list(self.get_pythons(sub_finder)) + if not values and alternate_sub_finder is not None: + values = list(self.get_pythons(alternate_sub_finder)) + return values def find_python_version( - self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None + self, + major=None, + minor=None, + patch=None, + pre=None, + dev=None, + arch=None, + name=None, ): """Search for a specific python version on the path. @@ -233,10 +349,24 @@ class SystemPath(object): :param bool pre: Search for prereleases (default None) - prioritize releases if None :param bool dev: Search for devreleases (default None) - prioritize releases if None :param str arch: Architecture to include, e.g. '64bit', defaults to None + :param str name: The name of a python version, e.g. ``anaconda3-5.3.0`` :return: A :class:`~pythonfinder.models.PathEntry` instance matching the version requested. :rtype: :class:`~pythonfinder.models.PathEntry` """ + if isinstance(major, six.string_types) and not minor and not patch: + # Only proceed if this is in the format "x.y.z" or similar + if major.count(".") > 0 and major[0].isdigit(): + version = major.split(".", 2) + if len(version) > 3: + major, minor, patch, rest = version + elif len(version) == 3: + major, minor, patch = version + else: + major, minor = version + else: + name = "{0!s}".format(major) + major = None sub_finder = operator.methodcaller( "find_python_version", major, @@ -245,7 +375,15 @@ class SystemPath(object): pre=pre, dev=dev, arch=arch, + name=name, ) + alternate_sub_finder = None + if major and not (minor or patch or pre or dev or arch or name): + alternate_sub_finder = operator.methodcaller( + "find_all_python_versions", + major=None, + name=major + ) if major and minor and patch: _tuple_pre = pre if pre is not None else False _tuple_dev = dev if dev is not None else False @@ -255,12 +393,9 @@ class SystemPath(object): windows_finder_version = sub_finder(self.windows_finder) if windows_finder_version: return windows_finder_version - paths = (self.get_path(k) for k in self.path_order) - path_filter = filter(None, (sub_finder(p) for p in paths if p is not None)) - version_sort = operator.attrgetter("as_python.version_sort") - ver = next( - (c for c in sorted(path_filter, key=version_sort, reverse=True)), None - ) + ver = next(iter(self.get_pythons(sub_finder)), None) + if not ver and alternate_sub_finder is not None: + ver = next(iter(self.get_pythons(alternate_sub_finder)), None) if ver: if ver.as_python.version_tuple[:5] in self.python_version_dict: self.python_version_dict[ver.as_python.version_tuple[:5]].append(ver) @@ -269,23 +404,29 @@ class SystemPath(object): return ver @classmethod - def create(cls, path=None, system=False, only_python=False, global_search=True, ignore_unsupported=False): + def create( + cls, + path=None, + system=False, + only_python=False, + global_search=True, + ignore_unsupported=True, + ): """Create a new :class:`pythonfinder.models.SystemPath` instance. :param path: Search path to prepend when searching, defaults to None :param path: str, optional - :param system: Whether to use the running python by default instead of searching, defaults to False - :param system: bool, optional - :param only_python: Whether to search only for python executables, defaults to False - :param only_python: bool, optional - :param ignore_unsupported: Whether to ignore unsupported python versions, if False, an error is raised, defaults to True - :param ignore_unsupported: bool, optional + :param bool system: Whether to use the running python by default instead of searching, defaults to False + :param bool only_python: Whether to search only for python executables, defaults to False + :param bool ignore_unsupported: Whether to ignore unsupported python versions, if False, an error is raised, defaults to True :return: A new :class:`pythonfinder.models.SystemPath` instance. :rtype: :class:`pythonfinder.models.SystemPath` """ path_entries = defaultdict(PathEntry) paths = [] + if ignore_unsupported: + os.environ["PYTHONFINDER_IGNORE_UNSUPPORTED"] = fs_str("1") if global_search: paths = os.environ.get("PATH").split(os.pathsep) if path: @@ -298,6 +439,7 @@ class SystemPath(object): path=p.absolute(), is_root=True, only_python=only_python ) for p in _path_objects + if not any(shim in normalize_path(str(p)) for shim in SHIM_PATHS) } ) return cls( @@ -310,14 +452,15 @@ class SystemPath(object): ) -@attr.s +@attr.s(slots=True) class PathEntry(BasePath): path = attr.ib(default=None, validator=optional_instance_of(Path)) _children = attr.ib(default=attr.Factory(dict)) is_root = attr.ib(default=True) only_python = attr.ib(default=False) - py_version = attr.ib(default=None) - pythons = attr.ib() + name = attr.ib() + py_version = attr.ib() + _pythons = attr.ib(default=attr.Factory(defaultdict)) def __str__(self): return fs_str("{0}".format(self.path.as_posix())) @@ -329,78 +472,119 @@ class PathEntry(BasePath): children = self.path.iterdir() return children + def _gen_children(self): + pass_name = self.name != self.path.name + pass_args = {"is_root": False, "only_python": self.only_python} + if pass_name: + pass_args["name"] = self.name + + if not self.is_dir: + yield (self.path.as_posix(), copy.deepcopy(self)) + elif self.is_root: + for child in self._filter_children(): + if any(shim in normalize_path(str(child)) for shim in SHIM_PATHS): + continue + yield (child.as_posix(), PathEntry.create(path=child, **pass_args)) + return + @cached_property def children(self): - if not self._children and self.is_dir and self.is_root: - self._children = { - child.as_posix(): PathEntry.create(path=child, is_root=False) - for child in self._filter_children() - } - elif not self.is_dir: - self._children = {self.path.as_posix(): self} + if not self._children: + children = {} + for child_key, child_val in self._gen_children(): + children[child_key] = child_val + self._children = children return self._children - @pythons.default - def get_pythons(self): - pythons = defaultdict() + @name.default + def get_name(self): + return self.path.name + + @py_version.default + def get_py_version(self): + from ..environment import IGNORE_UNSUPPORTED if self.is_dir: - for path, entry in self.children.items(): - _path = ensure_path(entry.path) - if entry.is_python: - pythons[_path.as_posix()] = entry - else: - if self.is_python: - _path = ensure_path(self.path) - pythons[_path.as_posix()] = copy.deepcopy(self) - return pythons + return None + if self.is_python: + try: + py_version = PythonVersion.from_path(path=self, name=self.name) + except InvalidPythonVersion: + py_version = None + except Exception: + if not IGNORE_UNSUPPORTED: + raise + return py_version + return + + @property + def pythons(self): + if not self._pythons: + if self.is_dir: + for path, entry in self.children.items(): + _path = ensure_path(entry.path) + if entry.is_python: + self._pythons[_path.as_posix()] = entry + else: + if self.is_python: + _path = ensure_path(self.path) + self._pythons[_path.as_posix()] = self + return self._pythons @cached_property def as_python(self): + py_version = None + if self.py_version: + return self.py_version if not self.is_dir and self.is_python: - if not self.py_version: - try: - from .python import PythonVersion - - self.py_version = PythonVersion.from_path(self.path) - except (ValueError, InvalidPythonVersion): - self.py_version = None - return self.py_version + try: + from .python import PythonVersion + py_version = PythonVersion.from_path(path=attr.evolve(self), name=self.name) + except (ValueError, InvalidPythonVersion): + py_version = None + return py_version @classmethod - def create(cls, path, is_root=False, only_python=False, pythons=None): + def create(cls, path, is_root=False, only_python=False, pythons=None, name=None): """Helper method for creating new :class:`pythonfinder.models.PathEntry` instances. - :param path: Path to the specified location. - :type path: str - :param is_root: Whether this is a root from the environment PATH variable, defaults to False - :param is_root: bool, optional - :param only_python: Whether to search only for python executables, defaults to False - :param only_python: bool, optional - :param pythons: A dictionary of existing python objects (usually from a finder), defaults to None - :param pythons: dict, optional + :param str path: Path to the specified location. + :param bool is_root: Whether this is a root from the environment PATH variable, defaults to False + :param bool only_python: Whether to search only for python executables, defaults to False + :param dict pythons: A dictionary of existing python objects (usually from a finder), defaults to None + :param str name: Name of the python version, e.g. ``anaconda3-5.3.0`` :return: A new instance of the class. :rtype: :class:`pythonfinder.models.PathEntry` """ target = ensure_path(path) - creation_args = {"path": target, "is_root": is_root, "only_python": only_python} + guessed_name = False + if not name: + guessed_name = True + name = target.name + creation_args = {"path": target, "is_root": is_root, "only_python": only_python, "name": name} if pythons: creation_args["pythons"] = pythons _new = cls(**creation_args) if pythons and only_python: children = {} + child_creation_args = { + "is_root": False, + "only_python": only_python + } + if not guessed_name: + child_creation_args["name"] = name for pth, python in pythons.items(): + if any(shim in normalize_path(str(pth)) for shim in SHIM_PATHS): + continue pth = ensure_path(pth) children[pth.as_posix()] = PathEntry( - path=pth, is_root=False, only_python=only_python, py_version=python + py_version=python, + path=pth, + **child_creation_args ) _new._children = children return _new - @cached_property - def name(self): - return self.path.name - @cached_property def is_dir(self): try: @@ -416,28 +600,31 @@ class PathEntry(BasePath): @cached_property def is_python(self): return self.is_executable and ( - self.py_version or looks_like_python(self.path.name) + looks_like_python(self.path.name) ) @attr.s class VersionPath(SystemPath): base = attr.ib(default=None, validator=optional_instance_of(Path)) + name = attr.ib(default=None) @classmethod - def create(cls, path, only_python=True, pythons=None): + def create(cls, path, only_python=True, pythons=None, name=None): """Accepts a path to a base python version directory. - Generates the pyenv version listings for it""" + Generates the version listings for it""" + from .path import PathEntry path = ensure_path(path) path_entries = defaultdict(PathEntry) - if not path.name.lower() in ["scripts", "bin"]: - bin_name = "Scripts" if os.name == "nt" else "bin" - bin_dir = path / bin_name - else: - bin_dir = path + bin_ = "{base}/bin" + if path.as_posix().endswith(Path(bin_).name): + path = path.parent + bin_dir = ensure_path(bin_.format(base=path.as_posix())) + if not name: + name = path.name current_entry = PathEntry.create( - bin_dir, is_root=True, only_python=True, pythons=pythons + bin_dir, is_root=True, only_python=True, pythons=pythons, name=name ) path_entries[bin_dir.as_posix()] = current_entry - return cls(base=bin_dir, paths=path_entries) + return cls(name=name, base=bin_dir, paths=path_entries) diff --git a/pipenv/vendor/pythonfinder/models/pyenv.py b/pipenv/vendor/pythonfinder/models/pyenv.py deleted file mode 100644 index 527c5f0a..00000000 --- a/pipenv/vendor/pythonfinder/models/pyenv.py +++ /dev/null @@ -1,98 +0,0 @@ -# -*- coding=utf-8 -*- -from __future__ import absolute_import, print_function - -import logging - -from collections import defaultdict - -import attr -import sysconfig - -from vistir.compat import Path - -from ..utils import ensure_path, optional_instance_of, get_python_version, filter_pythons -from .mixins import BaseFinder -from .path import VersionPath -from .python import PythonVersion - - -logger = logging.getLogger(__name__) - - -@attr.s -class PyenvFinder(BaseFinder): - root = attr.ib(default=None, validator=optional_instance_of(Path)) - # ignore_unsupported should come before versions, because its value is used - # in versions's default initializer. - ignore_unsupported = attr.ib(default=False) - versions = attr.ib() - pythons = attr.ib() - - @classmethod - def version_from_bin_dir(cls, base_dir): - pythons = [py for py in filter_pythons(base_dir)] - py_version = None - for py in pythons: - version = get_python_version(py.as_posix()) - try: - py_version = PythonVersion.parse(version) - except Exception: - continue - if py_version: - return py_version - return - - @versions.default - def get_versions(self): - versions = defaultdict(VersionPath) - bin_ = sysconfig._INSTALL_SCHEMES[sysconfig._get_default_scheme()]["scripts"] - for p in self.root.glob("versions/*"): - if p.parent.name == "envs": - continue - try: - version = PythonVersion.parse(p.name) - except ValueError: - bin_dir = Path(bin_.format(base=p.as_posix())) - if bin_dir.exists(): - version = self.version_from_bin_dir(bin_dir) - if not version: - if not self.ignore_unsupported: - raise - continue - except Exception: - if not self.ignore_unsupported: - raise - logger.warning( - 'Unsupported Python version %r, ignoring...', - p.name, exc_info=True - ) - continue - if not version: - continue - version_tuple = ( - version.get("major"), - version.get("minor"), - version.get("patch"), - version.get("is_prerelease"), - version.get("is_devrelease"), - version.get("is_debug") - ) - versions[version_tuple] = VersionPath.create( - path=p.resolve(), only_python=True - ) - return versions - - @pythons.default - def get_pythons(self): - pythons = defaultdict() - for v in self.versions.values(): - for p in v.paths.values(): - _path = ensure_path(p.path) - if p.is_python: - pythons[_path] = p - return pythons - - @classmethod - def create(cls, root, ignore_unsupported=False): - root = ensure_path(root) - return cls(root=root, ignore_unsupported=ignore_unsupported) diff --git a/pipenv/vendor/pythonfinder/models/python.py b/pipenv/vendor/pythonfinder/models/python.py index c71b9d9b..eac856c5 100644 --- a/pipenv/vendor/pythonfinder/models/python.py +++ b/pipenv/vendor/pythonfinder/models/python.py @@ -3,21 +3,240 @@ from __future__ import absolute_import, print_function import copy import platform +import operator +import logging from collections import defaultdict import attr -from packaging.version import Version, LegacyVersion +from packaging.version import Version from packaging.version import parse as parse_version +from vistir.compat import Path -from ..environment import SYSTEM_ARCH +from ..environment import SYSTEM_ARCH, PYENV_ROOT, ASDF_DATA_DIR +from .mixins import BaseFinder, BasePath from ..utils import ( - _filter_none, ensure_path, get_python_version, optional_instance_of + _filter_none, + ensure_path, + get_python_version, + optional_instance_of, + unnest, + is_in_path, + parse_pyenv_version_order, + parse_asdf_version_order, ) +logger = logging.getLogger(__name__) -@attr.s + +@attr.s(slots=True) +class PythonFinder(BaseFinder, BasePath): + root = attr.ib(default=None, validator=optional_instance_of(Path)) + #: ignore_unsupported should come before versions, because its value is used + #: in versions's default initializer. + ignore_unsupported = attr.ib(default=True) + #: The function to use to sort version order when returning an ordered verion set + sort_function = attr.ib(default=None) + paths = attr.ib(default=attr.Factory(list)) + roots = attr.ib(default=attr.Factory(defaultdict)) + #: Glob path for python versions off of the root directory + version_glob_path = attr.ib(default="versions/*") + versions = attr.ib() + pythons = attr.ib() + + @property + def expanded_paths(self): + return ( + path for path in unnest(p for p in self.versions.values()) + if path is not None + ) + + @property + def is_pyenv(self): + return is_in_path(str(self.root), PYENV_ROOT) + + @property + def is_asdf(self): + return is_in_path(str(self.root), ASDF_DATA_DIR) + + def get_version_order(self): + version_paths = [ + p for p in self.root.glob(self.version_glob_path) + if not (p.parent.name == "envs" or p.name == "envs") + ] + versions = {v.name: v for v in version_paths} + if self.is_pyenv: + version_order = [versions[v] for v in parse_pyenv_version_order() if v in versions] + elif self.is_asdf: + version_order = [versions[v] for v in parse_asdf_version_order() if v in versions] + for version in version_order: + version_paths.remove(version) + if version_order: + version_order += version_paths + else: + version_order = version_paths + return version_order + + @classmethod + def version_from_bin_dir(cls, base_dir, name=None): + from .path import PathEntry + py_version = None + version_path = PathEntry.create( + path=base_dir.absolute().as_posix(), + only_python=True, + name=base_dir.parent.name, + ) + py_version = next(iter(version_path.find_all_python_versions()), None) + return py_version + + @versions.default + def get_versions(self): + from .path import PathEntry + versions = defaultdict() + bin_ = "{base}/bin" + for p in self.get_version_order(): + bin_dir = Path(bin_.format(base=p.as_posix())) + version_path = None + if bin_dir.exists(): + version_path = PathEntry.create( + path=bin_dir.absolute().as_posix(), + only_python=False, + name=p.name, + is_root=True, + ) + version = None + try: + version = PythonVersion.parse(p.name) + except ValueError: + entry = next(iter(version_path.find_all_python_versions()), None) + if not entry: + if self.ignore_unsupported: + continue + raise + else: + version = entry.py_version.as_dict() + except Exception: + if not self.ignore_unsupported: + raise + logger.warning( + "Unsupported Python version %r, ignoring...", p.name, exc_info=True + ) + continue + if not version: + continue + version_tuple = ( + version.get("major"), + version.get("minor"), + version.get("patch"), + version.get("is_prerelease"), + version.get("is_devrelease"), + version.get("is_debug"), + ) + self.roots[p] = version_path + versions[version_tuple] = version_path + self.paths.append(version_path) + return versions + + @pythons.default + def get_pythons(self): + pythons = defaultdict() + for p in self.paths: + pythons.update(p.pythons) + return pythons + + @classmethod + def create(cls, root, sort_function=None, version_glob_path=None, ignore_unsupported=True): + root = ensure_path(root) + if not version_glob_path: + version_glob_path = "versions/*" + return cls(root=root, ignore_unsupported=ignore_unsupported, + sort_function=sort_function, version_glob_path=version_glob_path) + + def find_all_python_versions( + self, + major=None, + minor=None, + patch=None, + pre=None, + dev=None, + arch=None, + name=None, + ): + """Search for a specific python version on the path. Return all copies + + :param major: Major python version to search for. + :type major: int + :param int minor: Minor python version to search for, defaults to None + :param int patch: Patch python version to search for, defaults to None + :param bool pre: Search for prereleases (default None) - prioritize releases if None + :param bool dev: Search for devreleases (default None) - prioritize releases if None + :param str arch: Architecture to include, e.g. '64bit', defaults to None + :param str name: The name of a python version, e.g. ``anaconda3-5.3.0`` + :return: A list of :class:`~pythonfinder.models.PathEntry` instances matching the version requested. + :rtype: List[:class:`~pythonfinder.models.PathEntry`] + """ + + version_matcher = operator.methodcaller( + "matches", + major=major, + minor=minor, + patch=patch, + pre=pre, + dev=dev, + arch=arch, + name=name, + ) + py = operator.attrgetter("as_python") + pythons = ( + py_ver for py_ver in (py(p) for p in self.pythons.values() if p is not None) + if py_ver is not None + ) + # pythons = filter(None, [p.as_python for p in self.pythons.values()]) + matching_versions = filter(lambda py: version_matcher(py), pythons) + version_sort = operator.attrgetter("version_sort") + return sorted(matching_versions, key=version_sort, reverse=True) + + def find_python_version( + self, + major=None, + minor=None, + patch=None, + pre=None, + dev=None, + arch=None, + name=None, + ): + """Search or self for the specified Python version and return the first match. + + :param major: Major version number. + :type major: int + :param int minor: Minor python version to search for, defaults to None + :param int patch: Patch python version to search for, defaults to None + :param bool pre: Search for prereleases (default None) - prioritize releases if None + :param bool dev: Search for devreleases (default None) - prioritize releases if None + :param str arch: Architecture to include, e.g. '64bit', defaults to None + :param str name: The name of a python version, e.g. ``anaconda3-5.3.0`` + :returns: A :class:`~pythonfinder.models.PathEntry` instance matching the version requested. + """ + + version_matcher = operator.methodcaller( + "matches", + major=major, + minor=minor, + patch=patch, + pre=pre, + dev=dev, + arch=arch, + name=name, + ) + pythons = filter(None, [p.as_python for p in self.pythons.values()]) + matching_versions = filter(lambda py: version_matcher(py), pythons) + version_sort = operator.attrgetter("version_sort") + return next(iter(c for c in sorted(matching_versions, key=version_sort, reverse=True)), None) + + +@attr.s(slots=True) class PythonVersion(object): major = attr.ib(default=0) minor = attr.ib(default=None) @@ -30,6 +249,7 @@ class PythonVersion(object): architecture = attr.ib(default=None) comes_from = attr.ib(default=None) executable = attr.ib(default=None) + name = attr.ib(default=None) @property def version_sort(self): @@ -65,22 +285,37 @@ class PythonVersion(object): self.patch, self.is_prerelease, self.is_devrelease, - self.is_debug + self.is_debug, ) def matches( - self, major=None, minor=None, patch=None, pre=False, dev=False, arch=None, debug=False + self, + major=None, + minor=None, + patch=None, + pre=False, + dev=False, + arch=None, + debug=False, + name=None, ): - if arch and arch.isdigit(): - arch = "{0}bit".format(arch) + if arch: + own_arch = self.get_architecture() + if arch.isdigit(): + arch = "{0}bit".format(arch) return ( (major is None or self.major == major) and (minor is None or self.minor == minor) and (patch is None or self.patch == patch) and (pre is None or self.is_prerelease == pre) and (dev is None or self.is_devrelease == dev) - and (arch is None or self.architecture == arch) + and (arch is None or own_arch == arch) and (debug is None or self.is_debug == debug) + and ( + name is None + or (name and self.name) + and (self.name == name or self.name.startswith(name)) + ) ) def as_major(self): @@ -93,6 +328,18 @@ class PythonVersion(object): self_dict.update({"patch": None}) return self.create(**self_dict) + def as_dict(self): + return { + "major": self.major, + "minor": self.minor, + "patch": self.patch, + "is_prerelease": self.is_prerelease, + "is_postrelease": self.is_postrelease, + "is_devrelease": self.is_devrelease, + "is_debug": self.is_debug, + "version": self.version, + } + @classmethod def parse(cls, version): """Parse a valid version string into a dictionary @@ -138,8 +385,15 @@ class PythonVersion(object): "version": version, } + def get_architecture(self): + if self.architecture: + return self.architecture + arch, _ = platform.architecture(self.comes_from.path.as_posix()) + self.architecture = arch + return self.architecture + @classmethod - def from_path(cls, path): + def from_path(cls, path, name=None): """Parses a python version from a system path. Raises: @@ -147,29 +401,33 @@ class PythonVersion(object): :param path: A string or :class:`~pythonfinder.models.path.PathEntry` :type path: str or :class:`~pythonfinder.models.path.PathEntry` instance - :param launcher_entry: A python launcher environment object. + :param str name: Name of the python distribution in question :return: An instance of a PythonVersion. :rtype: :class:`~pythonfinder.models.python.PythonVersion` """ from .path import PathEntry + from ..environment import IGNORE_UNSUPPORTED if not isinstance(path, PathEntry): - path = PathEntry.create(path, is_root=False, only_python=True) - if not path.is_python: + path = PathEntry.create(path, is_root=False, only_python=True, name=name) + if not path.is_python and not IGNORE_UNSUPPORTED: raise ValueError("Not a valid python path: %s" % path.path) return - py_version = get_python_version(str(path.path)) - instance_dict = cls.parse(py_version) - if not isinstance(instance_dict.get("version"), Version): + py_version = get_python_version(path.path.absolute().as_posix()) + instance_dict = cls.parse(py_version.strip()) + if not isinstance(instance_dict.get("version"), Version) and not IGNORE_UNSUPPORTED: raise ValueError("Not a valid python path: %s" % path.path) return - architecture, _ = platform.architecture(path.path.as_posix()) - instance_dict.update({"comes_from": path, "architecture": architecture}) + if not name: + name = path.name + instance_dict.update( + {"comes_from": path, "name": name} + ) return cls(**instance_dict) @classmethod - def from_windows_launcher(cls, launcher_entry): + def from_windows_launcher(cls, launcher_entry, name=None): """Create a new PythonVersion instance from a Windows Launcher Entry :param launcher_entry: A python launcher environment object. @@ -193,12 +451,14 @@ class PythonVersion(object): launcher_entry.info, "sys_architecture", SYSTEM_ARCH ), "executable": exe_path, + "name": name } ) py_version = cls.create(**creation_dict) - comes_from = PathEntry.create(exe_path, only_python=True) + comes_from = PathEntry.create(exe_path, only_python=True, name=name) comes_from.py_version = copy.deepcopy(py_version) py_version.comes_from = comes_from + py_version.name = comes_from.name return py_version @classmethod diff --git a/pipenv/vendor/pythonfinder/models/windows.py b/pipenv/vendor/pythonfinder/models/windows.py index fcb4d42a..f985630f 100644 --- a/pipenv/vendor/pythonfinder/models/windows.py +++ b/pipenv/vendor/pythonfinder/models/windows.py @@ -22,7 +22,14 @@ class WindowsFinder(BaseFinder): pythons = attr.ib() def find_all_python_versions( - self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None + self, + major=None, + minor=None, + patch=None, + pre=None, + dev=None, + arch=None, + name=None, ): version_matcher = operator.methodcaller( "matches", @@ -32,6 +39,7 @@ class WindowsFinder(BaseFinder): pre=pre, dev=dev, arch=arch, + name=name, ) py_filter = filter( None, filter(lambda c: version_matcher(c), self.version_list) @@ -40,13 +48,26 @@ class WindowsFinder(BaseFinder): return [c.comes_from for c in sorted(py_filter, key=version_sort, reverse=True)] def find_python_version( - self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None + self, + major=None, + minor=None, + patch=None, + pre=None, + dev=None, + arch=None, + name=None, ): return next( ( v for v in self.find_all_python_versions( - major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch + major=major, + minor=minor, + patch=patch, + pre=pre, + dev=dev, + arch=arch, + name=None, ) ), None, @@ -60,10 +81,13 @@ class WindowsFinder(BaseFinder): env_versions = pep514env.findall() path = None for version_object in env_versions: - install_path = getattr(version_object.info, 'install_path', None) + install_path = getattr(version_object.info, "install_path", None) if install_path is None: continue - path = ensure_path(install_path.__getattr__("")) + try: + path = ensure_path(install_path.__getattr__("")) + except AttributeError: + continue try: py_version = PythonVersion.from_windows_launcher(version_object) except InvalidPythonVersion: diff --git a/pipenv/vendor/pythonfinder/pythonfinder.py b/pipenv/vendor/pythonfinder/pythonfinder.py index e965bb51..011754ea 100644 --- a/pipenv/vendor/pythonfinder/pythonfinder.py +++ b/pipenv/vendor/pythonfinder/pythonfinder.py @@ -4,16 +4,18 @@ import os import six import operator from .models import SystemPath +from vistir.compat import lru_cache class Finder(object): - def __init__(self, path=None, system=False, global_search=True, ignore_unsupported=False): - """Finder A cross-platform Finder for locating python and other executables. + def __init__(self, path=None, system=False, global_search=True, ignore_unsupported=True): + """ + Finder A cross-platform Finder for locating python and other executables. Searches for python and other specified binaries starting in `path`, if supplied, but searching the bin path of `sys.executable` if `system=True`, and then searching in the `os.environ['PATH']` if `global_search=True`. When `global_search` - is `False`, this search operation is restricted to the allowed locations of + is `False`, this search operation is restricted to the allowed locations of `path` and `system`. :param path: A bin-directory search location, defaults to None @@ -34,6 +36,14 @@ class Finder(object): self._system_path = None self._windows_finder = None + def __hash__(self): + return hash( + (self.path_prepend, self.system, self.global_search, self.ignore_unsupported) + ) + + def __eq__(self, other): + return self.__hash__() == other.__hash__() + @property def system_path(self): if not self._system_path: @@ -56,8 +66,9 @@ class Finder(object): def which(self, exe): return self.system_path.which(exe) + @lru_cache(maxsize=1024) def find_python_version( - self, major, minor=None, patch=None, pre=None, dev=None, arch=None + self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None, name=None ): from .models import PythonVersion @@ -69,12 +80,24 @@ class Finder(object): and patch is None ): if arch is None and "-" in major: - major, arch = major.rsplit("-", 1) - if not arch.isdigit(): - major = "{0}-{1}".format(major, arch) + orig_string = "{0!s}".format(major) + major, _, arch = major.rpartition("-") + if arch.startswith("x"): + arch = arch.lstrip("x") + if arch.lower().endswith("bit"): + arch = arch.lower().replace("bit", "") + if not (arch.isdigit() and (int(arch) & int(arch) - 1) == 0): + major = orig_string + arch = None else: arch = "{0}bit".format(arch) - version_dict = PythonVersion.parse(major) + try: + version_dict = PythonVersion.parse(major) + except ValueError: + if name is None: + name = "{0!s}".format(major) + major = None + version_dict = {} major = version_dict.get("major", major) minor = version_dict.get("minor", minor) patch = version_dict.get("patch", patch) @@ -83,16 +106,17 @@ class Finder(object): arch = version_dict.get("architecture", arch) if arch is None else arch if os.name == "nt": match = self.windows_finder.find_python_version( - major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch + major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch, name=name ) if match: return match return self.system_path.find_python_version( - major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch + major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch, name=name ) + @lru_cache(maxsize=1024) def find_all_python_versions( - self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None + self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None, name=None ): version_sort = operator.attrgetter("as_python.version_sort") python_version_dict = getattr(self.system_path, "python_version_dict") @@ -109,7 +133,7 @@ class Finder(object): paths = sorted(paths, key=version_sort, reverse=True) return paths versions = self.system_path.find_all_python_versions( - major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch + major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch, name=name ) if not isinstance(versions, list): versions = [versions] diff --git a/pipenv/vendor/pythonfinder/utils.py b/pipenv/vendor/pythonfinder/utils.py index dced9eab..2debd80e 100644 --- a/pipenv/vendor/pythonfinder/utils.py +++ b/pipenv/vendor/pythonfinder/utils.py @@ -1,25 +1,32 @@ # -*- coding=utf-8 -*- from __future__ import absolute_import, print_function -import locale +import itertools import os -import subprocess -import sys from fnmatch import fnmatch -from itertools import chain import attr +import io import six import vistir +from .environment import PYENV_ROOT, ASDF_DATA_DIR from .exceptions import InvalidPythonVersion +six.add_move(six.MovedAttribute("Iterable", "collections", "collections.abc")) +from six.moves import Iterable + +try: + from functools import lru_cache +except ImportError: + from backports.functools_lru_cache import lru_cache + PYTHON_IMPLEMENTATIONS = ( "python", "ironpython", "jython", "pypy", "anaconda", "miniconda", - "stackless", "activepython" + "stackless", "activepython", "micropython" ) RULES_BASE = ["*{0}", "*{0}?", "*{0}?.?", "*{0}?.?m"] RULES = [rule.format(impl) for impl in PYTHON_IMPLEMENTATIONS for rule in RULES_BASE] @@ -29,31 +36,39 @@ KNOWN_EXTS = KNOWN_EXTS | set( filter(None, os.environ.get("PATHEXT", "").split(os.pathsep)) ) +MATCH_RULES = [] +for rule in RULES: + MATCH_RULES.extend( + [ + "{0}.{1}".format(rule, ext) if ext else "{0}".format(rule) + for ext in KNOWN_EXTS + ] + ) + +@lru_cache(maxsize=128) def get_python_version(path): """Get python version string using subprocess from a given path.""" version_cmd = [path, "-c", "import sys; print(sys.version.split()[0])"] try: - out, _ = vistir.misc.run(version_cmd, block=True, nospin=True) + c = vistir.misc.run(version_cmd, block=True, nospin=True, return_object=True, + combine_stderr=False) except OSError: raise InvalidPythonVersion("%s is not a valid python path" % path) - if not out: + if not c.out: raise InvalidPythonVersion("%s is not a valid python path" % path) - return out.strip() + return c.out.strip() def optional_instance_of(cls): return attr.validators.optional(attr.validators.instance_of(cls)) -def path_and_exists(path): - return attr.validators.instance_of(vistir.compat.Path) and path.exists() - - def path_is_executable(path): return os.access(str(path), os.X_OK) +@lru_cache(maxsize=1024) def path_is_known_executable(path): return ( path_is_executable(path) @@ -62,26 +77,22 @@ def path_is_known_executable(path): ) +@lru_cache(maxsize=1024) def looks_like_python(name): - match_rules = [] - for rule in RULES: - match_rules.extend( - [ - "{0}.{1}".format(rule, ext) if ext else "{0}".format(rule) - for ext in KNOWN_EXTS - ] - ) if not any(name.lower().startswith(py_name) for py_name in PYTHON_IMPLEMENTATIONS): return False - return any(fnmatch(name, rule) for rule in match_rules) + return any(fnmatch(name, rule) for rule in MATCH_RULES) +@lru_cache(maxsize=1024) def path_is_python(path): return path_is_executable(path) and looks_like_python(path.name) +@lru_cache(maxsize=1024) def ensure_path(path): - """Given a path (either a string or a Path object), expand variables and return a Path object. + """ + Given a path (either a string or a Path object), expand variables and return a Path object. :param path: A string or a :class:`~pathlib.Path` object. :type path: str or :class:`~pathlib.Path` @@ -90,13 +101,9 @@ def ensure_path(path): """ if isinstance(path, vistir.compat.Path): - path = path.as_posix() + return path path = vistir.compat.Path(os.path.expandvars(path)) - try: - path = path.resolve() - except OSError: - path = path.absolute() - return path + return path.absolute() def _filter_none(k, v): @@ -105,16 +112,61 @@ def _filter_none(k, v): return False +# TODO: Reimplement in vistir +def normalize_path(path): + return os.path.normpath(os.path.normcase( + os.path.abspath(os.path.expandvars(os.path.expanduser(str(path)))) + )) + + +@lru_cache(maxsize=1024) def filter_pythons(path): """Return all valid pythons in a given path""" if not isinstance(path, vistir.compat.Path): path = vistir.compat.Path(str(path)) if not path.is_dir(): return path if path_is_python(path) else None - return filter(lambda x: path_is_python(x), path.iterdir()) + return filter(path_is_python, path.iterdir()) +# TODO: Port to vistir def unnest(item): - if isinstance(next((i for i in item), None), (list, tuple)): - return chain(*filter(None, item)) - return chain(filter(None, item)) + if isinstance(item, Iterable) and not isinstance(item, six.string_types): + item, target = itertools.tee(item, 2) + else: + target = item + for el in target: + if isinstance(el, Iterable) and not isinstance(el, six.string_types): + el, el_copy = itertools.tee(el, 2) + for sub in unnest(el_copy): + yield sub + else: + yield el + + +def parse_pyenv_version_order(filename="version"): + version_order_file = normalize_path(os.path.join(PYENV_ROOT, filename)) + if os.path.exists(version_order_file) and os.path.isfile(version_order_file): + with io.open(version_order_file, encoding="utf-8") as fh: + contents = fh.read() + version_order = [v for v in contents.splitlines()] + return version_order + + +def parse_asdf_version_order(filename=".tool-versions"): + version_order_file = normalize_path(os.path.join("~", filename)) + if os.path.exists(version_order_file) and os.path.isfile(version_order_file): + with io.open(version_order_file, encoding="utf-8") as fh: + contents = fh.read() + python_section = next(iter( + line for line in contents.splitlines() if line.startswith("python") + ), None) + if python_section: + python_key, _, versions = python_section.partition(" ") + if versions: + return versions.split() + + +# TODO: Reimplement in vistir +def is_in_path(path, parent): + return normalize_path(str(path)).startswith(normalize_path(str(parent))) diff --git a/pipenv/vendor/pytoml/LICENSE b/pipenv/vendor/pytoml/LICENSE deleted file mode 100644 index 9739fc67..00000000 --- a/pipenv/vendor/pytoml/LICENSE +++ /dev/null @@ -1,16 +0,0 @@ -No-notice MIT License - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/pipenv/vendor/pytoml/__init__.py b/pipenv/vendor/pytoml/__init__.py deleted file mode 100644 index 8dc73155..00000000 --- a/pipenv/vendor/pytoml/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .core import TomlError -from .parser import load, loads -from .writer import dump, dumps diff --git a/pipenv/vendor/pytoml/core.py b/pipenv/vendor/pytoml/core.py deleted file mode 100644 index c182734e..00000000 --- a/pipenv/vendor/pytoml/core.py +++ /dev/null @@ -1,13 +0,0 @@ -class TomlError(RuntimeError): - def __init__(self, message, line, col, filename): - RuntimeError.__init__(self, message, line, col, filename) - self.message = message - self.line = line - self.col = col - self.filename = filename - - def __str__(self): - return '{}({}, {}): {}'.format(self.filename, self.line, self.col, self.message) - - def __repr__(self): - return 'TomlError({!r}, {!r}, {!r}, {!r})'.format(self.message, self.line, self.col, self.filename) diff --git a/pipenv/vendor/pytoml/parser.py b/pipenv/vendor/pytoml/parser.py deleted file mode 100644 index 9f94e923..00000000 --- a/pipenv/vendor/pytoml/parser.py +++ /dev/null @@ -1,374 +0,0 @@ -import string, re, sys, datetime -from .core import TomlError - -if sys.version_info[0] == 2: - _chr = unichr -else: - _chr = chr - -def load(fin, translate=lambda t, x, v: v, object_pairs_hook=dict): - return loads(fin.read(), translate=translate, object_pairs_hook=object_pairs_hook, filename=getattr(fin, 'name', repr(fin))) - -def loads(s, filename='', translate=lambda t, x, v: v, object_pairs_hook=dict): - if isinstance(s, bytes): - s = s.decode('utf-8') - - s = s.replace('\r\n', '\n') - - root = object_pairs_hook() - tables = object_pairs_hook() - scope = root - - src = _Source(s, filename=filename) - ast = _p_toml(src, object_pairs_hook=object_pairs_hook) - - def error(msg): - raise TomlError(msg, pos[0], pos[1], filename) - - def process_value(v, object_pairs_hook): - kind, text, value, pos = v - if kind == 'str' and value.startswith('\n'): - value = value[1:] - if kind == 'array': - if value and any(k != value[0][0] for k, t, v, p in value[1:]): - error('array-type-mismatch') - value = [process_value(item, object_pairs_hook=object_pairs_hook) for item in value] - elif kind == 'table': - value = object_pairs_hook([(k, process_value(value[k], object_pairs_hook=object_pairs_hook)) for k in value]) - return translate(kind, text, value) - - for kind, value, pos in ast: - if kind == 'kv': - k, v = value - if k in scope: - error('duplicate_keys. Key "{0}" was used more than once.'.format(k)) - scope[k] = process_value(v, object_pairs_hook=object_pairs_hook) - else: - is_table_array = (kind == 'table_array') - cur = tables - for name in value[:-1]: - if isinstance(cur.get(name), list): - d, cur = cur[name][-1] - else: - d, cur = cur.setdefault(name, (None, object_pairs_hook())) - - scope = object_pairs_hook() - name = value[-1] - if name not in cur: - if is_table_array: - cur[name] = [(scope, object_pairs_hook())] - else: - cur[name] = (scope, object_pairs_hook()) - elif isinstance(cur[name], list): - if not is_table_array: - error('table_type_mismatch') - cur[name].append((scope, object_pairs_hook())) - else: - if is_table_array: - error('table_type_mismatch') - old_scope, next_table = cur[name] - if old_scope is not None: - error('duplicate_tables') - cur[name] = (scope, next_table) - - def merge_tables(scope, tables): - if scope is None: - scope = object_pairs_hook() - for k in tables: - if k in scope: - error('key_table_conflict') - v = tables[k] - if isinstance(v, list): - scope[k] = [merge_tables(sc, tbl) for sc, tbl in v] - else: - scope[k] = merge_tables(v[0], v[1]) - return scope - - return merge_tables(root, tables) - -class _Source: - def __init__(self, s, filename=None): - self.s = s - self._pos = (1, 1) - self._last = None - self._filename = filename - self.backtrack_stack = [] - - def last(self): - return self._last - - def pos(self): - return self._pos - - def fail(self): - return self._expect(None) - - def consume_dot(self): - if self.s: - self._last = self.s[0] - self.s = self[1:] - self._advance(self._last) - return self._last - return None - - def expect_dot(self): - return self._expect(self.consume_dot()) - - def consume_eof(self): - if not self.s: - self._last = '' - return True - return False - - def expect_eof(self): - return self._expect(self.consume_eof()) - - def consume(self, s): - if self.s.startswith(s): - self.s = self.s[len(s):] - self._last = s - self._advance(s) - return True - return False - - def expect(self, s): - return self._expect(self.consume(s)) - - def consume_re(self, re): - m = re.match(self.s) - if m: - self.s = self.s[len(m.group(0)):] - self._last = m - self._advance(m.group(0)) - return m - return None - - def expect_re(self, re): - return self._expect(self.consume_re(re)) - - def __enter__(self): - self.backtrack_stack.append((self.s, self._pos)) - - def __exit__(self, type, value, traceback): - if type is None: - self.backtrack_stack.pop() - else: - self.s, self._pos = self.backtrack_stack.pop() - return type == TomlError - - def commit(self): - self.backtrack_stack[-1] = (self.s, self._pos) - - def _expect(self, r): - if not r: - raise TomlError('msg', self._pos[0], self._pos[1], self._filename) - return r - - def _advance(self, s): - suffix_pos = s.rfind('\n') - if suffix_pos == -1: - self._pos = (self._pos[0], self._pos[1] + len(s)) - else: - self._pos = (self._pos[0] + s.count('\n'), len(s) - suffix_pos) - -_ews_re = re.compile(r'(?:[ \t]|#[^\n]*\n|#[^\n]*\Z|\n)*') -def _p_ews(s): - s.expect_re(_ews_re) - -_ws_re = re.compile(r'[ \t]*') -def _p_ws(s): - s.expect_re(_ws_re) - -_escapes = { 'b': '\b', 'n': '\n', 'r': '\r', 't': '\t', '"': '"', '\'': '\'', - '\\': '\\', '/': '/', 'f': '\f' } - -_basicstr_re = re.compile(r'[^"\\\000-\037]*') -_short_uni_re = re.compile(r'u([0-9a-fA-F]{4})') -_long_uni_re = re.compile(r'U([0-9a-fA-F]{8})') -_escapes_re = re.compile('[bnrt"\'\\\\/f]') -_newline_esc_re = re.compile('\n[ \t\n]*') -def _p_basicstr_content(s, content=_basicstr_re): - res = [] - while True: - res.append(s.expect_re(content).group(0)) - if not s.consume('\\'): - break - if s.consume_re(_newline_esc_re): - pass - elif s.consume_re(_short_uni_re) or s.consume_re(_long_uni_re): - res.append(_chr(int(s.last().group(1), 16))) - else: - s.expect_re(_escapes_re) - res.append(_escapes[s.last().group(0)]) - return ''.join(res) - -_key_re = re.compile(r'[0-9a-zA-Z-_]+') -def _p_key(s): - with s: - s.expect('"') - r = _p_basicstr_content(s, _basicstr_re) - s.expect('"') - return r - if s.consume('\''): - if s.consume('\'\''): - r = s.expect_re(_litstr_ml_re).group(0) - s.expect('\'\'\'') - else: - r = s.expect_re(_litstr_re).group(0) - s.expect('\'') - return r - return s.expect_re(_key_re).group(0) - -_float_re = re.compile(r'[+-]?(?:0|[1-9](?:_?\d)*)(?:\.\d(?:_?\d)*)?(?:[eE][+-]?(?:\d(?:_?\d)*))?') -_datetime_re = re.compile(r'(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(\.\d+)?(?:Z|([+-]\d{2}):(\d{2}))') - -_basicstr_ml_re = re.compile(r'(?:(?:|"|"")[^"\\\000-\011\013-\037])*') -_litstr_re = re.compile(r"[^'\000\010\012-\037]*") -_litstr_ml_re = re.compile(r"(?:(?:|'|'')(?:[^'\000-\010\013-\037]))*") -def _p_value(s, object_pairs_hook): - pos = s.pos() - - if s.consume('true'): - return 'bool', s.last(), True, pos - if s.consume('false'): - return 'bool', s.last(), False, pos - - if s.consume('"'): - if s.consume('""'): - r = _p_basicstr_content(s, _basicstr_ml_re) - s.expect('"""') - else: - r = _p_basicstr_content(s, _basicstr_re) - s.expect('"') - return 'str', r, r, pos - - if s.consume('\''): - if s.consume('\'\''): - r = s.expect_re(_litstr_ml_re).group(0) - s.expect('\'\'\'') - else: - r = s.expect_re(_litstr_re).group(0) - s.expect('\'') - return 'str', r, r, pos - - if s.consume_re(_datetime_re): - m = s.last() - s0 = m.group(0) - r = map(int, m.groups()[:6]) - if m.group(7): - micro = float(m.group(7)) - else: - micro = 0 - - if m.group(8): - g = int(m.group(8), 10) * 60 + int(m.group(9), 10) - tz = _TimeZone(datetime.timedelta(0, g * 60)) - else: - tz = _TimeZone(datetime.timedelta(0, 0)) - - y, m, d, H, M, S = r - dt = datetime.datetime(y, m, d, H, M, S, int(micro * 1000000), tz) - return 'datetime', s0, dt, pos - - if s.consume_re(_float_re): - m = s.last().group(0) - r = m.replace('_','') - if '.' in m or 'e' in m or 'E' in m: - return 'float', m, float(r), pos - else: - return 'int', m, int(r, 10), pos - - if s.consume('['): - items = [] - with s: - while True: - _p_ews(s) - items.append(_p_value(s, object_pairs_hook=object_pairs_hook)) - s.commit() - _p_ews(s) - s.expect(',') - s.commit() - _p_ews(s) - s.expect(']') - return 'array', None, items, pos - - if s.consume('{'): - _p_ws(s) - items = object_pairs_hook() - if not s.consume('}'): - k = _p_key(s) - _p_ws(s) - s.expect('=') - _p_ws(s) - items[k] = _p_value(s, object_pairs_hook=object_pairs_hook) - _p_ws(s) - while s.consume(','): - _p_ws(s) - k = _p_key(s) - _p_ws(s) - s.expect('=') - _p_ws(s) - items[k] = _p_value(s, object_pairs_hook=object_pairs_hook) - _p_ws(s) - s.expect('}') - return 'table', None, items, pos - - s.fail() - -def _p_stmt(s, object_pairs_hook): - pos = s.pos() - if s.consume( '['): - is_array = s.consume('[') - _p_ws(s) - keys = [_p_key(s)] - _p_ws(s) - while s.consume('.'): - _p_ws(s) - keys.append(_p_key(s)) - _p_ws(s) - s.expect(']') - if is_array: - s.expect(']') - return 'table_array' if is_array else 'table', keys, pos - - key = _p_key(s) - _p_ws(s) - s.expect('=') - _p_ws(s) - value = _p_value(s, object_pairs_hook=object_pairs_hook) - return 'kv', (key, value), pos - -_stmtsep_re = re.compile(r'(?:[ \t]*(?:#[^\n]*)?\n)+[ \t]*') -def _p_toml(s, object_pairs_hook): - stmts = [] - _p_ews(s) - with s: - stmts.append(_p_stmt(s, object_pairs_hook=object_pairs_hook)) - while True: - s.commit() - s.expect_re(_stmtsep_re) - stmts.append(_p_stmt(s, object_pairs_hook=object_pairs_hook)) - _p_ews(s) - s.expect_eof() - return stmts - -class _TimeZone(datetime.tzinfo): - def __init__(self, offset): - self._offset = offset - - def utcoffset(self, dt): - return self._offset - - def dst(self, dt): - return None - - def tzname(self, dt): - m = self._offset.total_seconds() // 60 - if m < 0: - res = '-' - m = -m - else: - res = '+' - h = m // 60 - m = m - h * 60 - return '{}{:.02}{:.02}'.format(res, h, m) diff --git a/pipenv/vendor/pytoml/writer.py b/pipenv/vendor/pytoml/writer.py deleted file mode 100644 index 6eaf5d76..00000000 --- a/pipenv/vendor/pytoml/writer.py +++ /dev/null @@ -1,127 +0,0 @@ -from __future__ import unicode_literals -import io, datetime, math, sys - -if sys.version_info[0] == 3: - long = int - unicode = str - - -def dumps(obj, sort_keys=False): - fout = io.StringIO() - dump(obj, fout, sort_keys=sort_keys) - return fout.getvalue() - - -_escapes = {'\n': 'n', '\r': 'r', '\\': '\\', '\t': 't', '\b': 'b', '\f': 'f', '"': '"'} - - -def _escape_string(s): - res = [] - start = 0 - - def flush(): - if start != i: - res.append(s[start:i]) - return i + 1 - - i = 0 - while i < len(s): - c = s[i] - if c in '"\\\n\r\t\b\f': - start = flush() - res.append('\\' + _escapes[c]) - elif ord(c) < 0x20: - start = flush() - res.append('\\u%04x' % ord(c)) - i += 1 - - flush() - return '"' + ''.join(res) + '"' - - -def _escape_id(s): - if any(not c.isalnum() and c not in '-_' for c in s): - return _escape_string(s) - return s - - -def _format_list(v): - return '[{0}]'.format(', '.join(_format_value(obj) for obj in v)) - -# Formula from: -# https://docs.python.org/2/library/datetime.html#datetime.timedelta.total_seconds -# Once support for py26 is dropped, this can be replaced by td.total_seconds() -def _total_seconds(td): - return ((td.microseconds - + (td.seconds + td.days * 24 * 3600) * 10**6) / 10.0**6) - -def _format_value(v): - if isinstance(v, bool): - return 'true' if v else 'false' - if isinstance(v, int) or isinstance(v, long): - return unicode(v) - if isinstance(v, float): - if math.isnan(v) or math.isinf(v): - raise ValueError("{0} is not a valid TOML value".format(v)) - else: - return repr(v) - elif isinstance(v, unicode) or isinstance(v, bytes): - return _escape_string(v) - elif isinstance(v, datetime.datetime): - offs = v.utcoffset() - offs = _total_seconds(offs) // 60 if offs is not None else 0 - - if offs == 0: - suffix = 'Z' - else: - if offs > 0: - suffix = '+' - else: - suffix = '-' - offs = -offs - suffix = '{0}{1:.02}{2:.02}'.format(suffix, offs // 60, offs % 60) - - if v.microsecond: - return v.strftime('%Y-%m-%dT%H:%M:%S.%f') + suffix - else: - return v.strftime('%Y-%m-%dT%H:%M:%S') + suffix - elif isinstance(v, list): - return _format_list(v) - else: - raise RuntimeError(v) - - -def dump(obj, fout, sort_keys=False): - tables = [((), obj, False)] - - while tables: - name, table, is_array = tables.pop() - if name: - section_name = '.'.join(_escape_id(c) for c in name) - if is_array: - fout.write('[[{0}]]\n'.format(section_name)) - else: - fout.write('[{0}]\n'.format(section_name)) - - table_keys = sorted(table.keys()) if sort_keys else table.keys() - new_tables = [] - has_kv = False - for k in table_keys: - v = table[k] - if isinstance(v, dict): - new_tables.append((name + (k,), v, False)) - elif isinstance(v, list) and v and all(isinstance(o, dict) for o in v): - new_tables.extend((name + (k,), d, True) for d in v) - elif v is None: - # based on mojombo's comment: https://github.com/toml-lang/toml/issues/146#issuecomment-25019344 - fout.write( - '#{} = null # To use: uncomment and replace null with value\n'.format(_escape_id(k))) - has_kv = True - else: - fout.write('{0} = {1}\n'.format(_escape_id(k), _format_value(v))) - has_kv = True - - tables.extend(reversed(new_tables)) - - if (name or has_kv) and tables: - fout.write('\n') diff --git a/pipenv/vendor/requests/LICENSE b/pipenv/vendor/requests/LICENSE index 2e68b82e..841c6023 100644 --- a/pipenv/vendor/requests/LICENSE +++ b/pipenv/vendor/requests/LICENSE @@ -4,7 +4,7 @@ Copyright 2018 Kenneth Reitz you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, diff --git a/pipenv/vendor/requests/__init__.py b/pipenv/vendor/requests/__init__.py index a5b3c9c3..bc168ee5 100644 --- a/pipenv/vendor/requests/__init__.py +++ b/pipenv/vendor/requests/__init__.py @@ -22,7 +22,7 @@ usage: ... or POST: >>> payload = dict(key1='value1', key2='value2') - >>> r = requests.post('http://httpbin.org/post', data=payload) + >>> r = requests.post('https://httpbin.org/post', data=payload) >>> print(r.text) { ... @@ -57,10 +57,10 @@ def check_compatibility(urllib3_version, chardet_version): # Check urllib3 for compatibility. major, minor, patch = urllib3_version # noqa: F811 major, minor, patch = int(major), int(minor), int(patch) - # urllib3 >= 1.21.1, <= 1.23 + # urllib3 >= 1.21.1, <= 1.24 assert major == 1 assert minor >= 21 - assert minor <= 23 + assert minor <= 24 # Check chardet for compatibility. major, minor, patch = chardet_version.split('.')[:3] @@ -79,14 +79,14 @@ def _check_cryptography(cryptography_version): return if cryptography_version < [1, 3, 4]: - warning = 'Old version of cryptography ({0}) may cause slowdown.'.format(cryptography_version) + warning = 'Old version of cryptography ({}) may cause slowdown.'.format(cryptography_version) warnings.warn(warning, RequestsDependencyWarning) # Check imported dependencies for compatibility. try: check_compatibility(urllib3.__version__, chardet.__version__) except (AssertionError, ValueError): - warnings.warn("urllib3 ({0}) or chardet ({1}) doesn't match a supported " + warnings.warn("urllib3 ({}) or chardet ({}) doesn't match a supported " "version!".format(urllib3.__version__, chardet.__version__), RequestsDependencyWarning) @@ -123,12 +123,7 @@ from .exceptions import ( # Set default logging handler to avoid "No handler found" warnings. import logging -try: # Python 2.7+ - from logging import NullHandler -except ImportError: - class NullHandler(logging.Handler): - def emit(self, record): - pass +from logging import NullHandler logging.getLogger(__name__).addHandler(NullHandler()) diff --git a/pipenv/vendor/requests/__version__.py b/pipenv/vendor/requests/__version__.py index ef61ec0f..803773a0 100644 --- a/pipenv/vendor/requests/__version__.py +++ b/pipenv/vendor/requests/__version__.py @@ -5,8 +5,8 @@ __title__ = 'requests' __description__ = 'Python HTTP for Humans.' __url__ = 'http://python-requests.org' -__version__ = '2.19.1' -__build__ = 0x021901 +__version__ = '2.20.1' +__build__ = 0x022001 __author__ = 'Kenneth Reitz' __author_email__ = 'me@kennethreitz.org' __license__ = 'Apache 2.0' diff --git a/pipenv/vendor/requests/adapters.py b/pipenv/vendor/requests/adapters.py index a4b02842..fa4d9b3c 100644 --- a/pipenv/vendor/requests/adapters.py +++ b/pipenv/vendor/requests/adapters.py @@ -26,6 +26,7 @@ from urllib3.exceptions import ProtocolError from urllib3.exceptions import ReadTimeoutError from urllib3.exceptions import SSLError as _SSLError from urllib3.exceptions import ResponseError +from urllib3.exceptions import LocationValueError from .models import Response from .compat import urlparse, basestring @@ -35,7 +36,8 @@ from .utils import (DEFAULT_CA_BUNDLE_PATH, extract_zipped_paths, from .structures import CaseInsensitiveDict from .cookies import extract_cookies_to_jar from .exceptions import (ConnectionError, ConnectTimeout, ReadTimeout, SSLError, - ProxyError, RetryError, InvalidSchema, InvalidProxyURL) + ProxyError, RetryError, InvalidSchema, InvalidProxyURL, + InvalidURL) from .auth import _basic_auth_str try: @@ -127,8 +129,7 @@ class HTTPAdapter(BaseAdapter): self.init_poolmanager(pool_connections, pool_maxsize, block=pool_block) def __getstate__(self): - return dict((attr, getattr(self, attr, None)) for attr in - self.__attrs__) + return {attr: getattr(self, attr, None) for attr in self.__attrs__} def __setstate__(self, state): # Can't handle by adding 'proxy_manager' to self.__attrs__ because @@ -224,7 +225,7 @@ class HTTPAdapter(BaseAdapter): if not cert_loc or not os.path.exists(cert_loc): raise IOError("Could not find a suitable TLS CA certificate bundle, " - "invalid path: {0}".format(cert_loc)) + "invalid path: {}".format(cert_loc)) conn.cert_reqs = 'CERT_REQUIRED' @@ -246,10 +247,10 @@ class HTTPAdapter(BaseAdapter): conn.key_file = None if conn.cert_file and not os.path.exists(conn.cert_file): raise IOError("Could not find the TLS certificate file, " - "invalid path: {0}".format(conn.cert_file)) + "invalid path: {}".format(conn.cert_file)) if conn.key_file and not os.path.exists(conn.key_file): raise IOError("Could not find the TLS key file, " - "invalid path: {0}".format(conn.key_file)) + "invalid path: {}".format(conn.key_file)) def build_response(self, req, resp): """Builds a :class:`Response ` object from a urllib3 @@ -378,7 +379,7 @@ class HTTPAdapter(BaseAdapter): when subclassing the :class:`HTTPAdapter `. - :param proxies: The url of the proxy being used for this request. + :param proxy: The url of the proxy being used for this request. :rtype: dict """ headers = {} @@ -407,7 +408,10 @@ class HTTPAdapter(BaseAdapter): :rtype: requests.Response """ - conn = self.get_connection(request.url, proxies) + try: + conn = self.get_connection(request.url, proxies) + except LocationValueError as e: + raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) @@ -421,7 +425,7 @@ class HTTPAdapter(BaseAdapter): timeout = TimeoutSauce(connect=connect, read=read) except ValueError as e: # this may raise a string formatting error. - err = ("Invalid timeout {0}. Pass a (connect, read) " + err = ("Invalid timeout {}. Pass a (connect, read) " "timeout tuple, or a single float to set " "both timeouts to the same value".format(timeout)) raise ValueError(err) @@ -471,11 +475,10 @@ class HTTPAdapter(BaseAdapter): # Receive the response from the server try: - # For Python 2.7+ versions, use buffering of HTTP - # responses + # For Python 2.7, use buffering of HTTP responses r = low_conn.getresponse(buffering=True) except TypeError: - # For compatibility with Python 2.6 versions and back + # For compatibility with Python 3.3+ r = low_conn.getresponse() resp = HTTPResponse.from_httplib( diff --git a/pipenv/vendor/requests/api.py b/pipenv/vendor/requests/api.py index a2cc84d7..abada96d 100644 --- a/pipenv/vendor/requests/api.py +++ b/pipenv/vendor/requests/api.py @@ -18,8 +18,10 @@ def request(method, url, **kwargs): :param method: method for the new :class:`Request` object. :param url: URL for the new :class:`Request` object. - :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. - :param data: (optional) Dictionary or list of tuples ``[(key, value)]`` (will be form-encoded), bytes, or file-like object to send in the body of the :class:`Request`. + :param params: (optional) Dictionary, list of tuples or bytes to send + in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) A JSON serializable Python object to send in the body of the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. @@ -47,7 +49,7 @@ def request(method, url, **kwargs): Usage:: >>> import requests - >>> req = requests.request('GET', 'http://httpbin.org/get') + >>> req = requests.request('GET', 'https://httpbin.org/get') """ @@ -62,7 +64,8 @@ def get(url, params=None, **kwargs): r"""Sends a GET request. :param url: URL for the new :class:`Request` object. - :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. + :param params: (optional) Dictionary, list of tuples or bytes to send + in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :return: :class:`Response ` object :rtype: requests.Response @@ -102,7 +105,8 @@ def post(url, data=None, json=None, **kwargs): r"""Sends a POST request. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary (will be form-encoded), bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) json data to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :return: :class:`Response ` object @@ -116,7 +120,8 @@ def put(url, data=None, **kwargs): r"""Sends a PUT request. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary (will be form-encoded), bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) json data to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :return: :class:`Response ` object @@ -130,7 +135,8 @@ def patch(url, data=None, **kwargs): r"""Sends a PATCH request. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary (will be form-encoded), bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) json data to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :return: :class:`Response ` object diff --git a/pipenv/vendor/requests/auth.py b/pipenv/vendor/requests/auth.py index 4ae45947..bdde51c7 100644 --- a/pipenv/vendor/requests/auth.py +++ b/pipenv/vendor/requests/auth.py @@ -38,7 +38,7 @@ def _basic_auth_str(username, password): if not isinstance(username, basestring): warnings.warn( "Non-string usernames will no longer be supported in Requests " - "3.0.0. Please convert the object you've passed in ({0!r}) to " + "3.0.0. Please convert the object you've passed in ({!r}) to " "a string or bytes object in the near future to avoid " "problems.".format(username), category=DeprecationWarning, @@ -48,7 +48,7 @@ def _basic_auth_str(username, password): if not isinstance(password, basestring): warnings.warn( "Non-string passwords will no longer be supported in Requests " - "3.0.0. Please convert the object you've passed in ({0!r}) to " + "3.0.0. Please convert the object you've passed in ({!r}) to " "a string or bytes object in the near future to avoid " "problems.".format(password), category=DeprecationWarning, diff --git a/pipenv/vendor/requests/compat.py b/pipenv/vendor/requests/compat.py index 6b9c6fac..c44b35ef 100644 --- a/pipenv/vendor/requests/compat.py +++ b/pipenv/vendor/requests/compat.py @@ -43,9 +43,8 @@ if is_py2: import cookielib from Cookie import Morsel from StringIO import StringIO - from collections import Callable, Mapping, MutableMapping + from collections import Callable, Mapping, MutableMapping, OrderedDict - from urllib3.packages.ordered_dict import OrderedDict builtin_str = str bytes = str diff --git a/pipenv/vendor/requests/cookies.py b/pipenv/vendor/requests/cookies.py index 50883a84..56fccd9c 100644 --- a/pipenv/vendor/requests/cookies.py +++ b/pipenv/vendor/requests/cookies.py @@ -444,20 +444,21 @@ def create_cookie(name, value, **kwargs): By default, the pair of `name` and `value` will be set for the domain '' and sent on every request (this is sometimes called a "supercookie"). """ - result = dict( - version=0, - name=name, - value=value, - port=None, - domain='', - path='/', - secure=False, - expires=None, - discard=True, - comment=None, - comment_url=None, - rest={'HttpOnly': None}, - rfc2109=False,) + result = { + 'version': 0, + 'name': name, + 'value': value, + 'port': None, + 'domain': '', + 'path': '/', + 'secure': False, + 'expires': None, + 'discard': True, + 'comment': None, + 'comment_url': None, + 'rest': {'HttpOnly': None}, + 'rfc2109': False, + } badargs = set(kwargs) - set(result) if badargs: @@ -511,6 +512,7 @@ def cookiejar_from_dict(cookie_dict, cookiejar=None, overwrite=True): :param cookiejar: (optional) A cookiejar to add the cookies to. :param overwrite: (optional) If False, will not replace cookies already in the jar with new ones. + :rtype: CookieJar """ if cookiejar is None: cookiejar = RequestsCookieJar() @@ -529,6 +531,7 @@ def merge_cookies(cookiejar, cookies): :param cookiejar: CookieJar object to add the cookies to. :param cookies: Dictionary or CookieJar object to be added. + :rtype: CookieJar """ if not isinstance(cookiejar, cookielib.CookieJar): raise ValueError('You can only merge into CookieJar') diff --git a/pipenv/vendor/requests/help.py b/pipenv/vendor/requests/help.py index 06e06b2a..e53d35ef 100644 --- a/pipenv/vendor/requests/help.py +++ b/pipenv/vendor/requests/help.py @@ -89,8 +89,7 @@ def info(): 'version': getattr(idna, '__version__', ''), } - # OPENSSL_VERSION_NUMBER doesn't exist in the Python 2.6 ssl module. - system_ssl = getattr(ssl, 'OPENSSL_VERSION_NUMBER', None) + system_ssl = ssl.OPENSSL_VERSION_NUMBER system_ssl_info = { 'version': '%x' % system_ssl if system_ssl is not None else '' } diff --git a/pipenv/vendor/requests/hooks.py b/pipenv/vendor/requests/hooks.py index 32b32de7..7a51f212 100644 --- a/pipenv/vendor/requests/hooks.py +++ b/pipenv/vendor/requests/hooks.py @@ -15,14 +15,14 @@ HOOKS = ['response'] def default_hooks(): - return dict((event, []) for event in HOOKS) + return {event: [] for event in HOOKS} # TODO: response is the only one def dispatch_hook(key, hooks, hook_data, **kwargs): """Dispatches a hook dictionary on a given piece of data.""" - hooks = hooks or dict() + hooks = hooks or {} hooks = hooks.get(key) if hooks: if hasattr(hooks, '__call__'): diff --git a/pipenv/vendor/requests/models.py b/pipenv/vendor/requests/models.py index 3d0e1f42..3dded57e 100644 --- a/pipenv/vendor/requests/models.py +++ b/pipenv/vendor/requests/models.py @@ -204,9 +204,13 @@ class Request(RequestHooksMixin): :param url: URL to send. :param headers: dictionary of headers to send. :param files: dictionary of {filename: fileobject} files to multipart upload. - :param data: the body to attach to the request. If a dictionary is provided, form-encoding will take place. + :param data: the body to attach to the request. If a dictionary or + list of tuples ``[(key, value)]`` is provided, form-encoding will + take place. :param json: json for the body to attach to the request (if files or data is not specified). - :param params: dictionary of URL parameters to append to the URL. + :param params: URL parameters to append to the URL. If a dictionary or + list of tuples ``[(key, value)]`` is provided, form-encoding will + take place. :param auth: Auth handler or (user, pass) tuple. :param cookies: dictionary or CookieJar of cookies to attach to this request. :param hooks: dictionary of callback hooks, for internal usage. @@ -214,7 +218,7 @@ class Request(RequestHooksMixin): Usage:: >>> import requests - >>> req = requests.Request('GET', 'http://httpbin.org/get') + >>> req = requests.Request('GET', 'https://httpbin.org/get') >>> req.prepare() """ @@ -274,7 +278,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): Usage:: >>> import requests - >>> req = requests.Request('GET', 'http://httpbin.org/get') + >>> req = requests.Request('GET', 'https://httpbin.org/get') >>> r = req.prepare() @@ -648,10 +652,7 @@ class Response(object): if not self._content_consumed: self.content - return dict( - (attr, getattr(self, attr, None)) - for attr in self.__attrs__ - ) + return {attr: getattr(self, attr, None) for attr in self.__attrs__} def __setstate__(self, state): for name, value in state.items(): diff --git a/pipenv/vendor/requests/sessions.py b/pipenv/vendor/requests/sessions.py index ba135268..d73d700f 100644 --- a/pipenv/vendor/requests/sessions.py +++ b/pipenv/vendor/requests/sessions.py @@ -19,7 +19,7 @@ from .cookies import ( from .models import Request, PreparedRequest, DEFAULT_REDIRECT_LIMIT from .hooks import default_hooks, dispatch_hook from ._internal_utils import to_native_string -from .utils import to_key_val_list, default_headers +from .utils import to_key_val_list, default_headers, DEFAULT_PORTS from .exceptions import ( TooManyRedirects, InvalidSchema, ChunkedEncodingError, ContentDecodingError) @@ -115,6 +115,31 @@ class SessionRedirectMixin(object): return to_native_string(location, 'utf8') return None + def should_strip_auth(self, old_url, new_url): + """Decide whether Authorization header should be removed when redirecting""" + old_parsed = urlparse(old_url) + new_parsed = urlparse(new_url) + if old_parsed.hostname != new_parsed.hostname: + return True + # Special case: allow http -> https redirect when using the standard + # ports. This isn't specified by RFC 7235, but is kept to avoid + # breaking backwards compatibility with older versions of requests + # that allowed any redirects on the same host. + if (old_parsed.scheme == 'http' and old_parsed.port in (80, None) + and new_parsed.scheme == 'https' and new_parsed.port in (443, None)): + return False + + # Handle default port usage corresponding to scheme. + changed_port = old_parsed.port != new_parsed.port + changed_scheme = old_parsed.scheme != new_parsed.scheme + default_port = (DEFAULT_PORTS.get(old_parsed.scheme, None), None) + if (not changed_scheme and old_parsed.port in default_port + and new_parsed.port in default_port): + return False + + # Standard case: root URI must match + return changed_port or changed_scheme + def resolve_redirects(self, resp, req, stream=False, timeout=None, verify=True, cert=None, proxies=None, yield_requests=False, **adapter_kwargs): """Receives a Response. Returns a generator of Responses or Requests.""" @@ -236,14 +261,10 @@ class SessionRedirectMixin(object): headers = prepared_request.headers url = prepared_request.url - if 'Authorization' in headers: + if 'Authorization' in headers and self.should_strip_auth(response.request.url, url): # If we get redirected to a new host, we should strip out any # authentication headers. - original_parsed = urlparse(response.request.url) - redirect_parsed = urlparse(url) - - if (original_parsed.hostname != redirect_parsed.hostname): - del headers['Authorization'] + del headers['Authorization'] # .netrc might have more auth for us on our new host. new_auth = get_netrc_auth(url) if self.trust_env else None @@ -299,7 +320,7 @@ class SessionRedirectMixin(object): """ method = prepared_request.method - # http://tools.ietf.org/html/rfc7231#section-6.4.4 + # https://tools.ietf.org/html/rfc7231#section-6.4.4 if response.status_code == codes.see_other and method != 'HEAD': method = 'GET' @@ -325,13 +346,13 @@ class Session(SessionRedirectMixin): >>> import requests >>> s = requests.Session() - >>> s.get('http://httpbin.org/get') + >>> s.get('https://httpbin.org/get') Or as a context manager:: >>> with requests.Session() as s: - >>> s.get('http://httpbin.org/get') + >>> s.get('https://httpbin.org/get') """ @@ -453,8 +474,8 @@ class Session(SessionRedirectMixin): :param url: URL for the new :class:`Request` object. :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. - :param data: (optional) Dictionary, bytes, or file-like object to send - in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) json to send in the body of the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to send with the @@ -550,7 +571,8 @@ class Session(SessionRedirectMixin): r"""Sends a POST request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) json to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :rtype: requests.Response @@ -562,7 +584,8 @@ class Session(SessionRedirectMixin): r"""Sends a PUT request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :rtype: requests.Response """ @@ -573,7 +596,8 @@ class Session(SessionRedirectMixin): r"""Sends a PATCH request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :rtype: requests.Response """ @@ -723,7 +747,7 @@ class Session(SessionRedirectMixin): self.adapters[key] = self.adapters.pop(key) def __getstate__(self): - state = dict((attr, getattr(self, attr, None)) for attr in self.__attrs__) + state = {attr: getattr(self, attr, None) for attr in self.__attrs__} return state def __setstate__(self, state): @@ -735,7 +759,12 @@ def session(): """ Returns a :class:`Session` for context-management. + .. deprecated:: 1.0.0 + + This method has been deprecated since version 1.0.0 and is only kept for + backwards compatibility. New code should use :class:`~requests.sessions.Session` + to create a session. This may be removed at a future date. + :rtype: Session """ - return Session() diff --git a/pipenv/vendor/requests/status_codes.py b/pipenv/vendor/requests/status_codes.py index ff462c6c..813e8c4e 100644 --- a/pipenv/vendor/requests/status_codes.py +++ b/pipenv/vendor/requests/status_codes.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -""" +r""" The ``codes`` object defines a mapping from common names for HTTP statuses to their numerical codes, accessible either as attributes or as dictionary items. diff --git a/pipenv/vendor/requests/utils.py b/pipenv/vendor/requests/utils.py index 431f6be0..8170a8d2 100644 --- a/pipenv/vendor/requests/utils.py +++ b/pipenv/vendor/requests/utils.py @@ -38,6 +38,8 @@ NETRC_FILES = ('.netrc', '_netrc') DEFAULT_CA_BUNDLE_PATH = certs.where() +DEFAULT_PORTS = {'http': 80, 'https': 443} + if sys.platform == 'win32': # provide a proxy_bypass version on Windows without DNS lookups @@ -173,10 +175,10 @@ def get_netrc_auth(url, raise_errors=False): for f in NETRC_FILES: try: - loc = os.path.expanduser('~/{0}'.format(f)) + loc = os.path.expanduser('~/{}'.format(f)) except KeyError: # os.path.expanduser can fail when $HOME is undefined and - # getpwuid fails. See http://bugs.python.org/issue20164 & + # getpwuid fails. See https://bugs.python.org/issue20164 & # https://github.com/requests/requests/issues/1846 return @@ -264,7 +266,7 @@ def from_key_val_list(value): >>> from_key_val_list([('key', 'val')]) OrderedDict([('key', 'val')]) >>> from_key_val_list('string') - ValueError: need more than 1 value to unpack + ValueError: cannot encode objects that are not 2-tuples >>> from_key_val_list({'key': 'val'}) OrderedDict([('key', 'val')]) @@ -466,7 +468,7 @@ def _parse_content_type_header(header): if index_of_equals != -1: key = param[:index_of_equals].strip(items_to_strip) value = param[index_of_equals + 1:].strip(items_to_strip) - params_dict[key] = value + params_dict[key.lower()] = value return content_type, params_dict @@ -706,6 +708,10 @@ def should_bypass_proxies(url, no_proxy): no_proxy = get_proxy('no_proxy') parsed = urlparse(url) + if parsed.hostname is None: + # URLs don't always have hostnames, e.g. file:/// urls. + return True + if no_proxy: # We need to check whether we match here. We need to see if we match # the end of the hostname, both with and without the port. @@ -725,7 +731,7 @@ def should_bypass_proxies(url, no_proxy): else: host_with_port = parsed.hostname if parsed.port: - host_with_port += ':{0}'.format(parsed.port) + host_with_port += ':{}'.format(parsed.port) for host in no_proxy: if parsed.hostname.endswith(host) or host_with_port.endswith(host): @@ -733,13 +739,8 @@ def should_bypass_proxies(url, no_proxy): # to apply the proxies on this URL. return True - # If the system proxy settings indicate that this URL should be bypassed, - # don't proxy. - # The proxy_bypass function is incredibly buggy on OS X in early versions - # of Python 2.6, so allow this call to fail. Only catch the specific - # exceptions we've seen, though: this call failing in other ways can reveal - # legitimate problems. with set_environ('no_proxy', no_proxy_arg): + # parsed.hostname can be `None` in cases such as a file URI. try: bypass = proxy_bypass(parsed.hostname) except (TypeError, socket.gaierror): diff --git a/pipenv/vendor/requirementslib/__init__.py b/pipenv/vendor/requirementslib/__init__.py index 1f3a2fcb..32415c61 100644 --- a/pipenv/vendor/requirementslib/__init__.py +++ b/pipenv/vendor/requirementslib/__init__.py @@ -1,6 +1,16 @@ # -*- coding=utf-8 -*- -__version__ = '1.1.9' +__version__ = '1.3.1' +import logging +import warnings +from vistir.compat import ResourceWarning -from .exceptions import RequirementError -from .models import Requirement, Lockfile, Pipfile +logger = logging.getLogger(__name__) +logger.addHandler(logging.NullHandler()) +warnings.filterwarnings("ignore", category=ResourceWarning) + +from .models.requirements import Requirement +from .models.lockfile import Lockfile +from .models.pipfile import Pipfile + +__all__ = ["Lockfile", "Pipfile", "Requirement"] diff --git a/pipenv/vendor/requirementslib/exceptions.py b/pipenv/vendor/requirementslib/exceptions.py index 82578624..17b884eb 100644 --- a/pipenv/vendor/requirementslib/exceptions.py +++ b/pipenv/vendor/requirementslib/exceptions.py @@ -1,7 +1,12 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import +from __future__ import absolute_import, print_function import errno +import os import six +import sys + + +from vistir.compat import FileNotFoundError if six.PY2: @@ -9,7 +14,79 @@ if six.PY2: def __init__(self, *args, **kwargs): self.errno = errno.EEXIST super(FileExistsError, self).__init__(*args, **kwargs) +else: + from six.moves.builtins import FileExistsError class RequirementError(Exception): pass + + +class MissingParameter(Exception): + def __init__(self, param): + Exception.__init__(self) + print("Missing parameter: %s" % param, file=sys.stderr, flush=True) + + +class FileCorruptException(OSError): + def __init__(self, path, *args, **kwargs): + path = path + backup_path = kwargs.pop("backup_path", None) + if not backup_path and args: + args = reversed(args) + backup_path = args.pop() + if not isinstance(backup_path, six.string_types) or not os.path.exists(os.path.abspath(os.path.dirname(backup_path))): + args.append(backup_path) + backup_path = None + if args: + args = reversed(args) + self.path = path + self.backup_path = backup_path + self.show(self.path, self.backup_path) + OSError.__init__(self, path, *args, **kwargs) + + @classmethod + def show(cls, path, backup_path=None): + print("ERROR: Failed to load file at %s" % path, file=sys.stderr, flush=True) + if backup_path: + msg = "it will be backed up to %s and removed" % backup_path + else: + msg = "it will be removed and replaced." + print("The file is corrupt, %s" % msg, file=sys.stderr, flush=True) + + +class LockfileCorruptException(FileCorruptException): + + @classmethod + def show(cls, path, backup_path=None): + print("ERROR: Failed to load lockfile at %s" % path, file=sys.stderr, flush=True) + if backup_path: + msg = "it will be backed up to %s and removed" % backup_path + else: + msg = "it will be removed and replaced on the next lock." + print("Your lockfile is corrupt, %s" % msg, file=sys.stderr, flush=True) + + +class PipfileCorruptException(FileCorruptException): + + @classmethod + def show(cls, path, backup_path=None): + print("ERROR: Failed to load Pipfile at %s" % path, file=sys.stderr, flush=True) + if backup_path: + msg = "it will be backed up to %s and removed" % backup_path + else: + msg = "it will be removed and replaced on the next lock." + print("Your Pipfile is corrupt, %s" % msg, file=sys.stderr, flush=True) + + +class PipfileNotFound(FileNotFoundError): + def __init__(self, path, *args, **kwargs): + self.errno = errno.ENOENT + self.path = path + self.show(path) + super(PipfileNotFound, self).__init__(*args, **kwargs) + + @classmethod + def show(cls, path): + print("ERROR: The file could not be found: %s" % path, file=sys.stderr, flush=True) + print("Aborting...", file=sys.stderr, flush=True) diff --git a/pipenv/vendor/requirementslib/models/__init__.py b/pipenv/vendor/requirementslib/models/__init__.py index 8d12da45..e819cdd5 100644 --- a/pipenv/vendor/requirementslib/models/__init__.py +++ b/pipenv/vendor/requirementslib/models/__init__.py @@ -1,11 +1,2 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import - - -__all__ = ["Requirement", "Lockfile", "Pipfile", "DependencyResolver"] - - -from .requirements import Requirement -from .lockfile import Lockfile -from .pipfile import Pipfile -from .resolvers import DependencyResolver +# This is intentionally left blank diff --git a/pipenv/vendor/requirementslib/models/baserequirement.py b/pipenv/vendor/requirementslib/models/baserequirement.py deleted file mode 100644 index b97dee40..00000000 --- a/pipenv/vendor/requirementslib/models/baserequirement.py +++ /dev/null @@ -1,37 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import absolute_import -import abc -import attr -import six - - -@six.add_metaclass(abc.ABCMeta) -class BaseRequirement: - @classmethod - def from_line(cls, line): - """Returns a requirement from a requirements.txt or pip-compatible line""" - raise NotImplementedError - - @abc.abstractmethod - def line_part(self): - """Returns the current requirement as a pip-compatible line""" - - @classmethod - def from_pipfile(cls, name, pipfile): - """Returns a requirement from a pipfile entry""" - raise NotImplementedError - - @abc.abstractmethod - def pipfile_part(self): - """Returns the current requirement as a pipfile entry""" - - @classmethod - def attr_fields(cls): - return [field.name for field in attr.fields(cls)] - - @property - def extras_as_pip(self): - if self.extras: - return "[{0}]".format(",".join(self.extras)) - - return "" diff --git a/pipenv/vendor/requirementslib/models/cache.py b/pipenv/vendor/requirementslib/models/cache.py index 16fc4ba8..f1639ea2 100644 --- a/pipenv/vendor/requirementslib/models/cache.py +++ b/pipenv/vendor/requirementslib/models/cache.py @@ -1,25 +1,22 @@ # -*- coding=utf-8 -*- from __future__ import absolute_import, print_function, unicode_literals +import atexit import copy import hashlib import json import os -import six import sys -import requests -import pip_shims import vistir from appdirs import user_cache_dir +from pip_shims.shims import FAVORITE_HASH, SafeFileCache from packaging.requirements import Requirement from .utils import as_tuple, key_from_req, lookup_table, get_pinned_version - -if six.PY2: - from ..exceptions import FileExistsError +from ..exceptions import FileExistsError CACHE_DIR = os.environ.get("PIPENV_CACHE_DIR", user_cache_dir("pipenv")) @@ -189,7 +186,7 @@ class DependencyCache(object): for dep_name in self.cache[name][version_and_extras]) -class HashCache(pip_shims.SafeFileCache): +class HashCache(SafeFileCache): """Caches hashes of PyPI artifacts so we do not need to re-download them. Hashes are only cached when the URL appears to contain a hash in it and the @@ -197,16 +194,21 @@ class HashCache(pip_shims.SafeFileCache): avoid ssues where the location on the server changes. """ def __init__(self, *args, **kwargs): - session = kwargs.pop('session', requests.session()) + session = kwargs.pop("session", None) + if not session: + import requests + session = requests.session() + atexit.register(session.close) cache_dir = kwargs.pop('cache_dir', CACHE_DIR) self.session = session kwargs.setdefault('directory', os.path.join(cache_dir, 'hash-cache')) super(HashCache, self).__init__(*args, **kwargs) def get_hash(self, location): + from pip_shims import VcsSupport # if there is no location hash (i.e., md5 / sha256 / etc) we on't want to store it hash_value = None - vcs = pip_shims.VcsSupport() + vcs = VcsSupport() orig_scheme = location.scheme new_location = copy.deepcopy(location) if orig_scheme in vcs.all_schemes: @@ -223,11 +225,11 @@ class HashCache(pip_shims.SafeFileCache): return hash_value.decode('utf8') def _get_file_hash(self, location): - h = hashlib.new(pip_shims.FAVORITE_HASH) + h = hashlib.new(FAVORITE_HASH) with vistir.contextmanagers.open_file(location, self.session) as fp: for chunk in iter(lambda: fp.read(8096), b""): h.update(chunk) - return ":".join([pip_shims.FAVORITE_HASH, h.hexdigest()]) + return ":".join([FAVORITE_HASH, h.hexdigest()]) class _JSONCache(object): diff --git a/pipenv/vendor/requirementslib/models/dependencies.py b/pipenv/vendor/requirementslib/models/dependencies.py index ae643517..f87fd585 100644 --- a/pipenv/vendor/requirementslib/models/dependencies.py +++ b/pipenv/vendor/requirementslib/models/dependencies.py @@ -1,5 +1,6 @@ # -*- coding=utf-8 -*- +import atexit import contextlib import copy import functools @@ -13,15 +14,13 @@ import requests from first import first from packaging.utils import canonicalize_name -from pip_shims import ( - FormatControl, InstallRequirement, PackageFinder, RequirementPreparer, - RequirementSet, RequirementTracker, Resolver, WheelCache, pip_version -) -from vistir.compat import JSONDecodeError, TemporaryDirectory, fs_str +import pip_shims.shims +from vistir.compat import JSONDecodeError, fs_str, ResourceWarning from vistir.contextmanagers import cd, temp_environ from vistir.misc import partialclass +from vistir.path import create_tracked_tempdir -from ..utils import get_pip_command, prepare_pip_source_args, _ensure_dir +from ..utils import prepare_pip_source_args, _ensure_dir from .cache import CACHE_DIR, DependencyCache from .utils import ( clean_requires_python, fix_requires_python_marker, format_requirement, @@ -34,7 +33,9 @@ PKGS_DOWNLOAD_DIR = fs_str(os.path.join(CACHE_DIR, "pkgs")) WHEEL_DOWNLOAD_DIR = fs_str(os.path.join(CACHE_DIR, "wheels")) DEPENDENCY_CACHE = DependencyCache() -WHEEL_CACHE = WheelCache(CACHE_DIR, FormatControl(set(), set())) +WHEEL_CACHE = pip_shims.shims.WheelCache( + CACHE_DIR, pip_shims.shims.FormatControl(set(), set()) +) def _get_filtered_versions(ireq, versions, prereleases): @@ -52,6 +53,8 @@ def find_all_matches(finder, ireq, pre=False): :return: A list of matching candidates. :rtype: list[:class:`~pip._internal.index.InstallationCandidate`] """ + + candidates = clean_requires_python(finder.find_all_candidates(ireq.name)) versions = {candidate.version for candidate in candidates} allowed_versions = _get_filtered_versions(ireq, versions, pre) @@ -61,6 +64,29 @@ def find_all_matches(finder, ireq, pre=False): return candidates +def get_pip_command(): + # Use pip's parser for pip.conf management and defaults. + # General options (find_links, index_url, extra_index_url, trusted_host, + # and pre) are defered to pip. + import optparse + + class PipCommand(pip_shims.shims.Command): + name = "PipCommand" + + pip_command = PipCommand() + pip_command.parser.add_option(pip_shims.shims.cmdoptions.no_binary()) + pip_command.parser.add_option(pip_shims.shims.cmdoptions.only_binary()) + index_opts = pip_shims.shims.cmdoptions.make_option_group( + pip_shims.shims.cmdoptions.index_group, pip_command.parser + ) + pip_command.parser.insert_option_group(0, index_opts) + pip_command.parser.add_option( + optparse.Option("--pre", action="store_true", default=False) + ) + + return pip_command + + @attr.s class AbstractDependency(object): name = attr.ib() @@ -238,7 +264,7 @@ def get_abstract_dependencies(reqs, sources=None, parent=None): from .requirements import Requirement for req in reqs: - if isinstance(req, InstallRequirement): + if isinstance(req, pip_shims.shims.InstallRequirement): requirement = Requirement.from_line( "{0}{1}".format(req.name, req.specifier) ) @@ -269,16 +295,16 @@ def get_dependencies(ireq, sources=None, parent=None): :return: A set of dependency lines for generating new InstallRequirements. :rtype: set(str) """ - if not isinstance(ireq, InstallRequirement): + if not isinstance(ireq, pip_shims.shims.InstallRequirement): name = getattr( ireq, "project_name", getattr(ireq, "project", ireq.name), ) version = getattr(ireq, "version", None) if not version: - ireq = InstallRequirement.from_line("{0}".format(name)) + ireq = pip_shims.shims.InstallRequirement.from_line("{0}".format(name)) else: - ireq = InstallRequirement.from_line("{0}=={1}".format(name, version)) + ireq = pip_shims.shims.InstallRequirement.from_line("{0}=={1}".format(name, version)) pip_options = get_pip_options(sources=sources) getters = [ get_dependencies_from_cache, @@ -336,18 +362,22 @@ def get_dependencies_from_json(ireq): return session = requests.session() + atexit.register(session.close) version = str(ireq.req.specifier).lstrip("=") def gen(ireq): info = None - info = session.get( - "https://pypi.org/pypi/{0}/{1}/json".format(ireq.req.name, version) - ).json()["info"] + try: + info = session.get( + "https://pypi.org/pypi/{0}/{1}/json".format(ireq.req.name, version) + ).json()["info"] + finally: + session.close() requires_dist = info.get("requires_dist", info.get("requires")) if not requires_dist: # The API can return None for this. return for requires in requires_dist: - i = InstallRequirement.from_line(requires) + i = pip_shims.shims.InstallRequirement.from_line(requires) # See above, we don't handle requirements with extras. if not _marker_contains_extra(i): yield format_requirement(i) @@ -382,7 +412,7 @@ def get_dependencies_from_cache(ireq): try: broken = False for line in cached: - dep_ireq = InstallRequirement.from_line(line) + dep_ireq = pip_shims.shims.InstallRequirement.from_line(line) name = canonicalize_name(dep_ireq.name) if _marker_contains_extra(dep_ireq): broken = True # The "extra =" marker breaks everything. @@ -419,7 +449,7 @@ def get_dependencies_from_index(dep, sources=None, pip_options=None, wheel_cache if not wheel_cache: wheel_cache = WHEEL_CACHE dep.is_direct = True - reqset = RequirementSet() + reqset = pip_shims.shims.RequirementSet() reqset.add_requirement(dep) requirements = None setup_requires = {} @@ -547,7 +577,8 @@ def get_finder(sources=None, pip_command=None, pip_options=None): if not pip_options: pip_options = get_pip_options(sources=sources, pip_command=pip_command) session = pip_command._build_session(pip_options) - finder = PackageFinder( + atexit.register(session.close) + finder = pip_shims.shims.PackageFinder( find_links=[], index_urls=[s.get("url") for s in sources], trusted_hosts=[], @@ -580,19 +611,19 @@ def start_resolver(finder=None, wheel_cache=None): download_dir = PKGS_DOWNLOAD_DIR _ensure_dir(download_dir) - _build_dir = TemporaryDirectory(fs_str("build")) - _source_dir = TemporaryDirectory(fs_str("source")) + _build_dir = create_tracked_tempdir(fs_str("build")) + _source_dir = create_tracked_tempdir(fs_str("source")) preparer = partialclass( - RequirementPreparer, - build_dir=_build_dir.name, - src_dir=_source_dir.name, + pip_shims.shims.RequirementPreparer, + build_dir=_build_dir, + src_dir=_source_dir, download_dir=download_dir, wheel_download_dir=WHEEL_DOWNLOAD_DIR, progress_bar="off", build_isolation=False, ) resolver = partialclass( - Resolver, + pip_shims.shims.Resolver, finder=finder, session=finder.session, upgrade_strategy="to-satisfy-only", @@ -604,13 +635,16 @@ def start_resolver(finder=None, wheel_cache=None): wheel_cache=wheel_cache, use_user_site=False, ) - if packaging.version.parse(pip_version) >= packaging.version.parse('18'): - with RequirementTracker() as req_tracker: - preparer = preparer(req_tracker=req_tracker) + try: + if packaging.version.parse(pip_shims.shims.pip_version) >= packaging.version.parse('18'): + with pip_shims.shims.RequirementTracker() as req_tracker: + preparer = preparer(req_tracker=req_tracker) + yield resolver(preparer=preparer) + else: + preparer = preparer() yield resolver(preparer=preparer) - else: - preparer = preparer() - yield resolver(preparer=preparer) + finally: + finder.session.close() def get_grouped_dependencies(constraints): diff --git a/pipenv/vendor/requirementslib/models/lockfile.py b/pipenv/vendor/requirementslib/models/lockfile.py index f9ca97b8..9d19edaf 100644 --- a/pipenv/vendor/requirementslib/models/lockfile.py +++ b/pipenv/vendor/requirementslib/models/lockfile.py @@ -1,17 +1,22 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import -import json +import copy import os +import attr +import itertools import plette.lockfiles import six -from vistir.compat import Path -from vistir.contextmanagers import atomic_open_for_write +from vistir.compat import Path, FileNotFoundError, JSONDecodeError +from .project import ProjectFile from .requirements import Requirement +from .utils import optional_instance_of +from ..exceptions import LockfileCorruptException, PipfileNotFound, MissingParameter +from ..utils import is_vcs, is_editable, merge_items DEFAULT_NEWLINES = u"\n" @@ -22,83 +27,279 @@ def preferred_newlines(f): return DEFAULT_NEWLINES -class Lockfile(plette.lockfiles.Lockfile): - def __init__(self, *args, **kwargs): - path = kwargs.pop("path", None) - self._requirements = kwargs.pop("requirements", []) - self._dev_requirements = kwargs.pop("dev_requirements", []) - self.path = Path(path) if path else None - self.newlines = u"\n" - super(Lockfile, self).__init__(*args, **kwargs) +is_lockfile = optional_instance_of(plette.lockfiles.Lockfile) +is_projectfile = optional_instance_of(ProjectFile) + + +@attr.s(slots=True) +class Lockfile(object): + path = attr.ib(validator=optional_instance_of(Path), type=Path) + _requirements = attr.ib(default=attr.Factory(list), type=list) + _dev_requirements = attr.ib(default=attr.Factory(list), type=list) + projectfile = attr.ib(validator=is_projectfile, type=ProjectFile) + _lockfile = attr.ib(validator=is_lockfile, type=plette.lockfiles.Lockfile) + newlines = attr.ib(default=DEFAULT_NEWLINES, type=six.text_type) + + @path.default + def _get_path(self): + return Path(os.curdir).absolute() + + @projectfile.default + def _get_projectfile(self): + return self.load_projectfile(self.path) + + @_lockfile.default + def _get_lockfile(self): + return self.projectfile.lockfile + + @property + def lockfile(self): + return self._lockfile + + @property + def section_keys(self): + return ["default", "develop"] + + @property + def extended_keys(self): + return [k for k in itertools.product(self.section_keys, ["", "vcs", "editable"])] + + def get(self, k): + return self.__getitem__(k) + + def __contains__(self, k): + check_lockfile = k in self.extended_keys or self.lockfile.__contains__(k) + if check_lockfile: + return True + return super(Lockfile, self).__contains__(k) + + def __setitem__(self, k, v): + lockfile = self._lockfile + lockfile.__setitem__(k, v) + + def __getitem__(self, k, *args, **kwargs): + retval = None + lockfile = self._lockfile + section = None + pkg_type = None + try: + retval = lockfile[k] + except KeyError: + if "-" in k: + section, _, pkg_type = k.rpartition("-") + vals = getattr(lockfile.get(section, {}), "_data", {}) + if pkg_type == "vcs": + retval = {k: v for k, v in vals.items() if is_vcs(v)} + elif pkg_type == "editable": + retval = {k: v for k, v in vals.items() if is_editable(v)} + if retval is None: + raise + else: + retval = getattr(retval, "_data", retval) + return retval + + def __getattr__(self, k, *args, **kwargs): + retval = None + lockfile = super(Lockfile, self).__getattribute__("_lockfile") + try: + return super(Lockfile, self).__getattribute__(k) + except AttributeError: + retval = getattr(lockfile, k, None) + if retval is not None: + return retval + return super(Lockfile, self).__getattribute__(k, *args, **kwargs) + + def get_deps(self, dev=False, only=True): + deps = {} + if dev: + deps.update(self.develop._data) + if only: + return deps + deps = merge_items([deps, self.default._data]) + return deps @classmethod - def load(cls, path): + def read_projectfile(cls, path): + """Read the specified project file and provide an interface for writing/updating. + + :param str path: Path to the target file. + :return: A project file with the model and location for interaction + :rtype: :class:`~requirementslib.models.project.ProjectFile` + """ + + pf = ProjectFile.read( + path, + plette.lockfiles.Lockfile, + invalid_ok=True + ) + return pf + + @classmethod + def lockfile_from_pipfile(cls, pipfile_path): + from .pipfile import Pipfile + if os.path.isfile(pipfile_path): + if not os.path.isabs(pipfile_path): + pipfile_path = os.path.abspath(pipfile_path) + pipfile = Pipfile.load(os.path.dirname(pipfile_path)) + return plette.lockfiles.Lockfile.with_meta_from(pipfile._pipfile) + raise PipfileNotFound(pipfile_path) + + @classmethod + def load_projectfile(cls, path, create=True, data=None): + """Given a path, load or create the necessary lockfile. + + :param str path: Path to the project root or lockfile + :param bool create: Whether to create the lockfile if not found, defaults to True + :raises OSError: Thrown if the project root directory doesn't exist + :raises FileNotFoundError: Thrown if the lockfile doesn't exist and ``create=False`` + :return: A project file instance for the supplied project + :rtype: :class:`~requirementslib.models.project.ProjectFile` + """ + if not path: path = os.curdir path = Path(path).absolute() - if path.is_dir(): - path = path / "Pipfile.lock" - elif path.name == "Pipfile": - path = path.parent / "Pipfile.lock" - if not path.exists(): - raise OSError("Path does not exist: %s" % path) - return cls.create(path.parent, lockfile_name=path.name) + project_path = path if path.is_dir() else path.parent + lockfile_path = project_path / "Pipfile.lock" + if not project_path.exists(): + raise OSError("Project does not exist: %s" % project_path.as_posix()) + elif not lockfile_path.exists() and not create: + raise FileNotFoundError("Lockfile does not exist: %s" % lockfile_path.as_posix()) + projectfile = cls.read_projectfile(lockfile_path.as_posix()) + if not lockfile_path.exists(): + if not data: + lf = cls.lockfile_from_pipfile(project_path.joinpath("Pipfile")) + else: + lf = plette.lockfiles.Lockfile(data) + projectfile.model = lf + return projectfile @classmethod - def create(cls, project_path, lockfile_name="Pipfile.lock"): - """Create a new lockfile instance + def from_data(cls, path, data, meta_from_project=True): + """Create a new lockfile instance from a dictionary. - :param project_path: Path to project root - :type project_path: str or :class:`~pathlib.Path` - :returns: List[:class:`~requirementslib.Requirement`] objects + :param str path: Path to the project root. + :param dict data: Data to load into the lockfile. + :param bool meta_from_project: Attempt to populate the meta section from the + project root, default True. """ - if not isinstance(project_path, Path): - project_path = Path(project_path) - lockfile_path = project_path / lockfile_name - with lockfile_path.open(encoding="utf-8") as f: - lockfile = super(Lockfile, cls).load(f) - lockfile.newlines = preferred_newlines(f) - lockfile.path = lockfile_path - return lockfile + if path is None: + raise MissingParameter("path") + if data is None: + raise MissingParameter("data") + if not isinstance(data, dict): + raise TypeError("Expecting a dictionary for parameter 'data'") + path = os.path.abspath(str(path)) + if os.path.isdir(path): + project_path = path + elif not os.path.isdir(path) and os.path.isdir(os.path.dirname(path)): + project_path = os.path.dirname(path) + pipfile_path = os.path.join(project_path, "Pipfile") + lockfile_path = os.path.join(project_path, "Pipfile.lock") + if meta_from_project: + lockfile = cls.lockfile_from_pipfile(pipfile_path) + lockfile.update(data) + else: + lockfile = plette.lockfiles.Lockfile(data) + projectfile = ProjectFile(line_ending=DEFAULT_NEWLINES, location=lockfile_path, model=lockfile) + return cls( + projectfile=projectfile, lockfile=lockfile, + newlines=projectfile.line_ending, path=Path(projectfile.location) + ) - def get_requirements(self, dev=False): - section = self.develop if dev else self.default - for k in section.keys(): - yield Requirement.from_pipfile(k, section[k]._data) + @classmethod + def load(cls, path, create=True): + """Create a new lockfile instance. + + :param project_path: Path to project root + :type project_path: str or :class:`pathlib.Path` + :param str lockfile_name: Name of the lockfile in the project root directory + :param pipfile_path: Path to the project pipfile + :type pipfile_path: :class:`pathlib.Path` + :returns: A new lockfile representing the supplied project paths + :rtype: :class:`~requirementslib.models.lockfile.Lockfile` + """ + + try: + projectfile = cls.load_projectfile(path, create=create) + except JSONDecodeError: + path = os.path.abspath(path) + if not os.path.isdir(path): + path = os.path.dirname(path) + path = Path(os.path.join(path, "Pipfile.lock")) + formatted_path = path.as_posix() + backup_path = "%s.bak" % formatted_path + LockfileCorruptException.show(formatted_path, backup_path=backup_path) + path.rename(backup_path) + cls.load(formatted_path, create=True) + lockfile_path = Path(projectfile.location) + creation_args = { + "projectfile": projectfile, + "lockfile": projectfile.model, + "newlines": projectfile.line_ending, + "path": lockfile_path + } + return cls(**creation_args) + + @classmethod + def create(cls, path, create=True): + return cls.load(path, create=create) + + @property + def develop(self): + return self._lockfile.develop + + @property + def default(self): + return self._lockfile.default + + def get_requirements(self, dev=True, only=False): + """Produces a generator which generates requirements from the desired section. + + :param bool dev: Indicates whether to use dev requirements, defaults to False + :return: Requirements from the relevant the relevant pipfile + :rtype: :class:`~requirementslib.models.requirements.Requirement` + """ + + deps = self.get_deps(dev=dev, only=only) + for k, v in deps.items(): + yield Requirement.from_pipfile(k, v) @property def dev_requirements(self): if not self._dev_requirements: - self._dev_requirements = list(self.get_requirements(dev=True)) + self._dev_requirements = list(self.get_requirements(dev=True, only=True)) return self._dev_requirements @property def requirements(self): if not self._requirements: - self._requirements = list(self.get_requirements(dev=False)) + self._requirements = list(self.get_requirements(dev=False, only=True)) return self._requirements @property def dev_requirements_list(self): - return [{name: entry._data} for name, entry in self.develop.items()] + return [{name: entry._data} for name, entry in self._lockfile.develop.items()] @property def requirements_list(self): - return [{name: entry._data} for name, entry in self.develop.items()] + return [{name: entry._data} for name, entry in self._lockfile.default.items()] def write(self): - open_kwargs = {"newline": self.newlines} - with atomic_open_for_write(self.path.as_posix(), **open_kwargs) as f: - super(Lockfile, self).dump(f, encoding="utf-8") + self.projectfile.model = copy.deepcopy(self._lockfile) + self.projectfile.write() def as_requirements(self, include_hashes=False, dev=False): """Returns a list of requirements in pip-style format""" lines = [] section = self.dev_requirements if dev else self.requirements for req in section: - r = req.as_line() - if not include_hashes: - r = r.split("--hash", 1)[0] + kwargs = { + "include_hashes": include_hashes, + } + if req.editable: + kwargs["include_markers"] = False + r = req.as_line(**kwargs) lines.append(r.strip()) return lines diff --git a/pipenv/vendor/requirementslib/models/markers.py b/pipenv/vendor/requirementslib/models/markers.py index 83b44b63..70fe3bc0 100644 --- a/pipenv/vendor/requirementslib/models/markers.py +++ b/pipenv/vendor/requirementslib/models/markers.py @@ -4,12 +4,11 @@ import attr from packaging.markers import InvalidMarker, Marker from ..exceptions import RequirementError -from .baserequirement import BaseRequirement from .utils import filter_none, validate_markers @attr.s -class PipenvMarkers(BaseRequirement): +class PipenvMarkers(object): """System-level requirements - see PEP508 for more detail""" os_name = attr.ib( @@ -78,7 +77,8 @@ class PipenvMarkers(BaseRequirement): @classmethod def from_pipfile(cls, name, pipfile): - found_keys = [k for k in pipfile.keys() if k in cls.attr_fields()] + attr_fields = [field.name for field in attr.fields(cls)] + found_keys = [k for k in pipfile.keys() if k in attr_fields] marker_strings = ["{0} {1}".format(k, pipfile[k]) for k in found_keys] if pipfile.get("markers"): marker_strings.append(pipfile.get("markers")) diff --git a/pipenv/vendor/requirementslib/models/pipfile.py b/pipenv/vendor/requirementslib/models/pipfile.py index 3a6f5b1e..0f6de6bf 100644 --- a/pipenv/vendor/requirementslib/models/pipfile.py +++ b/pipenv/vendor/requirementslib/models/pipfile.py @@ -1,64 +1,263 @@ # -*- coding: utf-8 -*- -from vistir.compat import Path + +from __future__ import absolute_import, unicode_literals, print_function + +import attr +import copy +import os + +import tomlkit + +from vistir.compat import Path, FileNotFoundError from .requirements import Requirement +from .project import ProjectFile +from .utils import optional_instance_of from ..exceptions import RequirementError +from ..utils import is_vcs, is_editable, merge_items import plette.pipfiles -class Pipfile(plette.pipfiles.Pipfile): +is_pipfile = optional_instance_of(plette.pipfiles.Pipfile) +is_path = optional_instance_of(Path) +is_projectfile = optional_instance_of(ProjectFile) + + +def reorder_source_keys(data): + for i, entry in enumerate(data["source"]): + table = tomlkit.table() + table["name"] = entry["name"] + table["url"] = entry["url"] + table["verify_ssl"] = entry["verify_ssl"] + data["source"][i] = table + return data + + +class PipfileLoader(plette.pipfiles.Pipfile): + @classmethod + def validate(cls, data): + for key, klass in plette.pipfiles.PIPFILE_SECTIONS.items(): + if key not in data or key == "source": + continue + try: + klass.validate(data[key]) + except Exception: + pass + + @classmethod + def load(cls, f, encoding=None): + content = f.read() + if encoding is not None: + content = content.decode(encoding) + _data = tomlkit.loads(content) + _data["source"] = _data.get("source", []) + _data.get("sources", []) + _data = reorder_source_keys(_data) + if "source" not in _data: + if "sources" in _data: + _data["source"] = _data["sources"] + content = tomlkit.dumps(_data) + else: + # HACK: There is no good way to prepend a section to an existing + # TOML document, but there's no good way to copy non-structural + # content from one TOML document to another either. Modify the + # TOML content directly, and load the new in-memory document. + sep = "" if content.startswith("\n") else "\n" + content = plette.pipfiles.DEFAULT_SOURCE_TOML + sep + content + data = tomlkit.loads(content) + instance = cls(data) + instance._data = dict(instance._data) + return instance + + def __getattribute__(self, key): + if key == "source": + return self._data[key] + return super(PipfileLoader, self).__getattribute__(key) + + +@attr.s(slots=True) +class Pipfile(object): + path = attr.ib(validator=is_path, type=Path) + projectfile = attr.ib(validator=is_projectfile, type=ProjectFile) + _pipfile = attr.ib(type=plette.pipfiles.Pipfile) + _pyproject = attr.ib(default=attr.Factory(tomlkit.document), type=tomlkit.toml_document.TOMLDocument) + build_system = attr.ib(default=attr.Factory(dict), type=dict) + requirements = attr.ib(default=attr.Factory(list), type=list) + dev_requirements = attr.ib(default=attr.Factory(list), type=list) + + @path.default + def _get_path(self): + return Path(os.curdir).absolute() + + @projectfile.default + def _get_projectfile(self): + return self.load_projectfile(os.curdir, create=False) + + @_pipfile.default + def _get_pipfile(self): + return self.projectfile.model + + @property + def pipfile(self): + return self._pipfile + + def get_deps(self, dev=False, only=True): + deps = {} + if dev: + deps.update(self.pipfile._data["dev-packages"]) + if only: + return deps + return merge_items([deps, self.pipfile._data["packages"]]) + + def get(self, k): + return self.__getitem__(k) + + def __contains__(self, k): + check_pipfile = k in self.extended_keys or self.pipfile.__contains__(k) + if check_pipfile: + return True + return super(Pipfile, self).__contains__(k) + + def __getitem__(self, k, *args, **kwargs): + retval = None + pipfile = self._pipfile + section = None + pkg_type = None + try: + retval = pipfile[k] + except KeyError: + if "-" in k: + section, _, pkg_type = k.rpartition("-") + vals = getattr(pipfile.get(section, {}), "_data", {}) + if pkg_type == "vcs": + retval = {k: v for k, v in vals.items() if is_vcs(v)} + elif pkg_type == "editable": + retval = {k: v for k, v in vals.items() if is_editable(v)} + if retval is None: + raise + else: + retval = getattr(retval, "_data", retval) + return retval + + def __getattr__(self, k, *args, **kwargs): + retval = None + pipfile = super(Pipfile, self).__getattribute__("_pipfile") + try: + retval = super(Pipfile, self).__getattribute__(k) + except AttributeError: + retval = getattr(pipfile, k, None) + if retval is not None: + return retval + return super(Pipfile, self).__getattribute__(k, *args, **kwargs) @property def requires_python(self): - return self.requires.requires_python + return self._pipfile.requires.requires_python @property def allow_prereleases(self): - return self.get("pipenv", {}).get("allow_prereleases", False) + return self._pipfile.get("pipenv", {}).get("allow_prereleases", False) @classmethod - def load(cls, path): + def read_projectfile(cls, path): + """Read the specified project file and provide an interface for writing/updating. + + :param str path: Path to the target file. + :return: A project file with the model and location for interaction + :rtype: :class:`~requirementslib.models.project.ProjectFile` + """ + pf = ProjectFile.read( + path, + PipfileLoader, + invalid_ok=True + ) + return pf + + @classmethod + def load_projectfile(cls, path, create=False): + """Given a path, load or create the necessary pipfile. + + :param str path: Path to the project root or pipfile + :param bool create: Whether to create the pipfile if not found, defaults to True + :raises OSError: Thrown if the project root directory doesn't exist + :raises FileNotFoundError: Thrown if the pipfile doesn't exist and ``create=False`` + :return: A project file instance for the supplied project + :rtype: :class:`~requirementslib.models.project.ProjectFile` + """ + if not path: + raise RuntimeError("Must pass a path to classmethod 'Pipfile.load'") if not isinstance(path, Path): - path = Path(path) - pipfile_path = path / "Pipfile" - if not path.exists(): + path = Path(path).absolute() + pipfile_path = path if path.name == "Pipfile" else path.joinpath("Pipfile") + project_path = pipfile_path.parent + if not project_path.exists(): raise FileNotFoundError("%s is not a valid project path!" % path) elif not pipfile_path.exists() or not pipfile_path.is_file(): - raise RequirementError("%s is not a valid Pipfile" % pipfile_path) - with pipfile_path.open(encoding="utf-8") as fp: - pipfile = super(Pipfile, cls).load(fp) - pipfile.dev_requirements = [ - Requirement.from_pipfile(k, v) for k, v in pipfile.get("dev-packages", {}).items() - ] - pipfile.requirements = [ - Requirement.from_pipfile(k, v) for k, v in pipfile.get("packages", {}).items() - ] - pipfile.path = pipfile_path - return pipfile + if not create: + raise RequirementError("%s is not a valid Pipfile" % pipfile_path) + return cls.read_projectfile(pipfile_path.as_posix()) - # def resolve(self): - # It would be nice to still use this api someday - # option_sources = [s.expanded for s in self.sources] - # pip_args = [] - # if self.pipenv.allow_prereleases: - # pip_args.append('--pre') - # pip_options = get_pip_options(pip_args, sources=option_sources) - # finder = get_finder(sources=option_sources, pip_options=pip_options) - # resolver = DependencyResolver.create(finder=finder, allow_prereleases=self.pipenv.allow_prereleases) - # pkg_dict = {} - # for pkg in self.dev_packages.requirements + self.packages.requirements: - # pkg_dict[pkg.name] = pkg - # resolver.resolve(list(pkg_dict.values())) - # return resolver + @classmethod + def load(cls, path, create=False): + """Given a path, load or create the necessary pipfile. + + :param str path: Path to the project root or pipfile + :param bool create: Whether to create the pipfile if not found, defaults to True + :raises OSError: Thrown if the project root directory doesn't exist + :raises FileNotFoundError: Thrown if the pipfile doesn't exist and ``create=False`` + :return: A pipfile instance pointing at the supplied project + :rtype:: class:`~requirementslib.models.pipfile.Pipfile` + """ + + projectfile = cls.load_projectfile(path, create=create) + pipfile = projectfile.model + dev_requirements = [ + Requirement.from_pipfile(k, getattr(v, "_data", v)) for k, v in pipfile.get("dev-packages", {}).items() + ] + requirements = [ + Requirement.from_pipfile(k, getattr(v, "_data", v)) for k, v in pipfile.get("packages", {}).items() + ] + creation_args = { + "projectfile": projectfile, + "pipfile": pipfile, + "dev_requirements": dev_requirements, + "requirements": requirements, + "path": Path(projectfile.location) + } + return cls(**creation_args) + + def write(self): + self.projectfile.model = copy.deepcopy(self._pipfile) + self.projectfile.write() @property def dev_packages(self, as_requirements=True): if as_requirements: return self.dev_requirements - return self.get('dev-packages', {}) + return self._pipfile.get('dev-packages', {}) @property def packages(self, as_requirements=True): if as_requirements: return self.requirements - return self.get('packages', {}) + return self._pipfile.get('packages', {}) + + def _read_pyproject(self): + pyproject = self.path.parent.joinpath("pyproject.toml") + if pyproject.exists(): + self._pyproject = tomlkit.load(pyproject) + build_system = self._pyproject.get("build-system", None) + if not os.path.exists(self.path_to("setup.py")): + if not build_system or not build_system.get("requires"): + build_system = { + "requires": ["setuptools>=38.2.5", "wheel"], + "build-backend": "setuptools.build_meta", + } + self._build_system = build_system + + @property + def build_requires(self): + return self.build_system.get("requires", []) + + @property + def build_backend(self): + return self.build_system.get("build-backend", None) diff --git a/pipenv/vendor/requirementslib/models/project.py b/pipenv/vendor/requirementslib/models/project.py new file mode 100644 index 00000000..f6e037d6 --- /dev/null +++ b/pipenv/vendor/requirementslib/models/project.py @@ -0,0 +1,241 @@ +# -*- coding=utf-8 -*- + +from __future__ import absolute_import, unicode_literals + +import collections +import io +import os + +import attr +import packaging.markers +import packaging.utils +import plette +import plette.models +import six +import tomlkit + + +SectionDifference = collections.namedtuple("SectionDifference", [ + "inthis", "inthat", +]) +FileDifference = collections.namedtuple("FileDifference", [ + "default", "develop", +]) + + +def _are_pipfile_entries_equal(a, b): + a = {k: v for k, v in a.items() if k not in ("markers", "hashes", "hash")} + b = {k: v for k, v in b.items() if k not in ("markers", "hashes", "hash")} + if a != b: + return False + try: + marker_eval_a = packaging.markers.Marker(a["markers"]).evaluate() + except (AttributeError, KeyError, TypeError, ValueError): + marker_eval_a = True + try: + marker_eval_b = packaging.markers.Marker(b["markers"]).evaluate() + except (AttributeError, KeyError, TypeError, ValueError): + marker_eval_b = True + return marker_eval_a == marker_eval_b + + +DEFAULT_NEWLINES = "\n" + + +def preferred_newlines(f): + if isinstance(f.newlines, six.text_type): + return f.newlines + return DEFAULT_NEWLINES + + +@attr.s +class ProjectFile(object): + """A file in the Pipfile project. + """ + location = attr.ib() + line_ending = attr.ib() + model = attr.ib() + + @classmethod + def read(cls, location, model_cls, invalid_ok=False): + try: + with io.open(location, encoding="utf-8") as f: + model = model_cls.load(f) + line_ending = preferred_newlines(f) + except Exception: + if not invalid_ok: + raise + model = None + line_ending = DEFAULT_NEWLINES + return cls(location=location, line_ending=line_ending, model=model) + + def write(self): + kwargs = {"encoding": "utf-8", "newline": self.line_ending} + with io.open(self.location, "w", **kwargs) as f: + self.model.dump(f) + + def dumps(self): + strio = six.StringIO() + self.model.dump(strio) + return strio.getvalue() + + +@attr.s +class Project(object): + + root = attr.ib() + _p = attr.ib(init=False) + _l = attr.ib(init=False) + + def __attrs_post_init__(self): + self.root = root = os.path.abspath(self.root) + self._p = ProjectFile.read( + os.path.join(root, "Pipfile"), + plette.Pipfile, + ) + self._l = ProjectFile.read( + os.path.join(root, "Pipfile.lock"), + plette.Lockfile, + invalid_ok=True, + ) + + @property + def pipfile(self): + return self._p.model + + @property + def pipfile_location(self): + return self._p.location + + @property + def lockfile(self): + return self._l.model + + @property + def lockfile_location(self): + return self._l.location + + @lockfile.setter + def lockfile(self, new): + self._l.model = new + + def is_synced(self): + return self.lockfile and self.lockfile.is_up_to_date(self.pipfile) + + def _get_pipfile_section(self, develop, insert=True): + name = "dev-packages" if develop else "packages" + try: + section = self.pipfile[name] + except KeyError: + section = plette.models.PackageCollection(tomlkit.table()) + if insert: + self.pipfile[name] = section + return section + + def contains_key_in_pipfile(self, key): + sections = [ + self._get_pipfile_section(develop=False, insert=False), + self._get_pipfile_section(develop=True, insert=False), + ] + return any( + (packaging.utils.canonicalize_name(name) == + packaging.utils.canonicalize_name(key)) + for section in sections + for name in section + ) + + def add_line_to_pipfile(self, line, develop): + from requirementslib import Requirement + requirement = Requirement.from_line(line) + section = self._get_pipfile_section(develop=develop) + key = requirement.normalized_name + entry = next(iter(requirement.as_pipfile().values())) + if isinstance(entry, dict): + # HACK: TOMLKit prefers to expand tables by default, but we + # always want inline tables here. Also tomlkit.inline_table + # does not have `update()`. + table = tomlkit.inline_table() + for k, v in entry.items(): + table[k] = v + entry = table + section[key] = entry + + def remove_keys_from_pipfile(self, keys, default, develop): + keys = {packaging.utils.canonicalize_name(key) for key in keys} + sections = [] + if default: + sections.append(self._get_pipfile_section( + develop=False, insert=False, + )) + if develop: + sections.append(self._get_pipfile_section( + develop=True, insert=False, + )) + for section in sections: + removals = set() + for name in section: + if packaging.utils.canonicalize_name(name) in keys: + removals.add(name) + for key in removals: + del section._data[key] + + def remove_keys_from_lockfile(self, keys): + keys = {packaging.utils.canonicalize_name(key) for key in keys} + removed = False + for section_name in ("default", "develop"): + try: + section = self.lockfile[section_name] + except KeyError: + continue + removals = set() + for name in section: + if packaging.utils.canonicalize_name(name) in keys: + removals.add(name) + removed = removed or bool(removals) + for key in removals: + del section._data[key] + + if removed: + # HACK: The lock file no longer represents the Pipfile at this + # point. Set the hash to an arbitrary invalid value. + self.lockfile.meta.hash = plette.models.Hash({"__invalid__": ""}) + + def difference_lockfile(self, lockfile): + """Generate a difference between the current and given lockfiles. + + Returns a 2-tuple containing differences in default in develop + sections. + + Each element is a 2-tuple of dicts. The first, `inthis`, contains + entries only present in the current lockfile; the second, `inthat`, + contains entries only present in the given one. + + If a key exists in both this and that, but the values differ, the key + is present in both dicts, pointing to values from each file. + """ + diff_data = { + "default": SectionDifference({}, {}), + "develop": SectionDifference({}, {}), + } + for section_name, section_diff in diff_data.items(): + try: + this = self.lockfile[section_name]._data + except (KeyError, TypeError): + this = {} + try: + that = lockfile[section_name]._data + except (KeyError, TypeError): + that = {} + for key, this_value in this.items(): + try: + that_value = that[key] + except KeyError: + section_diff.inthis[key] = this_value + continue + if not _are_pipfile_entries_equal(this_value, that_value): + section_diff.inthis[key] = this_value + section_diff.inthat[key] = that_value + for key, that_value in that.items(): + if key not in this: + section_diff.inthat[key] = that_value + return FileDifference(**diff_data) diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index 248ca777..d034a12d 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- + from __future__ import absolute_import -import atexit import collections import hashlib import os @@ -9,46 +9,59 @@ import os from contextlib import contextmanager import attr +import pip_shims from first import first from packaging.markers import Marker from packaging.requirements import Requirement as PackagingRequirement -from packaging.specifiers import Specifier, SpecifierSet +from packaging.specifiers import Specifier, SpecifierSet, LegacySpecifier, InvalidSpecifier from packaging.utils import canonicalize_name -from pip_shims.shims import ( - InstallRequirement, Link, Wheel, _strip_extras, parse_version, path_to_url, - url_to_path -) from six.moves.urllib import parse as urllib_parse from six.moves.urllib.parse import unquote -from vistir.compat import FileNotFoundError, Path, TemporaryDirectory +from vistir.compat import FileNotFoundError, Path from vistir.misc import dedup from vistir.path import ( - create_tracked_tempdir, get_converted_relative_path, is_file_url, - is_valid_url, mkdir_p + create_tracked_tempdir, + get_converted_relative_path, + is_file_url, + is_valid_url, ) from ..exceptions import RequirementError -from ..utils import VCS_LIST, is_installable_file, is_vcs, ensure_setup_py -from .baserequirement import BaseRequirement -from .dependencies import ( - AbstractDependency, find_all_matches, get_abstract_dependencies, - get_dependencies, get_finder +from ..utils import ( + VCS_LIST, + is_installable_file, + is_vcs, + ensure_setup_py, + add_ssh_scheme_to_git_uri, + strip_ssh_from_git_uri, ) -from .markers import PipenvMarkers +from .setup_info import SetupInfo from .utils import ( - HASH_STRING, add_ssh_scheme_to_git_uri, build_vcs_link, extras_to_string, - filter_none, format_requirement, get_version, init_requirement, - is_pinned_requirement, make_install_requirement, optional_instance_of, - parse_extras, specs_to_string, split_markers_from_line, - split_vcs_method_from_uri, strip_ssh_from_git_uri, validate_path, - validate_specifiers, validate_vcs + HASH_STRING, + build_vcs_link, + extras_to_string, + filter_none, + format_requirement, + get_version, + init_requirement, + is_pinned_requirement, + make_install_requirement, + parse_extras, + specs_to_string, + split_markers_from_line, + split_vcs_method_from_uri, + validate_path, + validate_specifiers, + validate_vcs, + normalize_name, + create_link, + get_pyproject ) -from .vcs import VCSRepository -@attr.s -class NamedRequirement(BaseRequirement): +@attr.s(slots=True) +class NamedRequirement(object): name = attr.ib() version = attr.ib(validator=attr.validators.optional(validate_specifiers)) req = attr.ib() @@ -57,7 +70,9 @@ class NamedRequirement(BaseRequirement): @req.default def get_requirement(self): - req = init_requirement("{0}{1}".format(canonicalize_name(self.name), self.version)) + req = init_requirement( + "{0}{1}".format(canonicalize_name(self.name), self.version) + ) return req @classmethod @@ -83,14 +98,15 @@ class NamedRequirement(BaseRequirement): def from_pipfile(cls, name, pipfile): creation_args = {} if hasattr(pipfile, "keys"): - creation_args = {k: v for k, v in pipfile.items() if k in cls.attr_fields()} + attr_fields = [field.name for field in attr.fields(cls)] + creation_args = {k: v for k, v in pipfile.items() if k in attr_fields} creation_args["name"] = name version = get_version(pipfile) extras = creation_args.get("extras", None) creation_args["version"] = version req = init_requirement("{0}{1}".format(name, version)) if extras: - req.extras += tuple(extras,) + req.extras += tuple(extras) creation_args["req"] = req return cls(**creation_args) @@ -99,7 +115,7 @@ class NamedRequirement(BaseRequirement): # FIXME: This should actually be canonicalized but for now we have to # simply lowercase it and replace underscores, since full canonicalization # also replaces dots and that doesn't actually work when querying the index - return "{0}".format(self.name.lower().replace("_", "-")) + return "{0}".format(normalize_name(self.name)) @property def pipfile_part(self): @@ -115,22 +131,37 @@ LinkInfo = collections.namedtuple( ) -@attr.s -class FileRequirement(BaseRequirement): +@attr.s(slots=True) +class FileRequirement(object): """File requirements for tar.gz installable files or wheels or setup.py containing directories.""" + #: Path to the relevant `setup.py` location setup_path = attr.ib(default=None) + #: path to hit - without any of the VCS prefixes (like git+ / http+ / etc) path = attr.ib(default=None, validator=attr.validators.optional(validate_path)) - # : path to hit - without any of the VCS prefixes (like git+ / http+ / etc) - editable = attr.ib(default=None) + #: Whether the package is editable + editable = attr.ib(default=False) + #: Extras if applicable extras = attr.ib(default=attr.Factory(list)) - uri = attr.ib() - link = attr.ib() - name = attr.ib() - req = attr.ib() - _has_hashed_name = False _uri_scheme = attr.ib(default=None) + #: URI of the package + uri = attr.ib() + #: Link object representing the package to clone + link = attr.ib() + #: PyProject Requirements + pyproject_requires = attr.ib(default=attr.Factory(list)) + #: PyProject Build System + pyproject_backend = attr.ib(default=None) + #: PyProject Path + pyproject_path = attr.ib(default=None) + _has_hashed_name = attr.ib(default=False) + #: Package name + name = attr.ib() + #: A :class:`~pkg_resources.Requirement` isntance + req = attr.ib() + #: Setup metadata e.g. dependencies + setup_info = attr.ib(default=None) @classmethod def get_link_from_line(cls, line): @@ -164,6 +195,7 @@ class FileRequirement(BaseRequirement): See `https://bugs.python.org/issue23505#msg277350`. """ + # Git allows `git@github.com...` lines that are not really URIs. # Add "ssh://" so we can parse correctly, and restore afterwards. fixed_line = add_ssh_scheme_to_git_uri(line) @@ -174,7 +206,7 @@ class FileRequirement(BaseRequirement): p = Path(fixed_line).absolute() path = p.as_posix() uri = p.as_uri() - link = Link(uri) + link = create_link(uri) try: relpath = get_converted_relative_path(path) except ValueError: @@ -185,9 +217,9 @@ class FileRequirement(BaseRequirement): parsed_url = urllib_parse.urlsplit(fixed_line) original_url = parsed_url._replace() - if added_ssh_scheme and ':' in parsed_url.netloc: - original_netloc, original_path_start = parsed_url.netloc.rsplit(':', 1) - uri_path = '/{0}{1}'.format(original_path_start, parsed_url.path) + if added_ssh_scheme and ":" in parsed_url.netloc: + original_netloc, original_path_start = parsed_url.netloc.rsplit(":", 1) + uri_path = "/{0}{1}".format(original_path_start, parsed_url.path) parsed_url = original_url._replace(netloc=original_netloc, path=uri_path) # Split the VCS part out if needed. @@ -203,12 +235,14 @@ class FileRequirement(BaseRequirement): if parsed_url.scheme == "file" and parsed_url.path: # This is a "file://" URI. Use url_to_path and path_to_url to # ensure the path is absolute. Also we need to build relpath. - path = Path(url_to_path(urllib_parse.urlunsplit(parsed_url))).as_posix() + path = Path( + pip_shims.shims.url_to_path(urllib_parse.urlunsplit(parsed_url)) + ).as_posix() try: relpath = get_converted_relative_path(path) except ValueError: relpath = None - uri = path_to_url(path) + uri = pip_shims.shims.path_to_url(path) else: # This is a remote URI. Simply use it. path = None @@ -219,92 +253,116 @@ class FileRequirement(BaseRequirement): ) if added_ssh_scheme: - original_uri = urllib_parse.urlunsplit(original_url._replace(scheme=original_scheme, fragment="")) + original_uri = urllib_parse.urlunsplit( + original_url._replace(scheme=original_scheme, fragment="") + ) uri = strip_ssh_from_git_uri(original_uri) # Re-attach VCS prefix to build a Link. - link = Link( + link = create_link( urllib_parse.urlunsplit(parsed_url._replace(scheme=original_scheme)) ) return LinkInfo(vcs_type, prefer, relpath, path, uri, link) + @property + def setup_py_dir(self): + if self.setup_path: + return os.path.dirname(os.path.abspath(self.setup_path)) + + @property + def dependencies(self): + build_deps = [] + setup_deps = [] + deps = {} + if self.setup_info: + setup_info = self.setup_info.as_dict() + deps.update(setup_info.get("requires", {})) + setup_deps.extend(setup_info.get("setup_requires", [])) + build_deps.extend(setup_info.get("build_requires", [])) + if self.pyproject_requires: + build_deps.extend(self.pyproject_requires) + return deps, setup_deps, build_deps + @uri.default def get_uri(self): if self.path and not self.uri: self._uri_scheme = "path" - self.uri = path_to_url(os.path.abspath(self.path)) + return pip_shims.shims.path_to_url(os.path.abspath(self.path)) + elif self.req and getattr(self.req, "url"): + return self.req.url @name.default def get_name(self): loc = self.path or self.uri - if loc: - self._uri_scheme = "path" if self.path else "uri" + if loc and not self._uri_scheme: + self._uri_scheme = "path" if self.path else "file" name = None - if self.link and self.link.egg_fragment: + if getattr(self, "req", None) and getattr(self.req, "name") and self.req.name is not None: + if self.is_direct_url: + return self.req.name + if self.link and self.link.egg_fragment and not self._has_hashed_name: return self.link.egg_fragment elif self.link and self.link.is_wheel: + from pip_shims import Wheel + self._has_hashed_name = False return Wheel(self.link.filename).name - if ( - self._uri_scheme != "uri" - and self.path - and self.setup_path - and self.setup_path.exists() - ): - from setuptools.dist import distutils - - old_curdir = os.path.abspath(os.getcwd()) - try: - os.chdir(str(self.setup_path.parent)) - dist = distutils.core.run_setup(self.setup_path.as_posix()) - name = dist.get_name() - except (FileNotFoundError, IOError) as e: - dist = None - except Exception as e: - from pip_shims.shims import InstallRequirement, make_abstract_dist - - try: - if not isinstance(Path, self.path): - _path = Path(self.path) - else: - _path = self.path - if self.editable: - _ireq = InstallRequirement.from_editable(_path.as_uri()) - else: - _ireq = InstallRequirement.from_line(_path.as_posix()) - dist = make_abstract_dist(_ireq).get_dist() - name = dist.project_name - except (TypeError, ValueError, AttributeError) as e: - dist = None - finally: - os.chdir(old_curdir) + elif self.link and ((self.link.scheme == "file" or self.editable) or ( + self.path and self.setup_path and os.path.isfile(str(self.setup_path)) + )): + if self.editable: + line = pip_shims.shims.path_to_url(self.setup_py_dir) + _ireq = pip_shims.shims.install_req_from_editable(line) + else: + _ireq = pip_shims.shims.install_req_from_line(Path(self.setup_py_dir).as_posix()) + from .setup_info import SetupInfo + subdir = getattr(self, "subdirectory", None) + setupinfo = SetupInfo.from_ireq(_ireq, subdir=subdir) + if setupinfo: + self.setup_info = setupinfo + setupinfo_dict = setupinfo.as_dict() + setup_name = setupinfo_dict.get("name", None) + if setup_name: + name = setup_name + self._has_hashed_name = False + build_requires = setupinfo_dict.get("build_requires") + build_backend = setupinfo_dict.get("build_backend") + if build_requires and not self.pyproject_requires: + self.pyproject_requires = build_requires + if build_backend and not self.pyproject_backend: + self.pyproject_backend = build_backend hashed_loc = hashlib.sha256(loc.encode("utf-8")).hexdigest() hashed_name = hashed_loc[-7:] - if not name or name == "UNKNOWN": + if not name or name.lower() == "unknown": self._has_hashed_name = True name = hashed_name - if self.link and not self._has_hashed_name: - self.link = Link("{0}#egg={1}".format(self.link.url, name)) + else: + self._has_hashed_name = False + name_in_link = getattr(self.link, "egg_fragment", "") if self.link else "" + if not self._has_hashed_name and name_in_link != name: + self.link = create_link("{0}#egg={1}".format(self.link.url, name)) return name @link.default def get_link(self): target = "{0}".format(self.uri) - if hasattr(self, "name"): + if hasattr(self, "name") and not self._has_hashed_name: target = "{0}#egg={1}".format(target, self.name) - link = Link(target) + link = create_link(target) return link @req.default def get_requirement(self): - req = init_requirement(canonicalize_name(self.name)) + req = init_requirement(normalize_name(self.name)) req.editable = False req.line = self.link.url_without_fragment if self.path and self.link and self.link.scheme.startswith("file"): req.local_file = True req.path = self.path - req.url = None - self._uri_scheme = "file" + if self.editable: + req.url = None + else: + req.url = self.link.url_without_fragment else: req.local_file = False req.path = None @@ -322,9 +380,13 @@ class FileRequirement(BaseRequirement): for scheme in ("http", "https", "ftp", "ftps", "uri") ) and (self.link.is_artifact or self.link.is_wheel) - and not self.req.editable + and not self.editable ) + @property + def is_direct_url(self): + return self.is_remote_artifact + @property def formatted_path(self): if self.path: @@ -334,6 +396,97 @@ class FileRequirement(BaseRequirement): return path.as_posix() return + @classmethod + def create( + cls, path=None, uri=None, editable=False, extras=None, link=None, vcs_type=None, + name=None, req=None, line=None, uri_scheme=None, setup_path=None, relpath=None + ): + if relpath and not path: + path = relpath + if not path and uri and link.scheme == "file": + path = os.path.abspath(pip_shims.shims.url_to_path(unquote(uri))) + try: + path = get_converted_relative_path(path) + except ValueError: # Vistir raises a ValueError if it can't make a relpath + path = path + if line and not (uri_scheme and uri and link): + vcs_type, uri_scheme, relpath, path, uri, link = cls.get_link_from_line(line) + if not uri_scheme: + uri_scheme = "path" if path else "file" + if path and not uri: + uri = unquote(pip_shims.shims.path_to_url(os.path.abspath(path))) + if not link: + link = create_link(uri) + if not uri: + uri = unquote(link.url_without_fragment) + if not extras: + extras = [] + pyproject_path = None + if path is not None: + pyproject_requires = get_pyproject(os.path.abspath(path)) + pyproject_backend = None + pyproject_requires = None + if pyproject_requires is not None: + pyproject_requires, pyproject_backend = pyproject_requires + if path: + pyproject_path = Path(path).joinpath("pyproject.toml") + if not pyproject_path.exists(): + pyproject_path = None + if not setup_path and path is not None: + setup_path = Path(path).joinpath("setup.py") + if setup_path and isinstance(setup_path, Path): + setup_path = setup_path.as_posix() + creation_kwargs = { + "editable": editable, + "extras": extras, + "pyproject_path": pyproject_path, + "setup_path": setup_path if setup_path else None, + "uri_scheme": uri_scheme, + "link": link, + "uri": uri, + "pyproject_requires": pyproject_requires, + "pyproject_backend": pyproject_backend + } + if vcs_type: + creation_kwargs["vcs_type"] = vcs_type + _line = None + if not name: + _line = unquote(link.url_without_fragment) if link.url else uri + if editable: + ireq = pip_shims.shims.install_req_from_editable(_line) + else: + _line = path if (uri_scheme and uri_scheme == "path") else _line + ireq = pip_shims.shims.install_req_from_line(_line) + setup_info = SetupInfo.from_ireq(ireq) + setupinfo_dict = setup_info.as_dict() + setup_name = setupinfo_dict.get("name", None) + if setup_name: + name = setup_name + build_requires = setupinfo_dict.get("build_requires", []) + build_backend = setupinfo_dict.get("build_backend", []) + if not creation_kwargs.get("pyproject_requires") and build_requires: + creation_kwargs["pyproject_requires"] = build_requires + if not creation_kwargs.get("pyproject_backend") and build_backend: + creation_kwargs["pyproject_backend"] = build_backend + creation_kwargs["setup_info"] = setup_info + if path or relpath: + creation_kwargs["path"] = relpath if relpath else path + if req: + creation_kwargs["req"] = req + if creation_kwargs.get("req") and line and not getattr(creation_kwargs["req"], "line", None): + creation_kwargs["req"].line = line + if name: + creation_kwargs["name"] = name + cls_inst = cls(**creation_kwargs) + if not _line: + if editable and uri_scheme == "path": + _line = relpath if relpath else path + else: + _line = unquote(cls_inst.link.url_without_fragment) or cls_inst.uri + _line = "{0}#egg={1}".format(line, cls_inst.name) if not cls_inst._has_hashed_name else _line + cls_inst.req.line = line if line else _line + return cls_inst + @classmethod def from_line(cls, line): line = line.strip('"').strip("'") @@ -342,12 +495,19 @@ class FileRequirement(BaseRequirement): editable = line.startswith("-e ") line = line.split(" ", 1)[1] if editable else line setup_path = None + name = None + req = None if not any([is_installable_file(line), is_valid_url(line), is_file_url(line)]): - raise RequirementError( - "Supplied requirement is not installable: {0!r}".format(line) - ) + try: + req = init_requirement(line) + except Exception: + raise RequirementError( + "Supplied requirement is not installable: {0!r}".format(line) + ) + else: + name = getattr(req, "name", None) + line = getattr(req, "url", None) vcs_type, prefer, relpath, path, uri, link = cls.get_link_from_line(line) - setup_path = Path(path) / "setup.py" if path else None arg_dict = { "path": relpath if relpath else path, "uri": unquote(link.url_without_fragment), @@ -355,13 +515,17 @@ class FileRequirement(BaseRequirement): "editable": editable, "setup_path": setup_path, "uri_scheme": prefer, + "line": line } if link and link.is_wheel: + from pip_shims import Wheel + arg_dict["name"] = Wheel(link.filename).name + elif name: + arg_dict["name"] = name elif link.egg_fragment: arg_dict["name"] = link.egg_fragment - created = cls(**arg_dict) - return created + return cls.create(**arg_dict) @classmethod def from_pipfile(cls, name, pipfile): @@ -395,9 +559,8 @@ class FileRequirement(BaseRequirement): uri_scheme = "file" if not uri: - uri = path_to_url(path) - link = Link(uri) - + uri = pip_shims.shims.path_to_url(path) + link = create_link(uri) arg_dict = { "name": name, "path": path, @@ -406,16 +569,17 @@ class FileRequirement(BaseRequirement): "link": link, "uri_scheme": uri_scheme, } - return cls(**arg_dict) + if link.scheme != "file" and not pipfile.get("editable", False): + arg_dict["line"] = "{0}@ {1}".format(name, link.url_without_fragment) + return cls.create(**arg_dict) @property def line_part(self): - if self._uri_scheme and self._uri_scheme == 'path': + if self._uri_scheme and self._uri_scheme == "path": + # We may need any one of these for passing to pip seed = self.path or unquote(self.link.url_without_fragment) or self.uri - elif ( - (self._uri_scheme and self._uri_scheme == "file") - or ((self.link.is_artifact or self.link.is_wheel) - and self.link.url) + elif (self._uri_scheme and self._uri_scheme == "file") or ( + (self.link.is_artifact or self.link.is_wheel) and self.link.url ): seed = unquote(self.link.url_without_fragment) or self.uri # add egg fragments to remote artifacts (valid urls only) @@ -426,12 +590,15 @@ class FileRequirement(BaseRequirement): @property def pipfile_part(self): - pipfile_dict = attr.asdict(self, filter=filter_none).copy() + excludes = [ + "_base_line", "_has_hashed_name", "setup_path", "pyproject_path", + "pyproject_requires", "pyproject_backend", "setup_info" + ] + filter_func = lambda k, v: bool(v) is True and k.name not in excludes + pipfile_dict = attr.asdict(self, filter=filter_func).copy() name = pipfile_dict.pop("name") if "_uri_scheme" in pipfile_dict: pipfile_dict.pop("_uri_scheme") - if "setup_path" in pipfile_dict: - pipfile_dict.pop("setup_path") # For local paths and remote installable artifacts (zipfiles, etc) collision_keys = {"file", "uri", "path"} if self._uri_scheme: @@ -473,14 +640,19 @@ class FileRequirement(BaseRequirement): return {name: pipfile_dict} -@attr.s +@attr.s(slots=True) class VCSRequirement(FileRequirement): + #: Whether the repository is editable editable = attr.ib(default=None) + #: URI for the repository uri = attr.ib(default=None) + #: path to the repository, if it's local path = attr.ib(default=None, validator=attr.validators.optional(validate_path)) + #: vcs type, i.e. git/hg/svn vcs = attr.ib(validator=attr.validators.optional(validate_vcs), default=None) - # : vcs reference name (branch / commit / tag) + #: vcs reference name (branch / commit / tag) ref = attr.ib(default=None) + #: Subdirectory to use for installation if applicable subdirectory = attr.ib(default=None) _repo = attr.ib(default=None) _base_line = attr.ib(default=None) @@ -491,7 +663,7 @@ class VCSRequirement(FileRequirement): def __attrs_post_init__(self): if not self.uri: if self.path: - self.uri = path_to_url(self.path) + self.uri = pip_shims.shims.path_to_url(self.path) split = urllib_parse.urlsplit(self.uri) scheme, rest = split[0], split[1:] vcs_type = "" @@ -504,14 +676,14 @@ class VCSRequirement(FileRequirement): @link.default def get_link(self): - uri = self.uri if self.uri else path_to_url(self.path) + uri = self.uri if self.uri else pip_shims.shims.path_to_url(self.path) return build_vcs_link( self.vcs, add_ssh_scheme_to_git_uri(uri), name=self.name, ref=self.ref, subdirectory=self.subdirectory, - extras=self.extras + extras=self.extras, ) @name.default @@ -541,7 +713,8 @@ class VCSRequirement(FileRequirement): ) req = init_requirement(canonicalize_name(self.name)) req.editable = self.editable - req.url = self.uri + if not getattr(req, "url") and self.uri: + req.url = self.uri req.line = self.link.url if self.ref: req.revision = self.ref @@ -574,29 +747,52 @@ class VCSRequirement(FileRequirement): return self._repo def get_checkout_dir(self, src_dir=None): - src_dir = os.environ.get('PIP_SRC', None) if not src_dir else src_dir + src_dir = os.environ.get("PIP_SRC", None) if not src_dir else src_dir checkout_dir = None if self.is_local: path = self.path if not path: - path = url_to_path(self.uri) + path = pip_shims.shims.url_to_path(self.uri) if path and os.path.exists(path): checkout_dir = os.path.abspath(path) return checkout_dir return os.path.join(create_tracked_tempdir(prefix="requirementslib"), self.name) def get_vcs_repo(self, src_dir=None): + from .vcs import VCSRepository + checkout_dir = self.get_checkout_dir(src_dir=src_dir) - url = "{0}#egg={1}".format(self.vcs_uri, self.name) + link = build_vcs_link( + self.vcs, + self.uri, + name=self.name, + ref=self.ref, + subdirectory=self.subdirectory, + extras=self.extras, + ) vcsrepo = VCSRepository( - url=url, + url=link.url, name=self.name, ref=self.ref if self.ref else None, checkout_directory=checkout_dir, - vcs_type=self.vcs + vcs_type=self.vcs, + subdirectory=self.subdirectory, ) if not self.is_local: vcsrepo.obtain() + pyproject_info = None + if self.subdirectory: + self.setup_path = os.path.join(checkout_dir, self.subdirectory, "setup.py") + self.pyproject_path = os.path.join(checkout_dir, self.subdirectory, "pyproject.toml") + pyproject_info = get_pyproject(os.path.join(checkout_dir, self.subdirectory)) + else: + self.setup_path = os.path.join(checkout_dir, "setup.py") + self.pyproject_path = os.path.join(checkout_dir, "pyproject.toml") + pyproject_info = get_pyproject(checkout_dir) + if pyproject_info is not None: + pyproject_requires, pyproject_backend = pyproject_info + self.pyproject_requires = pyproject_requires + self.pyproject_backend = pyproject_backend return vcsrepo def get_commit_hash(self): @@ -614,20 +810,21 @@ class VCSRequirement(FileRequirement): if not self.is_local and ref is not None: self.repo.checkout_ref(ref) repo_hash = self.repo.get_commit_hash() + self.req.revision = repo_hash return repo_hash @contextmanager def locked_vcs_repo(self, src_dir=None): + if not src_dir: + src_dir = create_tracked_tempdir(prefix="requirementslib-", suffix="-src") vcsrepo = self.get_vcs_repo(src_dir=src_dir) - if self.ref and not self.is_local: - vcsrepo.checkout_ref(self.ref) - self.ref = self.get_commit_hash() - self.req.revision = self.ref + self.req.revision = vcsrepo.get_commit_hash() # Remove potential ref in the end of uri after ref is parsed if "@" in self.link.show_url and "@" in self.uri: uri, ref = self.uri.rsplit("@", 1) - if ref in self.ref: + checkout = self.req.revision + if checkout and ref in checkout: self.uri = uri yield vcsrepo @@ -638,7 +835,16 @@ class VCSRequirement(FileRequirement): creation_args = {} pipfile_keys = [ k - for k in ("ref", "vcs", "subdirectory", "path", "editable", "file", "uri", "extras") + for k in ( + "ref", + "vcs", + "subdirectory", + "path", + "editable", + "file", + "uri", + "extras", + ) + VCS_LIST if k in pipfile ] @@ -651,13 +857,20 @@ class VCSRequirement(FileRequirement): creation_args["vcs"] = key target = pipfile.get(key) drive, path = os.path.splitdrive(target) - if not drive and not os.path.exists(target) and (is_valid_url(target) or - is_file_url(target) or target.startswith('git@')): + if ( + not drive + and not os.path.exists(target) + and ( + is_valid_url(target) + or is_file_url(target) + or target.startswith("git@") + ) + ): creation_args["uri"] = target else: creation_args["path"] = target if os.path.isabs(target): - creation_args["uri"] = path_to_url(target) + creation_args["uri"] = pip_shims.shims.path_to_url(target) else: creation_args[key] = pipfile.get(key) creation_args["name"] = name @@ -671,15 +884,15 @@ class VCSRequirement(FileRequirement): line = line.split(" ", 1)[1] vcs_type, prefer, relpath, path, uri, link = cls.get_link_from_line(line) if not extras and link.egg_fragment: - name, extras = _strip_extras(link.egg_fragment) + name, extras = pip_shims.shims._strip_extras(link.egg_fragment) if extras: extras = parse_extras(extras) else: name = link.egg_fragment subdirectory = link.subdirectory_fragment ref = None - if "@" in link.show_url and "@" in uri: - uri, ref = uri.rsplit("@", 1) + if "@" in link.path and "@" in uri: + uri, _, ref = uri.rpartition("@") if relpath and "@" in relpath: relpath, ref = relpath.rsplit("@", 1) return cls( @@ -692,7 +905,7 @@ class VCSRequirement(FileRequirement): editable=editable, uri=uri, extras=extras, - base_line=line + base_line=line, ) @property @@ -702,7 +915,11 @@ class VCSRequirement(FileRequirement): base_link = self.link if not self.link: base_link = self.get_link() - final_format = "{{0}}#egg={0}".format(base_link.egg_fragment) if base_link.egg_fragment else "{0}" + final_format = ( + "{{0}}#egg={0}".format(base_link.egg_fragment) + if base_link.egg_fragment + else "{0}" + ) base = final_format.format(self.vcs_uri) elif self._base_line: base = self._base_line @@ -731,12 +948,15 @@ class VCSRequirement(FileRequirement): @property def pipfile_part(self): - excludes = ["_repo", "_base_line"] + excludes = [ + "_repo", "_base_line", "setup_path", "_has_hashed_name", "pyproject_path", + "pyproject_requires", "pyproject_backend", "setup_info" + ] filter_func = lambda k, v: bool(v) is True and k.name not in excludes pipfile_dict = attr.asdict(self, filter=filter_func).copy() if "vcs" in pipfile_dict: pipfile_dict = self._choose_vcs_source(pipfile_dict) - name, _ = _strip_extras(pipfile_dict.pop("name")) + name, _ = pip_shims.shims._strip_extras(pipfile_dict.pop("name")) return {name: pipfile_dict} @@ -744,7 +964,7 @@ class VCSRequirement(FileRequirement): class Requirement(object): name = attr.ib() vcs = attr.ib(default=None, validator=attr.validators.optional(validate_vcs)) - req = attr.ib(default=None, validator=optional_instance_of(BaseRequirement)) + req = attr.ib(default=None) markers = attr.ib(default=None) specifiers = attr.ib(validator=attr.validators.optional(validate_specifiers)) index = attr.ib(default=None) @@ -783,7 +1003,9 @@ class Requirement(object): @property def extras_as_pip(self): if self.extras: - return "[{0}]".format(",".join(sorted([extra.lower() for extra in self.extras]))) + return "[{0}]".format( + ",".join(sorted([extra.lower() for extra in self.extras])) + ) return "" @@ -823,7 +1045,7 @@ class Requirement(object): @classmethod def from_line(cls, line): - if isinstance(line, InstallRequirement): + if isinstance(line, pip_shims.shims.InstallRequirement): line = format_requirement(line) hashes = None if "--hash=" in line: @@ -832,8 +1054,7 @@ class Requirement(object): editable = line.startswith("-e ") line = line.split(" ", 1)[1] if editable else line line, markers = split_markers_from_line(line) - line, extras = _strip_extras(line) - specifiers = '' + line, extras = pip_shims.shims._strip_extras(line) if extras: extras = parse_extras(extras) line = line.strip('"').strip("'").strip() @@ -842,7 +1063,12 @@ class Requirement(object): # Installable local files and installable non-vcs urls are handled # as files, generally speaking line_is_vcs = is_vcs(line) - if is_installable_file(line) or ((is_file_url(line) or is_valid_url(line)) and not line_is_vcs): + # check for pep-508 compatible requirements + name, _, possible_url = line.partition("@") + if is_installable_file(line) or ( + (is_valid_url(possible_url) or is_file_url(line) or is_valid_url(line)) and + not (line_is_vcs or is_vcs(possible_url)) + ): r = FileRequirement.from_line(line_with_prefix) elif line_is_vcs: r = VCSRequirement.from_line(line_with_prefix, extras=extras) @@ -860,9 +1086,8 @@ class Requirement(object): spec_idx = min((line.index(match) for match in spec_matches)) name = line[:spec_idx] version = line[spec_idx:] - specifiers = version if not extras: - name, extras = _strip_extras(name) + name, extras = pip_shims.shims._strip_extras(name) if extras: extras = parse_extras(extras) if version: @@ -871,7 +1096,7 @@ class Requirement(object): req_markers = None if markers: req_markers = PackagingRequirement("fakepkg; {0}".format(markers)) - r.req.marker = getattr(req_markers, "marker", None) + r.req.marker = getattr(req_markers, "marker", None) if req_markers else None r.req.local_file = getattr(r.req, "local_file", False) name = getattr(r.req, "name", None) if not name: @@ -897,7 +1122,22 @@ class Requirement(object): args["extras"] = sorted(dedup([extra.lower() for extra in r.extras])) if hashes: args["hashes"] = hashes - return cls(**args) + cls_inst = cls(**args) + if not cls_inst.is_named and not cls_inst.editable and not name: + if cls_inst.is_vcs: + ireq = pip_shims.shims.install_req_from_req(cls_inst.as_line(include_hashes=False)) + info = SetupInfo.from_ireq(ireq) + if info is not None: + info_dict = info.as_dict() + cls_inst.req.setup_info = info + else: + info_dict = {} + else: + info_dict = cls_inst.run_requires() + found_name = info_dict.get("name", old_name) + if old_name != found_name: + cls_inst.req.req.line.replace(old_name, found_name) + return cls_inst @classmethod def from_ireq(cls, ireq): @@ -905,12 +1145,14 @@ class Requirement(object): @classmethod def from_metadata(cls, name, version, extras, markers): - return cls.from_ireq(make_install_requirement( - name, version, extras=extras, markers=markers, - )) + return cls.from_ireq( + make_install_requirement(name, version, extras=extras, markers=markers) + ) @classmethod def from_pipfile(cls, name, pipfile): + from .markers import PipenvMarkers + _pipfile = {} if hasattr(pipfile, "keys"): _pipfile = dict(pipfile).copy() @@ -931,7 +1173,9 @@ class Requirement(object): r.req.marker = getattr(req_markers, "marker", None) r.req.specifier = SpecifierSet(_pipfile["version"]) extras = _pipfile.get("extras") - r.req.extras = sorted(dedup([extra.lower() for extra in extras])) if extras else [] + r.req.extras = ( + sorted(dedup([extra.lower() for extra in extras])) if extras else [] + ) args = { "name": r.name, "vcs": vcs, @@ -946,9 +1190,31 @@ class Requirement(object): cls_inst = cls(**args) if cls_inst.is_named: cls_inst.req.req.line = cls_inst.as_line() + old_name = cls_inst.req.req.name or cls_inst.req.name + if not cls_inst.is_named and not cls_inst.editable and not name: + if cls_inst.is_vcs: + ireq = pip_shims.shims.install_req_from_req(cls_inst.as_line(include_hashes=False)) + info = SetupInfo.from_ireq(ireq) + if info is not None: + info_dict = info.as_dict() + cls_inst.req.setup_info = info + else: + info_dict = {} + else: + info_dict = cls_inst.run_requires() + found_name = info_dict.get("name", old_name) + if old_name != found_name: + cls_inst.req.req.line.replace(old_name, found_name) return cls_inst - def as_line(self, sources=None, include_hashes=True, include_extras=True, as_list=False): + def as_line( + self, + sources=None, + include_hashes=True, + include_extras=True, + include_markers=True, + as_list=False, + ): """Format this requirement as a line in requirements.txt. If ``sources`` provided, it should be an sequence of mappings, containing @@ -967,7 +1233,7 @@ class Requirement(object): self.req.line_part, self.extras_as_pip if include_extras else "", self.specifiers if include_specifiers else "", - self.markers_as_pip, + self.markers_as_pip if include_markers else "", ] if as_list: # This is used for passing to a subprocess call @@ -997,25 +1263,28 @@ class Requirement(object): def get_markers(self): markers = self.markers if markers: - fake_pkg = PackagingRequirement('fakepkg; {0}'.format(markers)) + fake_pkg = PackagingRequirement("fakepkg; {0}".format(markers)) markers = fake_pkg.markers return markers def get_specifier(self): - return Specifier(self.specifiers) + try: + return Specifier(self.specifiers) + except InvalidSpecifier: + return LegacySpecifier(self.specifiers) def get_version(self): - return parse_version(self.get_specifier().version) + return pip_shims.shims.parse_version(self.get_specifier().version) def get_requirement(self): req_line = self.req.req.line - if req_line.startswith('-e '): + if req_line.startswith("-e "): _, req_line = req_line.split(" ", 1) req = init_requirement(self.name) req.line = req_line - req.specifier = SpecifierSet(self.specifiers if self.specifiers else '') + req.specifier = SpecifierSet(self.specifiers if self.specifiers else "") if self.is_vcs or self.is_file_or_url: - req.url = self.req.link.url_without_fragment + req.url = getattr(self.req.req, "url", self.req.link.url_without_fragment) req.marker = self.get_markers() req.extras = set(self.extras) if self.extras else set() return req @@ -1024,6 +1293,10 @@ class Requirement(object): def constraint_line(self): return self.as_line() + @property + def is_direct_url(self): + return self.is_file_or_url and self.req.is_direct_url + def as_pipfile(self): good_keys = ( "hashes", @@ -1039,8 +1312,8 @@ class Requirement(object): if k in good_keys } name = self.name - if 'markers' in req_dict and req_dict['markers']: - req_dict['markers'] = req_dict['markers'].replace('"', "'") + if "markers" in req_dict and req_dict["markers"]: + req_dict["markers"] = req_dict["markers"].replace('"', "'") base_dict = { k: v for k, v in self.req.pipfile_part[name].items() @@ -1069,11 +1342,11 @@ class Requirement(object): ireq_line = self.as_line(include_hashes=False) if self.editable or self.req.editable: if ireq_line.startswith("-e "): - ireq_line = ireq_line[len("-e "):] - with ensure_setup_py(self.req.path): - ireq = InstallRequirement.from_editable(ireq_line) + ireq_line = ireq_line[len("-e ") :] + with ensure_setup_py(self.req.setup_path): + ireq = pip_shims.shims.install_req_from_editable(ireq_line) else: - ireq = InstallRequirement.from_line(ireq_line) + ireq = pip_shims.shims.install_req_from_line(ireq_line) if not getattr(ireq, "req", None): ireq.req = self.req.req else: @@ -1100,12 +1373,13 @@ class Requirement(object): :return: A set of requirement strings of the dependencies of this requirement. :rtype: set(str) """ + + from .dependencies import get_dependencies + if not sources: - sources = [{ - 'name': 'pypi', - 'url': 'https://pypi.org/simple', - 'verify_ssl': True, - }] + sources = [ + {"name": "pypi", "url": "https://pypi.org/simple", "verify_ssl": True} + ] return get_dependencies(self.as_ireq(), sources=sources) def get_abstract_dependencies(self, sources=None): @@ -1119,17 +1393,27 @@ class Requirement(object): :rtype: list[ :class:`~requirementslib.models.dependency.AbstractDependency` ] """ + from .dependencies import ( + AbstractDependency, + get_dependencies, + get_abstract_dependencies, + ) + if not self.abstract_dep: - parent = getattr(self, 'parent', None) + parent = getattr(self, "parent", None) self.abstract_dep = AbstractDependency.from_requirement(self, parent=parent) if not sources: - sources = [{'url': 'https://pypi.org/simple', 'name': 'pypi', 'verify_ssl': True},] + sources = [ + {"url": "https://pypi.org/simple", "name": "pypi", "verify_ssl": True} + ] if is_pinned_requirement(self.ireq): deps = self.get_dependencies() else: ireq = sorted(self.find_all_matches(), key=lambda k: k.version) deps = get_dependencies(ireq.pop(), sources=sources) - return get_abstract_dependencies(deps, sources=sources, parent=self.abstract_dep) + return get_abstract_dependencies( + deps, sources=sources, parent=self.abstract_dep + ) def find_all_matches(self, sources=None, finder=None): """Find all matching candidates for the current requirement. @@ -1141,10 +1425,33 @@ class Requirement(object): :return: A list of Installation Candidates :rtype: list[ :class:`~pip._internal.index.InstallationCandidate` ] """ + + from .dependencies import get_finder, find_all_matches + if not finder: finder = get_finder(sources=sources) return find_all_matches(finder, self.as_ireq()) + def run_requires(self, sources=None, finder=None): + if self.req and self.req.setup_info is not None: + info_dict = self.req.setup_info.as_dict() + else: + from .setup_info import SetupInfo + if not finder: + from .dependencies import get_finder + finder = get_finder(sources=sources) + info = SetupInfo.from_requirement(self, finder=finder) + if info is None: + return {} + info_dict = info.get_info() + if self.req and not self.req.setup_info: + self.req.setup_info = info + if self.req._has_hashed_name and info_dict.get("name"): + self.req.name = self.name = info_dict["name"] + if self.req.req.name != info_dict["name"]: + self.req.req.name = info_dict["name"] + return info_dict + def merge_markers(self, markers): if not isinstance(markers, Marker): markers = Marker(markers) diff --git a/pipenv/vendor/requirementslib/models/resolvers.py b/pipenv/vendor/requirementslib/models/resolvers.py index da6d0dda..bd773ba6 100644 --- a/pipenv/vendor/requirementslib/models/resolvers.py +++ b/pipenv/vendor/requirementslib/models/resolvers.py @@ -4,11 +4,9 @@ from contextlib import contextmanager import attr import six -from pip_shims.shims import VcsSupport, Wheel +from pip_shims.shims import Wheel -from ..utils import log from .cache import HashCache -from .dependencies import AbstractDependency, find_all_matches, get_finder from .utils import format_requirement, is_pinned_requirement, version_from_ireq @@ -41,6 +39,7 @@ class DependencyResolver(object): @classmethod def create(cls, finder=None, allow_prereleases=False, get_all_hashes=True): if not finder: + from .dependencies import get_finder finder_args = [] if allow_prereleases: finder_args.append('--pre') @@ -140,6 +139,8 @@ class DependencyResolver(object): # Coerce input into AbstractDependency instances. # We accept str, Requirement, and AbstractDependency as input. + from .dependencies import AbstractDependency + from ..utils import log for dep in root_nodes: if isinstance(dep, six.string_types): dep = AbstractDependency.from_string(dep) @@ -183,6 +184,7 @@ class DependencyResolver(object): def get_hashes_for_one(self, ireq): if not self.finder: + from .dependencies import get_finder finder_args = [] if self.allow_prereleases: finder_args.append('--pre') @@ -191,6 +193,7 @@ class DependencyResolver(object): if ireq.editable: return set() + from pip_shims import VcsSupport vcs = VcsSupport() if ireq.link and ireq.link.scheme in vcs.all_schemes and 'ssh' in ireq.link.scheme: return set() @@ -201,6 +204,7 @@ class DependencyResolver(object): matching_candidates = set() with self.allow_all_wheels(): + from .dependencies import find_all_matches matching_candidates = ( find_all_matches(self.finder, ireq, pre=self.allow_prereleases) ) diff --git a/pipenv/vendor/requirementslib/models/setup_info.py b/pipenv/vendor/requirementslib/models/setup_info.py new file mode 100644 index 00000000..006ee609 --- /dev/null +++ b/pipenv/vendor/requirementslib/models/setup_info.py @@ -0,0 +1,414 @@ +# -*- coding=utf-8 -*- +import contextlib +import os +import sys + +import attr +import packaging.version +import packaging.specifiers +import packaging.utils + +try: + from setuptools.dist import distutils +except ImportError: + import distutils + +from appdirs import user_cache_dir +from six.moves import configparser +from six.moves.urllib.parse import unquote +from vistir.compat import Path +from vistir.contextmanagers import cd +from vistir.misc import run +from vistir.path import create_tracked_tempdir, ensure_mkdir_p, mkdir_p + +from .utils import init_requirement, get_pyproject + +try: + from os import scandir +except ImportError: + from scandir import scandir + + +CACHE_DIR = os.environ.get("PIPENV_CACHE_DIR", user_cache_dir("pipenv")) + +# The following are necessary for people who like to use "if __name__" conditionals +# in their setup.py scripts +_setup_stop_after = None +_setup_distribution = None + + +@contextlib.contextmanager +def _suppress_distutils_logs(): + """Hack to hide noise generated by `setup.py develop`. + + There isn't a good way to suppress them now, so let's monky-patch. + See https://bugs.python.org/issue25392. + """ + + f = distutils.log.Log._log + + def _log(log, level, msg, args): + if level >= distutils.log.ERROR: + f(log, level, msg, args) + + distutils.log.Log._log = _log + yield + distutils.log.Log._log = f + + +@ensure_mkdir_p(mode=0o775) +def _get_src_dir(): + src = os.environ.get("PIP_SRC") + if src: + return src + virtual_env = os.environ.get("VIRTUAL_ENV") + if virtual_env: + return os.path.join(virtual_env, "src") + return os.path.join(os.getcwd(), "src") # Match pip's behavior. + + +def _prepare_wheel_building_kwargs(ireq): + download_dir = os.path.join(CACHE_DIR, "pkgs") + mkdir_p(download_dir) + + wheel_download_dir = os.path.join(CACHE_DIR, "wheels") + mkdir_p(wheel_download_dir) + + if ireq.source_dir is not None: + src_dir = ireq.source_dir + elif ireq.editable: + src_dir = _get_src_dir() + else: + src_dir = create_tracked_tempdir(prefix="reqlib-src") + + # This logic matches pip's behavior, although I don't fully understand the + # intention. I guess the idea is to build editables in-place, otherwise out + # of the source tree? + if ireq.editable: + build_dir = src_dir + else: + build_dir = create_tracked_tempdir(prefix="reqlib-build") + + return { + "build_dir": build_dir, + "src_dir": src_dir, + "download_dir": download_dir, + "wheel_download_dir": wheel_download_dir, + } + + +def iter_egginfos(path, pkg_name=None): + for entry in scandir(path): + if entry.is_dir(): + if not entry.name.endswith("egg-info"): + for dir_entry in iter_egginfos(entry.path, pkg_name=pkg_name): + yield dir_entry + elif pkg_name is None or entry.name.startswith(pkg_name): + yield entry + + +def find_egginfo(target, pkg_name=None): + egg_dirs = (egg_dir for egg_dir in iter_egginfos(target, pkg_name=pkg_name)) + if pkg_name: + yield next(iter(egg_dirs), None) + else: + for egg_dir in egg_dirs: + yield egg_dir + + +def get_metadata(path, pkg_name=None): + if pkg_name: + pkg_name = packaging.utils.canonicalize_name(pkg_name) + egg_dir = next(iter(find_egginfo(path, pkg_name=pkg_name)), None) + if egg_dir is not None: + import pkg_resources + + egg_dir = os.path.abspath(egg_dir.path) + base_dir = os.path.dirname(egg_dir) + path_metadata = pkg_resources.PathMetadata(base_dir, egg_dir) + dist = next( + iter(pkg_resources.distributions_from_metadata(path_metadata.egg_info)), + None, + ) + if dist: + requires = dist.requires() + dep_map = dist._build_dep_map() + deps = [] + for k in dep_map.keys(): + if k is None: + deps.extend(dep_map.get(k)) + continue + else: + _deps = dep_map.get(k) + k = k.replace(":", "; ") + _deps = [ + pkg_resources.Requirement.parse("{0}{1}".format(str(req), k)) + for req in _deps + ] + deps.extend(_deps) + return { + "name": dist.project_name, + "version": dist.version, + "requires": requires, + } + + +@attr.s(slots=True) +class SetupInfo(object): + name = attr.ib(type=str, default=None) + base_dir = attr.ib(type=Path, default=None) + version = attr.ib(type=packaging.version.Version, default=None) + extras = attr.ib(type=list, default=attr.Factory(list)) + requires = attr.ib(type=dict, default=attr.Factory(dict)) + build_requires = attr.ib(type=list, default=attr.Factory(list)) + build_backend = attr.ib(type=list, default=attr.Factory(list)) + setup_requires = attr.ib(type=dict, default=attr.Factory(list)) + python_requires = attr.ib(type=packaging.specifiers.SpecifierSet, default=None) + extras = attr.ib(type=dict, default=attr.Factory(dict)) + setup_cfg = attr.ib(type=Path, default=None) + setup_py = attr.ib(type=Path, default=None) + pyproject = attr.ib(type=Path, default=None) + ireq = attr.ib(default=None) + extra_kwargs = attr.ib(default=attr.Factory(dict), type=dict) + + def parse_setup_cfg(self): + if self.setup_cfg is not None and self.setup_cfg.exists(): + default_opts = { + "metadata": {"name": "", "version": ""}, + "options": { + "install_requires": "", + "python_requires": "", + "build_requires": "", + "setup_requires": "", + "extras": "", + }, + } + parser = configparser.ConfigParser(default_opts) + parser.read(self.setup_cfg.as_posix()) + if parser.has_option("metadata", "name"): + name = parser.get("metadata", "name") + if not self.name and name is not None: + self.name = name + if parser.has_option("metadata", "version"): + version = parser.get("metadata", "version") + if not self.version and version is not None: + self.version = version + if parser.has_option("options", "install_requires"): + self.requires.update( + { + dep.strip(): init_requirement(dep.strip()) + for dep in parser.get("options", "install_requires").split("\n") + if dep + } + ) + if parser.has_option("options", "python_requires"): + python_requires = parser.get("options", "python_requires") + if python_requires and not self.python_requires: + self.python_requires = python_requires + if parser.has_option("options", "extras_require"): + self.extras.update( + { + section: [ + dep.strip() + for dep in parser.get( + "options.extras_require", section + ).split("\n") + if dep + ] + for section in parser.options("options.extras_require") + } + ) + + def run_setup(self): + if self.setup_py is not None and self.setup_py.exists(): + target_cwd = self.setup_py.parent.as_posix() + with cd(target_cwd), _suppress_distutils_logs(): + script_name = self.setup_py.as_posix() + args = ["egg_info", "--egg-base", self.base_dir] + g = {"__file__": script_name, "__name__": "__main__"} + local_dict = {} + if sys.version_info < (3, 5): + save_argv = sys.argv + else: + save_argv = sys.argv.copy() + # This is for you, Hynek + # see https://github.com/hynek/environ_config/blob/69b1c8a/setup.py + try: + global _setup_distribution, _setup_stop_after + _setup_stop_after = "run" + sys.argv[0] = script_name + sys.argv[1:] = args + with open(script_name, 'rb') as f: + if sys.version_info < (3, 5): + exec(f.read(), g, local_dict) + else: + exec(f.read(), g) + # We couldn't import everything needed to run setup + except NameError: + python = os.environ.get('PIP_PYTHON_PATH', sys.executable) + out, _ = run([python, "setup.py"] + args, cwd=target_cwd, block=True, + combine_stderr=False, return_object=False, nospin=True) + finally: + _setup_stop_after = None + sys.argv = save_argv + dist = _setup_distribution + if not dist: + self.get_egg_metadata() + return + + name = dist.get_name() + if name: + self.name = name + if dist.python_requires and not self.python_requires: + self.python_requires = packaging.specifiers.SpecifierSet( + dist.python_requires + ) + if dist.extras_require and not self.extras: + self.extras = dist.extras_require + install_requires = dist.get_requires() + if not install_requires: + install_requires = dist.install_requires + if install_requires and not self.requires: + requirements = [init_requirement(req) for req in install_requires] + self.requires.update({req.key: req for req in requirements}) + if dist.setup_requires and not self.setup_requires: + self.setup_requires = dist.setup_requires + if not self.version: + self.version = dist.get_version() + + def get_egg_metadata(self): + if self.setup_py is not None and self.setup_py.exists(): + metadata = get_metadata(self.setup_py.parent.as_posix(), pkg_name=self.name) + if metadata: + if not self.name: + self.name = metadata.get("name", self.name) + if not self.version: + self.version = metadata.get("version", self.version) + self.requires.update( + {req.key: req for req in metadata.get("requires", {})} + ) + + def run_pyproject(self): + if self.pyproject and self.pyproject.exists(): + result = get_pyproject(self.pyproject.parent) + if result is not None: + requires, backend = result + if backend: + self.build_backend = backend + if requires and not self.build_requires: + self.build_requires = requires + + def get_info(self): + if self.setup_cfg and self.setup_cfg.exists(): + self.parse_setup_cfg() + if self.setup_py and self.setup_py.exists(): + if not self.requires or not self.name: + try: + self.run_setup() + except Exception: + self.get_egg_metadata() + if not self.requires or not self.name: + self.get_egg_metadata() + + if self.pyproject and self.pyproject.exists(): + self.run_pyproject() + return self.as_dict() + + def as_dict(self): + prop_dict = { + "name": self.name, + "version": self.version, + "base_dir": self.base_dir, + "ireq": self.ireq, + "build_backend": self.build_backend, + "build_requires": self.build_requires, + "requires": self.requires, + "setup_requires": self.setup_requires, + "python_requires": self.python_requires, + "extras": self.extras, + "extra_kwargs": self.extra_kwargs, + "setup_cfg": self.setup_cfg, + "setup_py": self.setup_py, + "pyproject": self.pyproject, + } + return {k: v for k, v in prop_dict.items() if v} + + @classmethod + def from_requirement(cls, requirement, finder=None): + ireq = requirement.as_ireq() + subdir = getattr(requirement.req, "subdirectory", None) + return cls.from_ireq(ireq, subdir=subdir, finder=finder) + + @classmethod + def from_ireq(cls, ireq, subdir=None, finder=None): + import pip_shims.shims + + if ireq.link.is_wheel: + return + if not finder: + from .dependencies import get_finder + + finder = get_finder() + kwargs = _prepare_wheel_building_kwargs(ireq) + ireq.populate_link(finder, False, False) + ireq.ensure_has_source_dir(kwargs["build_dir"]) + if not ( + ireq.editable + and pip_shims.shims.is_file_url(ireq.link) + and not ireq.link.is_artifact + ): + if ireq.is_wheel: + only_download = True + download_dir = kwargs["wheel_download_dir"] + else: + only_download = False + download_dir = kwargs["download_dir"] + ireq_src_dir = None + if ireq.link.scheme == "file": + path = pip_shims.shims.url_to_path(unquote(ireq.link.url_without_fragment)) + if pip_shims.shims.is_installable_dir(path): + ireq_src_dir = path + if not ireq.editable or not (pip_shims.is_file_url(ireq.link) and ireq_src_dir): + pip_shims.shims.unpack_url( + ireq.link, + ireq.source_dir, + download_dir, + only_download=only_download, + session=finder.session, + hashes=ireq.hashes(False), + progress_bar="off", + ) + if ireq.editable: + created = cls.create( + ireq.source_dir, subdirectory=subdir, ireq=ireq, kwargs=kwargs + ) + else: + build_dir = ireq.build_location(kwargs["build_dir"]) + ireq._temp_build_dir.path = kwargs["build_dir"] + created = cls.create( + build_dir, subdirectory=subdir, ireq=ireq, kwargs=kwargs + ) + created.get_info() + return created + + @classmethod + def create(cls, base_dir, subdirectory=None, ireq=None, kwargs=None): + if not base_dir or base_dir is None: + return + + creation_kwargs = {"extra_kwargs": kwargs} + if not isinstance(base_dir, Path): + base_dir = Path(base_dir) + creation_kwargs["base_dir"] = base_dir.as_posix() + pyproject = base_dir.joinpath("pyproject.toml") + + if subdirectory is not None: + base_dir = base_dir.joinpath(subdirectory) + setup_py = base_dir.joinpath("setup.py") + setup_cfg = base_dir.joinpath("setup.cfg") + creation_kwargs["pyproject"] = pyproject + creation_kwargs["setup_py"] = setup_py + creation_kwargs["setup_cfg"] = setup_cfg + if ireq: + creation_kwargs["ireq"] = ireq + return cls(**creation_kwargs) diff --git a/pipenv/vendor/requirementslib/models/utils.py b/pipenv/vendor/requirementslib/models/utils.py index cba63295..0fac2aa3 100644 --- a/pipenv/vendor/requirementslib/models/utils.py +++ b/pipenv/vendor/requirementslib/models/utils.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import +import io import os import sys @@ -9,19 +10,16 @@ from itertools import chain, groupby from operator import attrgetter import six +import tomlkit from attr import validators from first import first from packaging.markers import InvalidMarker, Marker, Op, Value, Variable from packaging.specifiers import InvalidSpecifier, Specifier, SpecifierSet -from packaging.version import parse as parse_version -from packaging.requirements import Requirement as PackagingRequirement -from pkg_resources import Requirement - from vistir.misc import dedup -from pip_shims.shims import InstallRequirement, Link -from ..utils import SCHEME_LIST, VCS_LIST, is_star + +from ..utils import SCHEME_LIST, VCS_LIST, is_star, add_ssh_scheme_to_git_uri HASH_STRING = " --hash={0}" @@ -37,7 +35,13 @@ def optional_instance_of(cls): return validators.optional(validators.instance_of(cls)) +def create_link(link): + from pip_shims import Link + return Link(link) + + def init_requirement(name): + from pkg_resources import Requirement req = Requirement.parse(name) req.vcs = None req.local_file = None @@ -59,6 +63,7 @@ def extras_to_string(extras): def parse_extras(extras_str): """Turn a string of extras into a parsed extras list""" + from pkg_resources import Requirement extras = Requirement.parse("fakepkg{0}".format(extras_to_string(extras_str))).extras return sorted(dedup([extra.lower() for extra in extras])) @@ -92,7 +97,7 @@ def build_vcs_link(vcs, uri, name=None, ref=None, subdirectory=None, extras=None uri = "{0}{1}".format(uri, extras) if subdirectory: uri = "{0}&subdirectory={1}".format(uri, subdirectory) - return Link(uri) + return create_link(uri) def get_version(pipfile_entry): @@ -109,20 +114,40 @@ def get_version(pipfile_entry): return "" -def strip_ssh_from_git_uri(uri): - """Return git+ssh:// formatted URI to git+git@ format""" - if isinstance(uri, six.string_types): - uri = uri.replace("git+ssh://", "git+", 1) - return uri - - -def add_ssh_scheme_to_git_uri(uri): - """Cleans VCS uris from pip format""" - if isinstance(uri, six.string_types): - # Add scheme for parsing purposes, this is also what pip does - if uri.startswith("git+") and "://" not in uri: - uri = uri.replace("git+", "git+ssh://", 1) - return uri +def get_pyproject(path): + from vistir.compat import Path + if not path: + return + if not isinstance(path, Path): + path = Path(path) + if not path.is_dir(): + path = path.parent + pp_toml = path.joinpath("pyproject.toml") + setup_py = path.joinpath("setup.py") + if not pp_toml.exists(): + if setup_py.exists(): + return None + else: + pyproject_data = {} + with io.open(pp_toml.as_posix(), encoding="utf-8") as fh: + pyproject_data = tomlkit.loads(fh.read()) + build_system = pyproject_data.get("build-system", None) + if build_system is None: + if setup_py.exists(): + requires = ["setuptools", "wheel"] + backend = "setuptools.build_meta" + else: + requires = ["setuptools>=38.2.5", "wheel"] + backend = "setuptools.build_meta" + build_system = { + "requires": requires, + "build-backend": backend + } + pyproject_data["build_system"] = build_system + else: + requires = build_system.get("requires") + backend = build_system.get("build-backend") + return (requires, backend) def split_markers_from_line(line): @@ -437,17 +462,18 @@ def make_install_requirement(name, version, extras, markers, constraint=False): """ # If no extras are specified, the extras string is blank + from pip_shims.shims import install_req_from_line extras_string = "" if extras: # Sort extras for stability extras_string = "[{}]".format(",".join(sorted(extras))) if not markers: - return InstallRequirement.from_line( + return install_req_from_line( str('{}{}=={}'.format(name, extras_string, version)), constraint=constraint) else: - return InstallRequirement.from_line( + return install_req_from_line( str('{}{}=={}; {}'.format(name, extras_string, version, str(markers))), constraint=constraint) @@ -468,6 +494,7 @@ def clean_requires_python(candidates): """Get a cleaned list of all the candidates with valid specifiers in the `requires_python` attributes.""" all_candidates = [] sys_version = '.'.join(map(str, sys.version_info[:3])) + from packaging.version import parse as parse_version py_version = parse_version(os.environ.get('PIP_PYTHON_VERSION', sys_version)) for c in candidates: from_location = attrgetter("location.requires_python") @@ -489,6 +516,7 @@ def clean_requires_python(candidates): def fix_requires_python_marker(requires_python): + from packaging.requirements import Requirement as PackagingRequirement marker_str = '' if any(requires_python.startswith(op) for op in Specifier._operators.keys()): spec_dict = defaultdict(set) @@ -508,3 +536,15 @@ def fix_requires_python_marker(requires_python): ]) marker_to_add = PackagingRequirement('fakepkg; {0}'.format(marker_str)).marker return marker_to_add + + +def normalize_name(pkg): + """Given a package name, return its normalized, non-canonicalized form. + + :param str pkg: The name of a package + :return: A normalized package name + :rtype: str + """ + + assert isinstance(pkg, six.string_types) + return pkg.replace("_", "-").lower() diff --git a/pipenv/vendor/requirementslib/models/vcs.py b/pipenv/vendor/requirementslib/models/vcs.py index fb2e6bc3..6a15db3f 100644 --- a/pipenv/vendor/requirementslib/models/vcs.py +++ b/pipenv/vendor/requirementslib/models/vcs.py @@ -1,11 +1,7 @@ # -*- coding=utf-8 -*- import attr -from pip_shims import VcsSupport, parse_version, pip_version -import vistir import os - - -VCS_SUPPORT = VcsSupport() +import pip_shims @attr.s @@ -14,12 +10,15 @@ class VCSRepository(object): name = attr.ib() checkout_directory = attr.ib() vcs_type = attr.ib() + subdirectory = attr.ib(default=None) commit_sha = attr.ib(default=None) ref = attr.ib(default=None) repo_instance = attr.ib() @repo_instance.default def get_repo_instance(self): + from pip_shims import VcsSupport + VCS_SUPPORT = VcsSupport() backend = VCS_SUPPORT._registry.get(self.vcs_type) return backend(url=self.url) @@ -31,35 +30,31 @@ class VCSRepository(object): return url.startswith("file") def obtain(self): - if not os.path.exists(self.checkout_directory): + if (os.path.exists(self.checkout_directory) and not + self.repo_instance.is_repository_directory(self.checkout_directory)): + self.repo_instance.unpack(self.checkout_directory) + elif not os.path.exists(self.checkout_directory): self.repo_instance.obtain(self.checkout_directory) - if self.ref: - self.checkout_ref(self.ref) - self.commit_sha = self.get_commit_hash(self.ref) else: - if not self.commit_sha: - self.commit_sha = self.get_commit_hash() + if self.ref: + self.checkout_ref(self.ref) + if not self.commit_sha: + self.commit_sha = self.get_commit_hash() def checkout_ref(self, ref): if not self.repo_instance.is_commit_id_equal( - self.checkout_directory, self.get_commit_hash(ref) + self.checkout_directory, self.get_commit_hash() ) and not self.repo_instance.is_commit_id_equal(self.checkout_directory, ref): if not self.is_local: self.update(ref) def update(self, ref): target_ref = self.repo_instance.make_rev_options(ref) - sha = self.repo_instance.get_revision_sha(self.checkout_directory, target_ref.arg_rev) - target_rev = target_ref.make_new(sha) - if parse_version(pip_version) > parse_version("18.0"): + if pip_shims.parse_version(pip_shims.pip_version) > pip_shims.parse_version("18.0"): self.repo_instance.update(self.checkout_directory, self.url, target_ref) else: self.repo_instance.update(self.checkout_directory, target_ref) - self.commit_hash = self.get_commit_hash(ref) + self.commit_sha = self.get_commit_hash() def get_commit_hash(self, ref=None): - if ref: - target_ref = self.repo_instance.make_rev_options(ref) - return self.repo_instance.get_revision_sha(self.checkout_directory, target_ref.arg_rev) - # return self.repo_instance.get_revision(self.checkout_directory) return self.repo_instance.get_revision(self.checkout_directory) diff --git a/pipenv/vendor/requirementslib/utils.py b/pipenv/vendor/requirementslib/utils.py index b490d3cf..f3653e32 100644 --- a/pipenv/vendor/requirementslib/utils.py +++ b/pipenv/vendor/requirementslib/utils.py @@ -6,30 +6,28 @@ import logging import os import six +import sys import tomlkit +six.add_move(six.MovedAttribute("Mapping", "collections", "collections.abc")) +six.add_move(six.MovedAttribute("Sequence", "collections", "collections.abc")) +six.add_move(six.MovedAttribute("Set", "collections", "collections.abc")) +six.add_move(six.MovedAttribute("ItemsView", "collections", "collections.abc")) +from six.moves import Mapping, Sequence, Set, ItemsView from six.moves.urllib.parse import urlparse, urlsplit -from pip_shims.shims import ( - Command, VcsSupport, cmdoptions, is_archive_file, - is_installable_dir as _is_installable_dir -) +import pip_shims.shims from vistir.compat import Path from vistir.path import is_valid_url, ensure_mkdir_p, create_tracked_tempdir VCS_LIST = ("git", "svn", "hg", "bzr") -VCS_SCHEMES = [] -SCHEME_LIST = ("http://", "https://", "ftp://", "ftps://", "file://") - -if not VCS_SCHEMES: - VCS_SCHEMES = VcsSupport().all_schemes def setup_logger(): logger = logging.getLogger("requirementslib") loglevel = logging.DEBUG - handler = logging.StreamHandler() + handler = logging.StreamHandler(stream=sys.stderr) handler.setLevel(loglevel) logger.addHandler(handler) logger.setLevel(loglevel) @@ -39,8 +37,38 @@ def setup_logger(): log = setup_logger() +SCHEME_LIST = ("http://", "https://", "ftp://", "ftps://", "file://") + + +VCS_SCHEMES = [ + "git", + "git+http", + "git+https", + "git+ssh", + "git+git", + "git+file", + "hg", + "hg+http", + "hg+https", + "hg+ssh", + "hg+static-http", + "svn", + "svn+ssh", + "svn+http", + "svn+https", + "svn+svn", + "bzr", + "bzr+http", + "bzr+https", + "bzr+ssh", + "bzr+sftp", + "bzr+ftp", + "bzr+lp", +] + + def is_installable_dir(path): - if _is_installable_dir(path): + if pip_shims.shims.is_installable_dir(path): return True path = Path(path) pyproject = path.joinpath("pyproject.toml") @@ -52,20 +80,42 @@ def is_installable_dir(path): return False +def strip_ssh_from_git_uri(uri): + """Return git+ssh:// formatted URI to git+git@ format""" + if isinstance(uri, six.string_types): + uri = uri.replace("git+ssh://", "git+", 1) + return uri + + +def add_ssh_scheme_to_git_uri(uri): + """Cleans VCS uris from pipenv.patched.notpip format""" + if isinstance(uri, six.string_types): + # Add scheme for parsing purposes, this is also what pip does + if uri.startswith("git+") and "://" not in uri: + uri = uri.replace("git+", "git+ssh://", 1) + return uri + + def is_vcs(pipfile_entry): """Determine if dictionary entry from Pipfile is for a vcs dependency.""" - if hasattr(pipfile_entry, "keys"): + if isinstance(pipfile_entry, Mapping): return any(key for key in pipfile_entry.keys() if key in VCS_LIST) elif isinstance(pipfile_entry, six.string_types): if not is_valid_url(pipfile_entry) and pipfile_entry.startswith("git+"): - from .models.utils import add_ssh_scheme_to_git_uri pipfile_entry = add_ssh_scheme_to_git_uri(pipfile_entry) + parsed_entry = urlsplit(pipfile_entry) return parsed_entry.scheme in VCS_SCHEMES return False +def is_editable(pipfile_entry): + if isinstance(pipfile_entry, Mapping): + return pipfile_entry.get("editable", False) is True + return False + + def multi_split(s, split): """Splits on multiple given separators.""" for r in split: @@ -74,7 +124,9 @@ def multi_split(s, split): def is_star(val): - return isinstance(val, six.string_types) and val == "*" + return (isinstance(val, six.string_types) and val == "*") or ( + isinstance(val, Mapping) and val.get("version", "") == "*" + ) def is_installable_file(path): @@ -111,7 +163,7 @@ def is_installable_file(path): if lookup_path.is_dir() and is_installable_dir(absolute_path): return True - elif lookup_path.is_file() and is_archive_file(absolute_path): + elif lookup_path.is_file() and pip_shims.shims.is_archive_file(absolute_path): return True return False @@ -125,9 +177,7 @@ def prepare_pip_source_args(sources, pip_args=None): pip_args.extend(["-i", sources[0]["url"]]) # Trust the host if it's not verified. if not sources[0].get("verify_ssl", True): - pip_args.extend( - ["--trusted-host", urlparse(sources[0]["url"]).hostname] - ) + pip_args.extend(["--trusted-host", urlparse(sources[0]["url"]).hostname]) # Add additional sources as extra indexes. if len(sources) > 1: for source in sources[1:]: @@ -140,28 +190,6 @@ def prepare_pip_source_args(sources, pip_args=None): return pip_args -class PipCommand(Command): - name = 'PipCommand' - - -def get_pip_command(): - # Use pip's parser for pip.conf management and defaults. - # General options (find_links, index_url, extra_index_url, trusted_host, - # and pre) are defered to pip. - import optparse - pip_command = PipCommand() - pip_command.parser.add_option(cmdoptions.no_binary()) - pip_command.parser.add_option(cmdoptions.only_binary()) - index_opts = cmdoptions.make_option_group( - cmdoptions.index_group, - pip_command.parser, - ) - pip_command.parser.insert_option_group(0, index_opts) - pip_command.parser.add_option(optparse.Option('--pre', action='store_true', default=False)) - - return pip_command - - @ensure_mkdir_p(mode=0o777) def _ensure_dir(path): return path @@ -172,7 +200,14 @@ def ensure_setup_py(base_dir): if not base_dir: base_dir = create_tracked_tempdir(prefix="requirementslib-setup") base_dir = Path(base_dir) + if base_dir.exists() and base_dir.name == "setup.py": + base_dir = base_dir.parent + elif not (base_dir.exists() and base_dir.is_dir()): + base_dir = base_dir.parent + if not (base_dir.exists() and base_dir.is_dir()): + base_dir = base_dir.parent setup_py = base_dir.joinpath("setup.py") + is_new = False if setup_py.exists() else True if not setup_py.exists(): setup_py.write_text(u"") @@ -181,3 +216,375 @@ def ensure_setup_py(base_dir): finally: if is_new: setup_py.unlink() + + +_UNSET = object() +_REMAP_EXIT = object() + + +# The following functionality is either borrowed or modified from the itertools module +# in the boltons library by Mahmoud Hashemi and distributed under the BSD license +# the text of which is included below: + +# (original text from https://github.com/mahmoud/boltons/blob/master/LICENSE) +# Copyright (c) 2013, Mahmoud Hashemi +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# +# * The names of the contributors may not be used to endorse or +# promote products derived from this software without specific +# prior written permission. +# +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +class PathAccessError(KeyError, IndexError, TypeError): + """An amalgamation of KeyError, IndexError, and TypeError, + representing what can occur when looking up a path in a nested + object. + """ + + def __init__(self, exc, seg, path): + self.exc = exc + self.seg = seg + self.path = path + + def __repr__(self): + cn = self.__class__.__name__ + return "%s(%r, %r, %r)" % (cn, self.exc, self.seg, self.path) + + def __str__(self): + return "could not access %r from path %r, got error: %r" % ( + self.seg, + self.path, + self.exc, + ) + + +def get_path(root, path, default=_UNSET): + """Retrieve a value from a nested object via a tuple representing the + lookup path. + >>> root = {'a': {'b': {'c': [[1], [2], [3]]}}} + >>> get_path(root, ('a', 'b', 'c', 2, 0)) + 3 + The path format is intentionally consistent with that of + :func:`remap`. + One of get_path's chief aims is improved error messaging. EAFP is + great, but the error messages are not. + For instance, ``root['a']['b']['c'][2][1]`` gives back + ``IndexError: list index out of range`` + What went out of range where? get_path currently raises + ``PathAccessError: could not access 2 from path ('a', 'b', 'c', 2, + 1), got error: IndexError('list index out of range',)``, a + subclass of IndexError and KeyError. + You can also pass a default that covers the entire operation, + should the lookup fail at any level. + Args: + root: The target nesting of dictionaries, lists, or other + objects supporting ``__getitem__``. + path (tuple): A list of strings and integers to be successively + looked up within *root*. + default: The value to be returned should any + ``PathAccessError`` exceptions be raised. + """ + if isinstance(path, six.string_types): + path = path.split(".") + cur = root + try: + for seg in path: + try: + cur = cur[seg] + except (KeyError, IndexError) as exc: + raise PathAccessError(exc, seg, path) + except TypeError as exc: + # either string index in a list, or a parent that + # doesn't support indexing + try: + seg = int(seg) + cur = cur[seg] + except (ValueError, KeyError, IndexError, TypeError): + if not getattr(cur, "__iter__", None): + exc = TypeError( + "%r object is not indexable" % type(cur).__name__ + ) + raise PathAccessError(exc, seg, path) + except PathAccessError: + if default is _UNSET: + raise + return default + return cur + + +def default_visit(path, key, value): + return key, value + + +_orig_default_visit = default_visit + + +# Modified from https://github.com/mahmoud/boltons/blob/master/boltons/iterutils.py +def dict_path_enter(path, key, value): + if isinstance(value, six.string_types): + return value, False + elif isinstance(value, (Mapping, dict)): + return value.__class__(), ItemsView(value) + elif isinstance(value, tomlkit.items.Array): + return value.__class__([], value.trivia), enumerate(value) + elif isinstance(value, (Sequence, list)): + return value.__class__(), enumerate(value) + elif isinstance(value, (Set, set)): + return value.__class__(), enumerate(value) + else: + return value, False + + +def dict_path_exit(path, key, old_parent, new_parent, new_items): + ret = new_parent + if isinstance(new_parent, (Mapping, dict)): + vals = dict(new_items) + try: + new_parent.update(new_items) + except AttributeError: + # Handle toml containers specifically + try: + new_parent.update(vals) + # Now use default fallback if needed + except AttributeError: + ret = new_parent.__class__(vals) + elif isinstance(new_parent, tomlkit.items.Array): + vals = tomlkit.items.item([v for i, v in new_items]) + try: + new_parent._value.extend(vals._value) + except AttributeError: + ret = tomlkit.items.item(vals) + elif isinstance(new_parent, (Sequence, list)): + vals = [v for i, v in new_items] + try: + new_parent.extend(vals) + except AttributeError: + ret = new_parent.__class__(vals) # tuples + elif isinstance(new_parent, (Set, set)): + vals = [v for i, v in new_items] + try: + new_parent.update(vals) + except AttributeError: + ret = new_parent.__class__(vals) # frozensets + else: + raise RuntimeError("unexpected iterable type: %r" % type(new_parent)) + return ret + + +def remap( + root, visit=default_visit, enter=dict_path_enter, exit=dict_path_exit, **kwargs +): + """The remap ("recursive map") function is used to traverse and + transform nested structures. Lists, tuples, sets, and dictionaries + are just a few of the data structures nested into heterogenous + tree-like structures that are so common in programming. + Unfortunately, Python's built-in ways to manipulate collections + are almost all flat. List comprehensions may be fast and succinct, + but they do not recurse, making it tedious to apply quick changes + or complex transforms to real-world data. + remap goes where list comprehensions cannot. + Here's an example of removing all Nones from some data: + >>> from pprint import pprint + >>> reviews = {'Star Trek': {'TNG': 10, 'DS9': 8.5, 'ENT': None}, + ... 'Babylon 5': 6, 'Dr. Who': None} + >>> pprint(remap(reviews, lambda p, k, v: v is not None)) + {'Babylon 5': 6, 'Star Trek': {'DS9': 8.5, 'TNG': 10}} + Notice how both Nones have been removed despite the nesting in the + dictionary. Not bad for a one-liner, and that's just the beginning. + See `this remap cookbook`_ for more delicious recipes. + .. _this remap cookbook: http://sedimental.org/remap.html + remap takes four main arguments: the object to traverse and three + optional callables which determine how the remapped object will be + created. + Args: + root: The target object to traverse. By default, remap + supports iterables like :class:`list`, :class:`tuple`, + :class:`dict`, and :class:`set`, but any object traversable by + *enter* will work. + visit (callable): This function is called on every item in + *root*. It must accept three positional arguments, *path*, + *key*, and *value*. *path* is simply a tuple of parents' + keys. *visit* should return the new key-value pair. It may + also return ``True`` as shorthand to keep the old item + unmodified, or ``False`` to drop the item from the new + structure. *visit* is called after *enter*, on the new parent. + The *visit* function is called for every item in root, + including duplicate items. For traversable values, it is + called on the new parent object, after all its children + have been visited. The default visit behavior simply + returns the key-value pair unmodified. + enter (callable): This function controls which items in *root* + are traversed. It accepts the same arguments as *visit*: the + path, the key, and the value of the current item. It returns a + pair of the blank new parent, and an iterator over the items + which should be visited. If ``False`` is returned instead of + an iterator, the value will not be traversed. + The *enter* function is only called once per unique value. The + default enter behavior support mappings, sequences, and + sets. Strings and all other iterables will not be traversed. + exit (callable): This function determines how to handle items + once they have been visited. It gets the same three + arguments as the other functions -- *path*, *key*, *value* + -- plus two more: the blank new parent object returned + from *enter*, and a list of the new items, as remapped by + *visit*. + Like *enter*, the *exit* function is only called once per + unique value. The default exit behavior is to simply add + all new items to the new parent, e.g., using + :meth:`list.extend` and :meth:`dict.update` to add to the + new parent. Immutable objects, such as a :class:`tuple` or + :class:`namedtuple`, must be recreated from scratch, but + use the same type as the new parent passed back from the + *enter* function. + reraise_visit (bool): A pragmatic convenience for the *visit* + callable. When set to ``False``, remap ignores any errors + raised by the *visit* callback. Items causing exceptions + are kept. See examples for more details. + remap is designed to cover the majority of cases with just the + *visit* callable. While passing in multiple callables is very + empowering, remap is designed so very few cases should require + passing more than one function. + When passing *enter* and *exit*, it's common and easiest to build + on the default behavior. Simply add ``from boltons.iterutils import + default_enter`` (or ``default_exit``), and have your enter/exit + function call the default behavior before or after your custom + logic. See `this example`_. + Duplicate and self-referential objects (aka reference loops) are + automatically handled internally, `as shown here`_. + .. _this example: http://sedimental.org/remap.html#sort_all_lists + .. _as shown here: http://sedimental.org/remap.html#corner_cases + """ + # TODO: improve argument formatting in sphinx doc + # TODO: enter() return (False, items) to continue traverse but cancel copy? + if not callable(visit): + raise TypeError("visit expected callable, not: %r" % visit) + if not callable(enter): + raise TypeError("enter expected callable, not: %r" % enter) + if not callable(exit): + raise TypeError("exit expected callable, not: %r" % exit) + reraise_visit = kwargs.pop("reraise_visit", True) + if kwargs: + raise TypeError("unexpected keyword arguments: %r" % kwargs.keys()) + + path, registry, stack = (), {}, [(None, root)] + new_items_stack = [] + while stack: + key, value = stack.pop() + id_value = id(value) + if key is _REMAP_EXIT: + key, new_parent, old_parent = value + id_value = id(old_parent) + path, new_items = new_items_stack.pop() + value = exit(path, key, old_parent, new_parent, new_items) + registry[id_value] = value + if not new_items_stack: + continue + elif id_value in registry: + value = registry[id_value] + else: + res = enter(path, key, value) + try: + new_parent, new_items = res + except TypeError: + # TODO: handle False? + raise TypeError( + "enter should return a tuple of (new_parent," + " items_iterator), not: %r" % res + ) + if new_items is not False: + # traverse unless False is explicitly passed + registry[id_value] = new_parent + new_items_stack.append((path, [])) + if value is not root: + path += (key,) + stack.append((_REMAP_EXIT, (key, new_parent, value))) + if new_items: + stack.extend(reversed(list(new_items))) + continue + if visit is _orig_default_visit: + # avoid function call overhead by inlining identity operation + visited_item = (key, value) + else: + try: + visited_item = visit(path, key, value) + except Exception: + if reraise_visit: + raise + visited_item = True + if visited_item is False: + continue # drop + elif visited_item is True: + visited_item = (key, value) + # TODO: typecheck? + # raise TypeError('expected (key, value) from visit(),' + # ' not: %r' % visited_item) + try: + new_items_stack[-1][1].append(visited_item) + except IndexError: + raise TypeError("expected remappable root, not: %r" % root) + return value + + +def merge_items(target_list, sourced=False): + if not sourced: + target_list = [(id(t), t) for t in target_list] + + ret = None + source_map = {} + + def remerge_enter(path, key, value): + new_parent, new_items = dict_path_enter(path, key, value) + if ret and not path and key is None: + new_parent = ret + + try: + cur_val = get_path(ret, path + (key,)) + except KeyError as ke: + pass + else: + new_parent = cur_val + + return new_parent, new_items + + def remerge_exit(path, key, old_parent, new_parent, new_items): + return dict_path_exit(path, key, old_parent, new_parent, new_items) + + for t_name, target in target_list: + if sourced: + + def remerge_visit(path, key, value): + source_map[path + (key,)] = t_name + return True + + else: + remerge_visit = default_visit + + ret = remap(target, enter=remerge_enter, visit=remerge_visit, exit=remerge_exit) + + if not sourced: + return ret + return ret, source_map diff --git a/pipenv/vendor/resolvelib/LICENSE b/pipenv/vendor/resolvelib/LICENSE new file mode 100644 index 00000000..b9077766 --- /dev/null +++ b/pipenv/vendor/resolvelib/LICENSE @@ -0,0 +1,13 @@ +Copyright (c) 2018, Tzu-ping Chung + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/pipenv/vendor/resolvelib/__init__.py b/pipenv/vendor/resolvelib/__init__.py new file mode 100644 index 00000000..e0e37434 --- /dev/null +++ b/pipenv/vendor/resolvelib/__init__.py @@ -0,0 +1,16 @@ +__all__ = [ + '__version__', + 'AbstractProvider', 'BaseReporter', 'Resolver', + 'NoVersionsAvailable', 'RequirementsConflicted', + 'ResolutionError', 'ResolutionImpossible', 'ResolutionTooDeep', +] + +__version__ = '0.2.2' + + +from .providers import AbstractProvider +from .reporters import BaseReporter +from .resolvers import ( + NoVersionsAvailable, RequirementsConflicted, + Resolver, ResolutionError, ResolutionImpossible, ResolutionTooDeep, +) diff --git a/pipenv/vendor/resolvelib/providers.py b/pipenv/vendor/resolvelib/providers.py new file mode 100644 index 00000000..515c0db4 --- /dev/null +++ b/pipenv/vendor/resolvelib/providers.py @@ -0,0 +1,78 @@ +class AbstractProvider(object): + """Delegate class to provide requirment interface for the resolver. + """ + def identify(self, dependency): + """Given a dependency, return an identifier for it. + + This is used in many places to identify the dependency, e.g. whether + two requirements should have their specifier parts merged, whether + two specifications would conflict with each other (because they the + same name but different versions). + """ + raise NotImplementedError + + def get_preference(self, resolution, candidates, information): + """Produce a sort key for given specification based on preference. + + The preference is defined as "I think this requirement should be + resolved first". The lower the return value is, the more preferred + this group of arguments is. + + :param resolution: Currently pinned candidate, or `None`. + :param candidates: A list of possible candidates. + :param information: A list of requirement information. + + Each information instance is a named tuple with two entries: + + * `requirement` specifies a requirement contributing to the current + candidate list + * `parent` specifies the candidate that provids (dependend on) the + requirement, or `None` to indicate a root requirement. + + The preference could depend on a various of issues, including (not + necessarily in this order): + + * Is this package pinned in the current resolution result? + * How relaxed is the requirement? Stricter ones should probably be + worked on first? (I don't know, actually.) + * How many possibilities are there to satisfy this requirement? Those + with few left should likely be worked on first, I guess? + * Are there any known conflicts for this requirement? We should + probably work on those with the most known conflicts. + + A sortable value should be returned (this will be used as the `key` + parameter of the built-in sorting function). The smaller the value is, + the more preferred this specification is (i.e. the sorting function + is called with `reverse=False`). + """ + raise NotImplementedError + + def find_matches(self, requirement): + """Find all possible candidates that satisfy a requirement. + + This should try to get candidates based on the requirement's type. + For VCS, local, and archive requirements, the one-and-only match is + returned, and for a "named" requirement, the index(es) should be + consulted to find concrete candidates for this requirement. + + The returned candidates should be sorted by reversed preference, e.g. + the latest should be LAST. This is done so list-popping can be as + efficient as possible. + """ + raise NotImplementedError + + def is_satisfied_by(self, requirement, candidate): + """Whether the given requirement can be satisfied by a candidate. + + A boolean should be retuened to indicate whether `candidate` is a + viable solution to the requirement. + """ + raise NotImplementedError + + def get_dependencies(self, candidate): + """Get dependencies of a candidate. + + This should return a collection of requirements that `candidate` + specifies as its dependencies. + """ + raise NotImplementedError diff --git a/pipenv/vendor/resolvelib/reporters.py b/pipenv/vendor/resolvelib/reporters.py new file mode 100644 index 00000000..c723031f --- /dev/null +++ b/pipenv/vendor/resolvelib/reporters.py @@ -0,0 +1,23 @@ +class BaseReporter(object): + """Delegate class to provider progress reporting for the resolver. + """ + def starting(self): + """Called before the resolution actually starts. + """ + + def starting_round(self, index): + """Called before each round of resolution starts. + + The index is zero-based. + """ + + def ending_round(self, index, state): + """Called before each round of resolution ends. + + This is NOT called if the resolution ends at this round. Use `ending` + if you want to report finalization. The index is zero-based. + """ + + def ending(self, state): + """Called before the resolution ends successfully. + """ diff --git a/pipenv/vendor/resolvelib/resolvers.py b/pipenv/vendor/resolvelib/resolvers.py new file mode 100644 index 00000000..9c69628e --- /dev/null +++ b/pipenv/vendor/resolvelib/resolvers.py @@ -0,0 +1,287 @@ +import collections + +from .structs import DirectedGraph + + +RequirementInformation = collections.namedtuple('RequirementInformation', [ + 'requirement', 'parent', +]) + + +class NoVersionsAvailable(Exception): + def __init__(self, requirement, parent): + super(NoVersionsAvailable, self).__init__() + self.requirement = requirement + self.parent = parent + + +class RequirementsConflicted(Exception): + def __init__(self, criterion): + super(RequirementsConflicted, self).__init__() + self.criterion = criterion + + +class Criterion(object): + """Internal representation of possible resolution results of a package. + + This holds two attributes: + + * `information` is a collection of `RequirementInformation` pairs. Each + pair is a requirement contributing to this criterion, and the candidate + that provides the requirement. + * `candidates` is a collection containing all possible candidates deducted + from the union of contributing requirements. It should never be empty. + """ + def __init__(self, candidates, information): + self.candidates = candidates + self.information = information + + @classmethod + def from_requirement(cls, provider, requirement, parent): + """Build an instance from a requirement. + """ + candidates = provider.find_matches(requirement) + if not candidates: + raise NoVersionsAvailable(requirement, parent) + return cls( + candidates=candidates, + information=[RequirementInformation(requirement, parent)], + ) + + def iter_requirement(self): + return (i.requirement for i in self.information) + + def iter_parent(self): + return (i.parent for i in self.information) + + def merged_with(self, provider, requirement, parent): + """Build a new instance from this and a new requirement. + """ + infos = list(self.information) + infos.append(RequirementInformation(requirement, parent)) + candidates = [ + c for c in self.candidates + if provider.is_satisfied_by(requirement, c) + ] + if not candidates: + raise RequirementsConflicted(self) + return type(self)(candidates, infos) + + +class ResolutionError(Exception): + pass + + +class ResolutionImpossible(ResolutionError): + def __init__(self, requirements): + super(ResolutionImpossible, self).__init__() + self.requirements = requirements + + +class ResolutionTooDeep(ResolutionError): + def __init__(self, round_count): + super(ResolutionTooDeep, self).__init__(round_count) + self.round_count = round_count + + +# Resolution state in a round. +State = collections.namedtuple('State', 'mapping graph') + + +class Resolution(object): + """Stateful resolution object. + + This is designed as a one-off object that holds information to kick start + the resolution process, and holds the results afterwards. + """ + def __init__(self, provider, reporter): + self._p = provider + self._r = reporter + self._criteria = {} + self._states = [] + + @property + def state(self): + try: + return self._states[-1] + except IndexError: + raise AttributeError('state') + + def _push_new_state(self): + """Push a new state into history. + + This new state will be used to hold resolution results of the next + coming round. + """ + try: + base = self._states[-1] + except IndexError: + graph = DirectedGraph() + graph.add(None) # Sentinel as root dependencies' parent. + state = State(mapping={}, graph=graph) + else: + state = State( + mapping=base.mapping.copy(), + graph=base.graph.copy(), + ) + self._states.append(state) + + def _contribute_to_criteria(self, name, requirement, parent): + try: + crit = self._criteria[name] + except KeyError: + crit = Criterion.from_requirement(self._p, requirement, parent) + else: + crit = crit.merged_with(self._p, requirement, parent) + self._criteria[name] = crit + + def _get_criterion_item_preference(self, item): + name, criterion = item + try: + pinned = self.state.mapping[name] + except (IndexError, KeyError): + pinned = None + return self._p.get_preference( + pinned, criterion.candidates, criterion.information, + ) + + def _is_current_pin_satisfying(self, name, criterion): + try: + current_pin = self.state.mapping[name] + except KeyError: + return False + return all( + self._p.is_satisfied_by(r, current_pin) + for r in criterion.iter_requirement() + ) + + def _check_pinnability(self, candidate, dependencies): + backup = self._criteria.copy() + contributed = set() + try: + for subdep in dependencies: + key = self._p.identify(subdep) + self._contribute_to_criteria(key, subdep, parent=candidate) + contributed.add(key) + except RequirementsConflicted: + self._criteria = backup + return None + return contributed + + def _pin_candidate(self, name, criterion, candidate, child_names): + try: + self.state.graph.remove(name) + except KeyError: + pass + self.state.mapping[name] = candidate + self.state.graph.add(name) + for parent in criterion.iter_parent(): + parent_name = None if parent is None else self._p.identify(parent) + try: + self.state.graph.connect(parent_name, name) + except KeyError: + # Parent is not yet pinned. Skip now; this edge will be + # connected when the parent is being pinned. + pass + for child_name in child_names: + try: + self.state.graph.connect(name, child_name) + except KeyError: + # Child is not yet pinned. Skip now; this edge will be + # connected when the child is being pinned. + pass + + def _pin_criteria(self): + criterion_names = [name for name, _ in sorted( + self._criteria.items(), + key=self._get_criterion_item_preference, + )] + for name in criterion_names: + # Any pin may modify any criterion during the loop. Criteria are + # replaced, not updated in-place, so we need to read this value + # in the loop instead of outside. (sarugaku/resolvelib#5) + criterion = self._criteria[name] + + if self._is_current_pin_satisfying(name, criterion): + # If the current pin already works, just use it. + continue + candidates = list(criterion.candidates) + while candidates: + candidate = candidates.pop() + dependencies = self._p.get_dependencies(candidate) + child_names = self._check_pinnability(candidate, dependencies) + if child_names is None: + continue + self._pin_candidate(name, criterion, candidate, child_names) + break + else: # All candidates tried, nothing works. Give up. (?) + raise ResolutionImpossible(list(criterion.iter_requirement())) + + def resolve(self, requirements, max_rounds): + if self._states: + raise RuntimeError('already resolved') + + for requirement in requirements: + try: + name = self._p.identify(requirement) + self._contribute_to_criteria(name, requirement, parent=None) + except RequirementsConflicted as e: + # If initial requirements conflict, nothing would ever work. + raise ResolutionImpossible(e.requirements + [requirement]) + + last = None + self._r.starting() + + for round_index in range(max_rounds): + self._r.starting_round(round_index) + + self._push_new_state() + self._pin_criteria() + + curr = self.state + if last is not None and len(curr.mapping) == len(last.mapping): + # Nothing new added. Done! Remove the duplicated entry. + del self._states[-1] + self._r.ending(last) + return + last = curr + + self._r.ending_round(round_index, curr) + + raise ResolutionTooDeep(max_rounds) + + +class Resolver(object): + """The thing that performs the actual resolution work. + """ + def __init__(self, provider, reporter): + self.provider = provider + self.reporter = reporter + + def resolve(self, requirements, max_rounds=20): + """Take a collection of constraints, spit out the resolution result. + + The return value is a representation to the final resolution result. It + is a tuple subclass with two public members: + + * `mapping`: A dict of resolved candidates. Each key is an identifier + of a requirement (as returned by the provider's `identify` method), + and the value is the resolved candidate. + * `graph`: A `DirectedGraph` instance representing the dependency tree. + The vertices are keys of `mapping`, and each edge represents *why* + a particular package is included. A special vertex `None` is + included to represent parents of user-supplied requirements. + + The following exceptions may be raised if a resolution cannot be found: + + * `NoVersionsAvailable`: A requirement has no available candidates. + * `ResolutionImpossible`: A resolution cannot be found for the given + combination of requirements. + * `ResolutionTooDeep`: The dependency tree is too deeply nested and + the resolver gave up. This is usually caused by a circular + dependency, but you can try to resolve this by increasing the + `max_rounds` argument. + """ + resolution = Resolution(self.provider, self.reporter) + resolution.resolve(requirements, max_rounds=max_rounds) + return resolution.state diff --git a/pipenv/vendor/resolvelib/structs.py b/pipenv/vendor/resolvelib/structs.py new file mode 100644 index 00000000..97bd0095 --- /dev/null +++ b/pipenv/vendor/resolvelib/structs.py @@ -0,0 +1,67 @@ +class DirectedGraph(object): + """A graph structure with directed edges. + """ + def __init__(self): + self._vertices = set() + self._forwards = {} # -> Set[] + self._backwards = {} # -> Set[] + + def __iter__(self): + return iter(self._vertices) + + def __len__(self): + return len(self._vertices) + + def __contains__(self, key): + return key in self._vertices + + def copy(self): + """Return a shallow copy of this graph. + """ + other = DirectedGraph() + other._vertices = set(self._vertices) + other._forwards = {k: set(v) for k, v in self._forwards.items()} + other._backwards = {k: set(v) for k, v in self._backwards.items()} + return other + + def add(self, key): + """Add a new vertex to the graph. + """ + if key in self._vertices: + raise ValueError('vertex exists') + self._vertices.add(key) + self._forwards[key] = set() + self._backwards[key] = set() + + def remove(self, key): + """Remove a vertex from the graph, disconnecting all edges from/to it. + """ + self._vertices.remove(key) + for f in self._forwards.pop(key): + self._backwards[f].remove(key) + for t in self._backwards.pop(key): + self._forwards[t].remove(key) + + def connected(self, f, t): + return f in self._backwards[t] and t in self._forwards[f] + + def connect(self, f, t): + """Connect two existing vertices. + + Nothing happens if the vertices are already connected. + """ + if t not in self._vertices: + raise KeyError(t) + self._forwards[f].add(t) + self._backwards[t].add(f) + + def iter_edges(self): + for f, children in self._forwards.items(): + for t in children: + yield f, t + + def iter_children(self, key): + return iter(self._forwards[key]) + + def iter_parents(self, key): + return iter(self._backwards[key]) diff --git a/pipenv/vendor/shellingham/__init__.py b/pipenv/vendor/shellingham/__init__.py index 90c00abb..576c4224 100644 --- a/pipenv/vendor/shellingham/__init__.py +++ b/pipenv/vendor/shellingham/__init__.py @@ -4,7 +4,7 @@ import os from ._core import ShellDetectionFailure -__version__ = '1.2.6' +__version__ = '1.2.7' def detect_shell(pid=None, max_depth=6): diff --git a/pipenv/vendor/shellingham/posix/ps.py b/pipenv/vendor/shellingham/posix/ps.py index ab4c2a9e..4a155ed5 100644 --- a/pipenv/vendor/shellingham/posix/ps.py +++ b/pipenv/vendor/shellingham/posix/ps.py @@ -21,6 +21,12 @@ def get_process_mapping(): if e.errno != errno.ENOENT: raise raise PsNotAvailable('ps not found') + except subprocess.CalledProcessError as e: + # `ps` can return 1 if the process list is completely empty. + # (sarugaku/shellingham#15) + if not e.output.strip(): + return {} + raise if not isinstance(output, str): encoding = sys.getfilesystemencoding() or sys.getdefaultencoding() output = output.decode(encoding) @@ -28,9 +34,9 @@ def get_process_mapping(): for line in output.split('\n'): try: pid, ppid, args = line.strip().split(None, 2) + processes[pid] = Process( + args=tuple(shlex.split(args)), pid=pid, ppid=ppid, + ) except ValueError: continue - processes[pid] = Process( - args=tuple(shlex.split(args)), pid=pid, ppid=ppid, - ) return processes diff --git a/pipenv/vendor/toml.LICENSE b/pipenv/vendor/toml.LICENSE deleted file mode 100644 index d8b406c9..00000000 --- a/pipenv/vendor/toml.LICENSE +++ /dev/null @@ -1,26 +0,0 @@ -The MIT License - -Copyright 2013-2017 Uiri Noyb -Copyright 2015-2016 Julien Enselme -Copyright 2016 Google Inc. -Copyright 2017 Samuel Vasko -Copyright 2017 Nate Prewitt -Copyright 2017 Jack Evans - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. \ No newline at end of file diff --git a/pipenv/vendor/toml.py b/pipenv/vendor/toml.py deleted file mode 100644 index dac39883..00000000 --- a/pipenv/vendor/toml.py +++ /dev/null @@ -1,1039 +0,0 @@ -"""Python module which parses and emits TOML. - -Released under the MIT license. -""" -import re -import io -import datetime -from os import linesep -import sys - -__version__ = "0.9.6" -_spec_ = "0.4.0" - - -class TomlDecodeError(Exception): - """Base toml Exception / Error.""" - pass - - -class TomlTz(datetime.tzinfo): - def __init__(self, toml_offset): - if toml_offset == "Z": - self._raw_offset = "+00:00" - else: - self._raw_offset = toml_offset - self._sign = -1 if self._raw_offset[0] == '-' else 1 - self._hours = int(self._raw_offset[1:3]) - self._minutes = int(self._raw_offset[4:6]) - - def tzname(self, dt): - return "UTC" + self._raw_offset - - def utcoffset(self, dt): - return self._sign * datetime.timedelta(hours=self._hours, - minutes=self._minutes) - - def dst(self, dt): - return datetime.timedelta(0) - - -class InlineTableDict(object): - """Sentinel subclass of dict for inline tables.""" - - -def _get_empty_inline_table(_dict): - class DynamicInlineTableDict(_dict, InlineTableDict): - """Concrete sentinel subclass for inline tables. - It is a subclass of _dict which is passed in dynamically at load time - It is also a subclass of InlineTableDict - """ - - return DynamicInlineTableDict() - - -try: - _range = xrange -except NameError: - unicode = str - _range = range - basestring = str - unichr = chr - -try: - FNFError = FileNotFoundError -except NameError: - FNFError = IOError - - -def load(f, _dict=dict): - """Parses named file or files as toml and returns a dictionary - - Args: - f: Path to the file to open, array of files to read into single dict - or a file descriptor - _dict: (optional) Specifies the class of the returned toml dictionary - - Returns: - Parsed toml file represented as a dictionary - - Raises: - TypeError -- When f is invalid type - TomlDecodeError: Error while decoding toml - IOError / FileNotFoundError -- When an array with no valid (existing) - (Python 2 / Python 3) file paths is passed - """ - - if isinstance(f, basestring): - with io.open(f, encoding='utf-8') as ffile: - return loads(ffile.read(), _dict) - elif isinstance(f, list): - from os import path as op - from warnings import warn - if not [path for path in f if op.exists(path)]: - error_msg = "Load expects a list to contain filenames only." - error_msg += linesep - error_msg += ("The list needs to contain the path of at least one " - "existing file.") - raise FNFError(error_msg) - d = _dict() - for l in f: - if op.exists(l): - d.update(load(l)) - else: - warn("Non-existent filename in list with at least one valid " - "filename") - return d - else: - try: - return loads(f.read(), _dict) - except AttributeError: - raise TypeError("You can only load a file descriptor, filename or " - "list") - - -_groupname_re = re.compile(r'^[A-Za-z0-9_-]+$') - - -def loads(s, _dict=dict): - """Parses string as toml - - Args: - s: String to be parsed - _dict: (optional) Specifies the class of the returned toml dictionary - - Returns: - Parsed toml file represented as a dictionary - - Raises: - TypeError: When a non-string is passed - TomlDecodeError: Error while decoding toml - """ - - implicitgroups = [] - retval = _dict() - currentlevel = retval - if not isinstance(s, basestring): - raise TypeError("Expecting something like a string") - - if not isinstance(s, unicode): - s = s.decode('utf8') - - sl = list(s) - openarr = 0 - openstring = False - openstrchar = "" - multilinestr = False - arrayoftables = False - beginline = True - keygroup = False - keyname = 0 - for i, item in enumerate(sl): - if item == '\r' and sl[i + 1] == '\n': - sl[i] = ' ' - continue - if keyname: - if item == '\n': - raise TomlDecodeError("Key name found without value." - " Reached end of line.") - if openstring: - if item == openstrchar: - keyname = 2 - openstring = False - openstrchar = "" - continue - elif keyname == 1: - if item.isspace(): - keyname = 2 - continue - elif item.isalnum() or item == '_' or item == '-': - continue - elif keyname == 2 and item.isspace(): - continue - if item == '=': - keyname = 0 - else: - raise TomlDecodeError("Found invalid character in key name: '" + - item + "'. Try quoting the key name.") - if item == "'" and openstrchar != '"': - k = 1 - try: - while sl[i - k] == "'": - k += 1 - if k == 3: - break - except IndexError: - pass - if k == 3: - multilinestr = not multilinestr - openstring = multilinestr - else: - openstring = not openstring - if openstring: - openstrchar = "'" - else: - openstrchar = "" - if item == '"' and openstrchar != "'": - oddbackslash = False - k = 1 - tripquote = False - try: - while sl[i - k] == '"': - k += 1 - if k == 3: - tripquote = True - break - if k == 1 or (k == 3 and tripquote): - while sl[i - k] == '\\': - oddbackslash = not oddbackslash - k += 1 - except IndexError: - pass - if not oddbackslash: - if tripquote: - multilinestr = not multilinestr - openstring = multilinestr - else: - openstring = not openstring - if openstring: - openstrchar = '"' - else: - openstrchar = "" - if item == '#' and (not openstring and not keygroup and - not arrayoftables): - j = i - try: - while sl[j] != '\n': - sl[j] = ' ' - j += 1 - except IndexError: - break - if item == '[' and (not openstring and not keygroup and - not arrayoftables): - if beginline: - if len(sl) > i + 1 and sl[i + 1] == '[': - arrayoftables = True - else: - keygroup = True - else: - openarr += 1 - if item == ']' and not openstring: - if keygroup: - keygroup = False - elif arrayoftables: - if sl[i - 1] == ']': - arrayoftables = False - else: - openarr -= 1 - if item == '\n': - if openstring or multilinestr: - if not multilinestr: - raise TomlDecodeError("Unbalanced quotes") - if ((sl[i - 1] == "'" or sl[i - 1] == '"') and ( - sl[i - 2] == sl[i - 1])): - sl[i] = sl[i - 1] - if sl[i - 3] == sl[i - 1]: - sl[i - 3] = ' ' - elif openarr: - sl[i] = ' ' - else: - beginline = True - elif beginline and sl[i] != ' ' and sl[i] != '\t': - beginline = False - if not keygroup and not arrayoftables: - if sl[i] == '=': - raise TomlDecodeError("Found empty keyname. ") - keyname = 1 - s = ''.join(sl) - s = s.split('\n') - multikey = None - multilinestr = "" - multibackslash = False - for line in s: - if not multilinestr or multibackslash or '\n' not in multilinestr: - line = line.strip() - if line == "" and (not multikey or multibackslash): - continue - if multikey: - if multibackslash: - multilinestr += line - else: - multilinestr += line - multibackslash = False - if len(line) > 2 and (line[-1] == multilinestr[0] and - line[-2] == multilinestr[0] and - line[-3] == multilinestr[0]): - try: - value, vtype = _load_value(multilinestr, _dict) - except ValueError as err: - raise TomlDecodeError(str(err)) - currentlevel[multikey] = value - multikey = None - multilinestr = "" - else: - k = len(multilinestr) - 1 - while k > -1 and multilinestr[k] == '\\': - multibackslash = not multibackslash - k -= 1 - if multibackslash: - multilinestr = multilinestr[:-1] - else: - multilinestr += "\n" - continue - if line[0] == '[': - arrayoftables = False - if len(line) == 1: - raise TomlDecodeError("Opening key group bracket on line by " - "itself.") - if line[1] == '[': - arrayoftables = True - line = line[2:] - splitstr = ']]' - else: - line = line[1:] - splitstr = ']' - i = 1 - quotesplits = _get_split_on_quotes(line) - quoted = False - for quotesplit in quotesplits: - if not quoted and splitstr in quotesplit: - break - i += quotesplit.count(splitstr) - quoted = not quoted - line = line.split(splitstr, i) - if len(line) < i + 1 or line[-1].strip() != "": - raise TomlDecodeError("Key group not on a line by itself.") - groups = splitstr.join(line[:-1]).split('.') - i = 0 - while i < len(groups): - groups[i] = groups[i].strip() - if len(groups[i]) > 0 and (groups[i][0] == '"' or - groups[i][0] == "'"): - groupstr = groups[i] - j = i + 1 - while not groupstr[0] == groupstr[-1]: - j += 1 - if j > len(groups) + 2: - raise TomlDecodeError("Invalid group name '" + - groupstr + "' Something " + - "went wrong.") - groupstr = '.'.join(groups[i:j]).strip() - groups[i] = groupstr[1:-1] - groups[i + 1:j] = [] - else: - if not _groupname_re.match(groups[i]): - raise TomlDecodeError("Invalid group name '" + - groups[i] + "'. Try quoting it.") - i += 1 - currentlevel = retval - for i in _range(len(groups)): - group = groups[i] - if group == "": - raise TomlDecodeError("Can't have a keygroup with an empty " - "name") - try: - currentlevel[group] - if i == len(groups) - 1: - if group in implicitgroups: - implicitgroups.remove(group) - if arrayoftables: - raise TomlDecodeError("An implicitly defined " - "table can't be an array") - elif arrayoftables: - currentlevel[group].append(_dict()) - else: - raise TomlDecodeError("What? " + group + - " already exists?" + - str(currentlevel)) - except TypeError: - currentlevel = currentlevel[-1] - try: - currentlevel[group] - except KeyError: - currentlevel[group] = _dict() - if i == len(groups) - 1 and arrayoftables: - currentlevel[group] = [_dict()] - except KeyError: - if i != len(groups) - 1: - implicitgroups.append(group) - currentlevel[group] = _dict() - if i == len(groups) - 1 and arrayoftables: - currentlevel[group] = [_dict()] - currentlevel = currentlevel[group] - if arrayoftables: - try: - currentlevel = currentlevel[-1] - except KeyError: - pass - elif line[0] == "{": - if line[-1] != "}": - raise TomlDecodeError("Line breaks are not allowed in inline" - "objects") - try: - _load_inline_object(line, currentlevel, _dict, multikey, - multibackslash) - except ValueError as err: - raise TomlDecodeError(str(err)) - elif "=" in line: - try: - ret = _load_line(line, currentlevel, _dict, multikey, - multibackslash) - except ValueError as err: - raise TomlDecodeError(str(err)) - if ret is not None: - multikey, multilinestr, multibackslash = ret - return retval - - -def _load_inline_object(line, currentlevel, _dict, multikey=False, - multibackslash=False): - candidate_groups = line[1:-1].split(",") - groups = [] - if len(candidate_groups) == 1 and not candidate_groups[0].strip(): - candidate_groups.pop() - while len(candidate_groups) > 0: - candidate_group = candidate_groups.pop(0) - try: - _, value = candidate_group.split('=', 1) - except ValueError: - raise ValueError("Invalid inline table encountered") - value = value.strip() - if ((value[0] == value[-1] and value[0] in ('"', "'")) or ( - value[0] in '-0123456789' or - value in ('true', 'false') or - (value[0] == "[" and value[-1] == "]") or - (value[0] == '{' and value[-1] == '}'))): - groups.append(candidate_group) - elif len(candidate_groups) > 0: - candidate_groups[0] = candidate_group + "," + candidate_groups[0] - else: - raise ValueError("Invalid inline table value encountered") - for group in groups: - status = _load_line(group, currentlevel, _dict, multikey, - multibackslash) - if status is not None: - break - - -# Matches a TOML number, which allows underscores for readability -_number_with_underscores = re.compile('([0-9])(_([0-9]))*') - - -def _strictly_valid_num(n): - n = n.strip() - if not n: - return False - if n[0] == '_': - return False - if n[-1] == '_': - return False - if "_." in n or "._" in n: - return False - if len(n) == 1: - return True - if n[0] == '0' and n[1] != '.': - return False - if n[0] == '+' or n[0] == '-': - n = n[1:] - if n[0] == '0' and n[1] != '.': - return False - if '__' in n: - return False - return True - - -def _get_split_on_quotes(line): - doublequotesplits = line.split('"') - quoted = False - quotesplits = [] - if len(doublequotesplits) > 1 and "'" in doublequotesplits[0]: - singlequotesplits = doublequotesplits[0].split("'") - doublequotesplits = doublequotesplits[1:] - while len(singlequotesplits) % 2 == 0 and len(doublequotesplits): - singlequotesplits[-1] += '"' + doublequotesplits[0] - doublequotesplits = doublequotesplits[1:] - if "'" in singlequotesplits[-1]: - singlequotesplits = (singlequotesplits[:-1] + - singlequotesplits[-1].split("'")) - quotesplits += singlequotesplits - for doublequotesplit in doublequotesplits: - if quoted: - quotesplits.append(doublequotesplit) - else: - quotesplits += doublequotesplit.split("'") - quoted = not quoted - return quotesplits - - -def _load_line(line, currentlevel, _dict, multikey, multibackslash): - i = 1 - quotesplits = _get_split_on_quotes(line) - quoted = False - for quotesplit in quotesplits: - if not quoted and '=' in quotesplit: - break - i += quotesplit.count('=') - quoted = not quoted - pair = line.split('=', i) - strictly_valid = _strictly_valid_num(pair[-1]) - if _number_with_underscores.match(pair[-1]): - pair[-1] = pair[-1].replace('_', '') - while len(pair[-1]) and (pair[-1][0] != ' ' and pair[-1][0] != '\t' and - pair[-1][0] != "'" and pair[-1][0] != '"' and - pair[-1][0] != '[' and pair[-1][0] != '{' and - pair[-1] != 'true' and pair[-1] != 'false'): - try: - float(pair[-1]) - break - except ValueError: - pass - if _load_date(pair[-1]) is not None: - break - i += 1 - prev_val = pair[-1] - pair = line.split('=', i) - if prev_val == pair[-1]: - raise ValueError("Invalid date or number") - if strictly_valid: - strictly_valid = _strictly_valid_num(pair[-1]) - pair = ['='.join(pair[:-1]).strip(), pair[-1].strip()] - if (pair[0][0] == '"' or pair[0][0] == "'") and \ - (pair[0][-1] == '"' or pair[0][-1] == "'"): - pair[0] = pair[0][1:-1] - if len(pair[1]) > 2 and ((pair[1][0] == '"' or pair[1][0] == "'") and - pair[1][1] == pair[1][0] and - pair[1][2] == pair[1][0] and - not (len(pair[1]) > 5 and - pair[1][-1] == pair[1][0] and - pair[1][-2] == pair[1][0] and - pair[1][-3] == pair[1][0])): - k = len(pair[1]) - 1 - while k > -1 and pair[1][k] == '\\': - multibackslash = not multibackslash - k -= 1 - if multibackslash: - multilinestr = pair[1][:-1] - else: - multilinestr = pair[1] + "\n" - multikey = pair[0] - else: - value, vtype = _load_value(pair[1], _dict, strictly_valid) - try: - currentlevel[pair[0]] - raise ValueError("Duplicate keys!") - except KeyError: - if multikey: - return multikey, multilinestr, multibackslash - else: - currentlevel[pair[0]] = value - - -def _load_date(val): - microsecond = 0 - tz = None - try: - if len(val) > 19: - if val[19] == '.': - if val[-1].upper() == 'Z': - subsecondval = val[20:-1] - tzval = "Z" - else: - subsecondvalandtz = val[20:] - if '+' in subsecondvalandtz: - splitpoint = subsecondvalandtz.index('+') - subsecondval = subsecondvalandtz[:splitpoint] - tzval = subsecondvalandtz[splitpoint:] - elif '-' in subsecondvalandtz: - splitpoint = subsecondvalandtz.index('-') - subsecondval = subsecondvalandtz[:splitpoint] - tzval = subsecondvalandtz[splitpoint:] - tz = TomlTz(tzval) - microsecond = int(int(subsecondval) * - (10 ** (6 - len(subsecondval)))) - else: - tz = TomlTz(val[19:]) - except ValueError: - tz = None - if "-" not in val[1:]: - return None - try: - d = datetime.datetime( - int(val[:4]), int(val[5:7]), - int(val[8:10]), int(val[11:13]), - int(val[14:16]), int(val[17:19]), microsecond, tz) - except ValueError: - return None - return d - - -def _load_unicode_escapes(v, hexbytes, prefix): - skip = False - i = len(v) - 1 - while i > -1 and v[i] == '\\': - skip = not skip - i -= 1 - for hx in hexbytes: - if skip: - skip = False - i = len(hx) - 1 - while i > -1 and hx[i] == '\\': - skip = not skip - i -= 1 - v += prefix - v += hx - continue - hxb = "" - i = 0 - hxblen = 4 - if prefix == "\\U": - hxblen = 8 - hxb = ''.join(hx[i:i + hxblen]).lower() - if hxb.strip('0123456789abcdef'): - raise ValueError("Invalid escape sequence: " + hxb) - if hxb[0] == "d" and hxb[1].strip('01234567'): - raise ValueError("Invalid escape sequence: " + hxb + - ". Only scalar unicode points are allowed.") - v += unichr(int(hxb, 16)) - v += unicode(hx[len(hxb):]) - return v - - -# Unescape TOML string values. - -# content after the \ -_escapes = ['0', 'b', 'f', 'n', 'r', 't', '"'] -# What it should be replaced by -_escapedchars = ['\0', '\b', '\f', '\n', '\r', '\t', '\"'] -# Used for substitution -_escape_to_escapedchars = dict(zip(_escapes, _escapedchars)) - - -def _unescape(v): - """Unescape characters in a TOML string.""" - i = 0 - backslash = False - while i < len(v): - if backslash: - backslash = False - if v[i] in _escapes: - v = v[:i - 1] + _escape_to_escapedchars[v[i]] + v[i + 1:] - elif v[i] == '\\': - v = v[:i - 1] + v[i:] - elif v[i] == 'u' or v[i] == 'U': - i += 1 - else: - raise ValueError("Reserved escape sequence used") - continue - elif v[i] == '\\': - backslash = True - i += 1 - return v - - -def _load_value(v, _dict, strictly_valid=True): - if not v: - raise ValueError("Empty value is invalid") - if v == 'true': - return (True, "bool") - elif v == 'false': - return (False, "bool") - elif v[0] == '"': - testv = v[1:].split('"') - triplequote = False - triplequotecount = 0 - if len(testv) > 1 and testv[0] == '' and testv[1] == '': - testv = testv[2:] - triplequote = True - closed = False - for tv in testv: - if tv == '': - if triplequote: - triplequotecount += 1 - else: - closed = True - else: - oddbackslash = False - try: - i = -1 - j = tv[i] - while j == '\\': - oddbackslash = not oddbackslash - i -= 1 - j = tv[i] - except IndexError: - pass - if not oddbackslash: - if closed: - raise ValueError("Stuff after closed string. WTF?") - else: - if not triplequote or triplequotecount > 1: - closed = True - else: - triplequotecount = 0 - escapeseqs = v.split('\\')[1:] - backslash = False - for i in escapeseqs: - if i == '': - backslash = not backslash - else: - if i[0] not in _escapes and (i[0] != 'u' and i[0] != 'U' and - not backslash): - raise ValueError("Reserved escape sequence used") - if backslash: - backslash = False - for prefix in ["\\u", "\\U"]: - if prefix in v: - hexbytes = v.split(prefix) - v = _load_unicode_escapes(hexbytes[0], hexbytes[1:], prefix) - v = _unescape(v) - if len(v) > 1 and v[1] == '"' and (len(v) < 3 or v[1] == v[2]): - v = v[2:-2] - return (v[1:-1], "str") - elif v[0] == "'": - if v[1] == "'" and (len(v) < 3 or v[1] == v[2]): - v = v[2:-2] - return (v[1:-1], "str") - elif v[0] == '[': - return (_load_array(v, _dict), "array") - elif v[0] == '{': - inline_object = _get_empty_inline_table(_dict) - _load_inline_object(v, inline_object, _dict) - return (inline_object, "inline_object") - else: - parsed_date = _load_date(v) - if parsed_date is not None: - return (parsed_date, "date") - if not strictly_valid: - raise ValueError("Weirdness with leading zeroes or " - "underscores in your number.") - itype = "int" - neg = False - if v[0] == '-': - neg = True - v = v[1:] - elif v[0] == '+': - v = v[1:] - v = v.replace('_', '') - if '.' in v or 'e' in v or 'E' in v: - if '.' in v and v.split('.', 1)[1] == '': - raise ValueError("This float is missing digits after " - "the point") - if v[0] not in '0123456789': - raise ValueError("This float doesn't have a leading digit") - v = float(v) - itype = "float" - else: - v = int(v) - if neg: - return (0 - v, itype) - return (v, itype) - - -def _bounded_string(s): - if len(s) == 0: - return True - if s[-1] != s[0]: - return False - i = -2 - backslash = False - while len(s) + i > 0: - if s[i] == "\\": - backslash = not backslash - i -= 1 - else: - break - return not backslash - - -def _load_array(a, _dict): - atype = None - retval = [] - a = a.strip() - if '[' not in a[1:-1] or "" != a[1:-1].split('[')[0].strip(): - strarray = False - tmpa = a[1:-1].strip() - if tmpa != '' and (tmpa[0] == '"' or tmpa[0] == "'"): - strarray = True - if not a[1:-1].strip().startswith('{'): - a = a[1:-1].split(',') - else: - # a is an inline object, we must find the matching parenthesis - # to define groups - new_a = [] - start_group_index = 1 - end_group_index = 2 - in_str = False - while end_group_index < len(a[1:]): - if a[end_group_index] == '"' or a[end_group_index] == "'": - if in_str: - backslash_index = end_group_index - 1 - while (backslash_index > -1 and - a[backslash_index] == '\\'): - in_str = not in_str - backslash_index -= 1 - in_str = not in_str - if in_str or a[end_group_index] != '}': - end_group_index += 1 - continue - - # Increase end_group_index by 1 to get the closing bracket - end_group_index += 1 - new_a.append(a[start_group_index:end_group_index]) - - # The next start index is at least after the closing bracket, a - # closing bracket can be followed by a comma since we are in - # an array. - start_group_index = end_group_index + 1 - while (start_group_index < len(a[1:]) and - a[start_group_index] != '{'): - start_group_index += 1 - end_group_index = start_group_index + 1 - a = new_a - b = 0 - if strarray: - while b < len(a) - 1: - ab = a[b].strip() - while (not _bounded_string(ab) or - (len(ab) > 2 and - ab[0] == ab[1] == ab[2] and - ab[-2] != ab[0] and - ab[-3] != ab[0])): - a[b] = a[b] + ',' + a[b + 1] - ab = a[b].strip() - if b < len(a) - 2: - a = a[:b + 1] + a[b + 2:] - else: - a = a[:b + 1] - b += 1 - else: - al = list(a[1:-1]) - a = [] - openarr = 0 - j = 0 - for i in _range(len(al)): - if al[i] == '[': - openarr += 1 - elif al[i] == ']': - openarr -= 1 - elif al[i] == ',' and not openarr: - a.append(''.join(al[j:i])) - j = i + 1 - a.append(''.join(al[j:])) - for i in _range(len(a)): - a[i] = a[i].strip() - if a[i] != '': - nval, ntype = _load_value(a[i], _dict) - if atype: - if ntype != atype: - raise ValueError("Not a homogeneous array") - else: - atype = ntype - retval.append(nval) - return retval - - -def dump(o, f): - """Writes out dict as toml to a file - - Args: - o: Object to dump into toml - f: File descriptor where the toml should be stored - - Returns: - String containing the toml corresponding to dictionary - - Raises: - TypeError: When anything other than file descriptor is passed - """ - - if not f.write: - raise TypeError("You can only dump an object to a file descriptor") - d = dumps(o) - f.write(d) - return d - - -def dumps(o, preserve=False): - """Stringifies input dict as toml - - Args: - o: Object to dump into toml - - preserve: Boolean parameter. If true, preserve inline tables. - - Returns: - String containing the toml corresponding to dict - """ - - retval = "" - addtoretval, sections = _dump_sections(o, "") - retval += addtoretval - while sections != {}: - newsections = {} - for section in sections: - addtoretval, addtosections = _dump_sections(sections[section], - section, preserve) - if addtoretval or (not addtoretval and not addtosections): - if retval and retval[-2:] != "\n\n": - retval += "\n" - retval += "[" + section + "]\n" - if addtoretval: - retval += addtoretval - for s in addtosections: - newsections[section + "." + s] = addtosections[s] - sections = newsections - return retval - - -def _dump_sections(o, sup, preserve=False): - retstr = "" - if sup != "" and sup[-1] != ".": - sup += '.' - retdict = o.__class__() - arraystr = "" - for section in o: - section = unicode(section) - qsection = section - if not re.match(r'^[A-Za-z0-9_-]+$', section): - if '"' in section: - qsection = "'" + section + "'" - else: - qsection = '"' + section + '"' - if not isinstance(o[section], dict): - arrayoftables = False - if isinstance(o[section], list): - for a in o[section]: - if isinstance(a, dict): - arrayoftables = True - if arrayoftables: - for a in o[section]: - arraytabstr = "\n" - arraystr += "[[" + sup + qsection + "]]\n" - s, d = _dump_sections(a, sup + qsection) - if s: - if s[0] == "[": - arraytabstr += s - else: - arraystr += s - while d != {}: - newd = {} - for dsec in d: - s1, d1 = _dump_sections(d[dsec], sup + qsection + - "." + dsec) - if s1: - arraytabstr += ("[" + sup + qsection + "." + - dsec + "]\n") - arraytabstr += s1 - for s1 in d1: - newd[dsec + "." + s1] = d1[s1] - d = newd - arraystr += arraytabstr - else: - if o[section] is not None: - retstr += (qsection + " = " + - unicode(_dump_value(o[section])) + '\n') - elif preserve and isinstance(o[section], InlineTableDict): - retstr += (qsection + " = " + _dump_inline_table(o[section])) - else: - retdict[qsection] = o[section] - retstr += arraystr - return (retstr, retdict) - - -def _dump_inline_table(section): - """Preserve inline table in its compact syntax instead of expanding - into subsection. - - https://github.com/toml-lang/toml#user-content-inline-table - """ - retval = "" - if isinstance(section, dict): - val_list = [] - for k, v in section.items(): - val = _dump_inline_table(v) - val_list.append(k + " = " + val) - retval += "{ " + ", ".join(val_list) + " }\n" - return retval - else: - return unicode(_dump_value(section)) - - -def _dump_value(v): - dump_funcs = { - str: _dump_str, - unicode: _dump_str, - list: _dump_list, - int: lambda v: v, - bool: lambda v: unicode(v).lower(), - float: _dump_float, - datetime.datetime: lambda v: v.isoformat().replace('+00:00', 'Z'), - } - # Lookup function corresponding to v's type - dump_fn = dump_funcs.get(type(v)) - if dump_fn is None and hasattr(v, '__iter__'): - dump_fn = dump_funcs[list] - # Evaluate function (if it exists) else return v - return dump_fn(v) if dump_fn is not None else dump_funcs[str](v) - - -def _dump_str(v): - if sys.version_info < (3,) and hasattr(v, 'decode') and isinstance(v, str): - v = v.decode('utf-8') - v = "%r" % v - if v[0] == 'u': - v = v[1:] - singlequote = v.startswith("'") - if singlequote or v.startswith('"'): - v = v[1:-1] - if singlequote: - v = v.replace("\\'", "'") - v = v.replace('"', '\\"') - v = v.split("\\x") - while len(v) > 1: - i = -1 - if not v[0]: - v = v[1:] - v[0] = v[0].replace("\\\\", "\\") - # No, I don't know why != works and == breaks - joinx = v[0][i] != "\\" - while v[0][:i] and v[0][i] == "\\": - joinx = not joinx - i -= 1 - if joinx: - joiner = "x" - else: - joiner = "u00" - v = [v[0] + joiner + v[1]] + v[2:] - return unicode('"' + v[0] + '"') - - -def _dump_list(v): - retval = "[" - for u in v: - retval += " " + unicode(_dump_value(u)) + "," - retval += "]" - return retval - - -def _dump_float(v): - return "{0:.16}".format(v).replace("e+0", "e+").replace("e-0", "e-") diff --git a/pipenv/vendor/tomlkit/__init__.py b/pipenv/vendor/tomlkit/__init__.py index 23d4ef74..92bfa27c 100644 --- a/pipenv/vendor/tomlkit/__init__.py +++ b/pipenv/vendor/tomlkit/__init__.py @@ -22,4 +22,4 @@ from .api import value from .api import ws -__version__ = "0.4.4" +__version__ = "0.5.2" diff --git a/pipenv/vendor/tomlkit/_compat.py b/pipenv/vendor/tomlkit/_compat.py index f94bb10e..b7407af6 100644 --- a/pipenv/vendor/tomlkit/_compat.py +++ b/pipenv/vendor/tomlkit/_compat.py @@ -141,9 +141,11 @@ PY36 = sys.version_info >= (3, 6) if PY2: unicode = unicode chr = unichr + long = long else: unicode = str chr = chr + long = int def decode(string, encodings=None): diff --git a/pipenv/vendor/tomlkit/_utils.py b/pipenv/vendor/tomlkit/_utils.py index f62a354a..0a68be9f 100644 --- a/pipenv/vendor/tomlkit/_utils.py +++ b/pipenv/vendor/tomlkit/_utils.py @@ -9,19 +9,30 @@ from datetime import timedelta from ._compat import decode from ._compat import timezone +RFC_3339_LOOSE = re.compile( + "^" + r"(([0-9]+)-(\d{2})-(\d{2}))?" # Date + "(" + "([T ])?" # Separator + r"(\d{2}):(\d{2}):(\d{2})(\.([0-9]+))?" # Time + r"((Z)|([\+|\-]([01][0-9]|2[0-3]):([0-5][0-9])))?" # Timezone + ")?" + "$" +) + RFC_3339_DATETIME = re.compile( "^" "([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])" # Date "[T ]" # Separator - "([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.([0-9]+))?" # Time - "((Z)|([\+|\-]([01][0-9]|2[0-3]):([0-5][0-9])))?" # Timezone + r"([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.([0-9]+))?" # Time + r"((Z)|([\+|\-]([01][0-9]|2[0-3]):([0-5][0-9])))?" # Timezone "$" ) RFC_3339_DATE = re.compile("^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])$") RFC_3339_TIME = re.compile( - "^([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.([0-9]+))?$" + r"^([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.([0-9]+))?$" ) _utc = timezone(timedelta(), "UTC") diff --git a/pipenv/vendor/tomlkit/container.py b/pipenv/vendor/tomlkit/container.py index bb3696d9..9b5db5cb 100644 --- a/pipenv/vendor/tomlkit/container.py +++ b/pipenv/vendor/tomlkit/container.py @@ -4,12 +4,12 @@ from ._compat import decode from .exceptions import KeyAlreadyPresent from .exceptions import NonExistentKey from .items import AoT -from .items import Bool from .items import Comment from .items import Item from .items import Key from .items import Null from .items import Table +from .items import Trivia from .items import Whitespace from .items import item as _item @@ -74,7 +74,7 @@ class Container(dict): return self.append(key, item) - def append(self, key, item): # type: (Union[Key, str], Item) -> Container + def append(self, key, item): # type: (Union[Key, str, None], Item) -> Container if not isinstance(key, Key) and key is not None: key = Key(key) @@ -99,7 +99,11 @@ class Container(dict): self.append(None, Whitespace("\n")) if key is not None and key in self: - current = self._body[self._map[key]][1] + current_idx = self._map[key] + if isinstance(current_idx, tuple): + current_idx = current_idx[0] + + current = self._body[current_idx][1] if isinstance(item, Table): if not isinstance(current, (Table, AoT)): raise KeyAlreadyPresent(key) @@ -121,7 +125,7 @@ class Container(dict): current.append(k, v) return self - else: + elif not item.is_super_table(): raise KeyAlreadyPresent(key) elif isinstance(item, AoT): if not isinstance(current, AoT): @@ -173,7 +177,23 @@ class Container(dict): else: return self._insert_at(0, key, item) - self._map[key] = len(self._body) + if key in self._map: + current_idx = self._map[key] + if isinstance(current_idx, tuple): + current_idx = current_idx[0] + + current = self._body[current_idx][1] + if key is not None and not isinstance(current, Table): + raise KeyAlreadyPresent(key) + + # Adding sub tables to a currently existing table + idx = self._map[key] + if not isinstance(idx, tuple): + idx = (idx,) + + self._map[key] = idx + (len(self._body),) + else: + self._map[key] = len(self._body) self._body.append((key, item)) @@ -190,7 +210,16 @@ class Container(dict): if idx is None: raise NonExistentKey(key) - self._body[idx] = (None, Null()) + if isinstance(idx, tuple): + for i in idx: + self._body[i] = (None, Null()) + else: + old_data = self._body[idx][1] + trivia = getattr(old_data, "trivia", None) + if trivia and trivia.comment: + self._body[idx] = (None, Comment(Trivia(comment_ws="", comment=trivia.comment))) + else: + self._body[idx] = (None, Null()) super(Container, self).__delitem__(key.key) @@ -220,7 +249,16 @@ class Container(dict): # Increment indices after the current index for k, v in self._map.items(): - if v > idx: + if isinstance(v, tuple): + new_indices = [] + for v_ in v: + if v_ > idx: + v_ = v_ + 1 + + new_indices.append(v_) + + self._map[k] = tuple(new_indices) + elif v > idx: self._map[k] = v + 1 self._map[other_key] = idx + 1 @@ -253,7 +291,16 @@ class Container(dict): # Increment indices after the current index for k, v in self._map.items(): - if v >= idx: + if isinstance(v, tuple): + new_indices = [] + for v_ in v: + if v_ >= idx: + v_ = v_ + 1 + + new_indices.append(v_) + + self._map[k] = tuple(new_indices) + elif v >= idx: self._map[k] = v + 1 self._map[key] = idx @@ -282,29 +329,7 @@ class Container(dict): s = "" for k, v in self._body: if k is not None: - if False: - key = k.as_string() - - for _k, _v in v.value.body: - if _k is None: - s += v.as_string() - elif isinstance(_v, Table): - s += v.as_string(prefix=key) - else: - _key = key - if prefix is not None: - _key = prefix + "." + _key - - s += "{}{}{}{}{}{}{}".format( - _v.trivia.indent, - _key + "." + decode(_k.as_string()), - _k.sep, - decode(_v.as_string()), - _v.trivia.comment_ws, - decode(_v.trivia.comment), - _v.trivia.trail, - ) - elif isinstance(v, Table): + if isinstance(v, Table): s += self._render_table(k, v) elif isinstance(v, AoT): s += self._render_aot(k, v) @@ -328,7 +353,12 @@ class Container(dict): if prefix is not None: _key = prefix + "." + _key - if not table.is_super_table(): + if not table.is_super_table() or ( + any( + not isinstance(v, (Table, AoT, Whitespace)) for _, v in table.value.body + ) + and not key.is_dotted() + ): open_, close = "[", "]" if table.is_aot_element(): open_, close = "[[", "]]" @@ -461,7 +491,7 @@ class Container(dict): return key in self._map - def __getitem__(self, key): # type: (Union[Key, str]) -> Item + def __getitem__(self, key): # type: (Union[Key, str]) -> Union[Item, Container] if not isinstance(key, Key): key = Key(key) @@ -469,6 +499,20 @@ class Container(dict): if idx is None: raise NonExistentKey(key) + if isinstance(idx, tuple): + container = Container(True) + + for i in idx: + item = self._body[i][1] + + if isinstance(item, Table): + for k, v in item.value.body: + container.append(k, v) + else: + container.append(key, item) + + return container + item = self._body[idx][1] return item.value @@ -499,11 +543,20 @@ class Container(dict): def _replace_at( self, idx, new_key, value - ): # type: (int, Union[Key, str], Item) -> None + ): # type: (Union[int, Tuple[int]], Union[Key, str], Item) -> None + if isinstance(idx, tuple): + for i in idx[1:]: + self._body[i] = (None, Null()) + + idx = idx[0] + k, v = self._body[idx] self._map[new_key] = self._map.pop(k) + if isinstance(self._map[new_key], tuple): + self._map[new_key] = self._map[new_key][0] + value = _item(value) # Copying trivia @@ -513,6 +566,10 @@ class Container(dict): value.trivia.comment = v.trivia.comment value.trivia.trail = v.trivia.trail + if isinstance(value, Table): + # Insert a cosmetic new line for tables + value.append(None, Whitespace("\n")) + self._body[idx] = (new_key, value) super(Container, self).__setitem__(new_key.key, value.value) @@ -525,3 +582,21 @@ class Container(dict): return NotImplemented return self.value == other + + def _getstate(self, protocol): + return (self._parsed,) + + def __reduce__(self): + return self.__reduce_ex__(2) + + def __reduce_ex__(self, protocol): + return ( + self.__class__, + self._getstate(protocol), + (self._map, self._body, self._parsed), + ) + + def __setstate__(self, state): + self._map = state[0] + self._body = state[1] + self._parsed = state[2] diff --git a/pipenv/vendor/tomlkit/exceptions.py b/pipenv/vendor/tomlkit/exceptions.py index d889a924..c1a4e620 100644 --- a/pipenv/vendor/tomlkit/exceptions.py +++ b/pipenv/vendor/tomlkit/exceptions.py @@ -1,3 +1,4 @@ + class TOMLKitError(Exception): pass @@ -23,6 +24,14 @@ class ParseError(ValueError, TOMLKitError): "{} at line {} col {}".format(message, self._line, self._col) ) + @property + def line(self): + return self._line + + @property + def col(self): + return self._col + class MixedArrayTypesError(ParseError): """ @@ -35,6 +44,50 @@ class MixedArrayTypesError(ParseError): super(MixedArrayTypesError, self).__init__(line, col, message=message) +class InvalidNumberError(ParseError): + """ + A numeric field was improperly specified. + """ + + def __init__(self, line, col): # type: (int, int) -> None + message = "Invalid number" + + super(InvalidNumberError, self).__init__(line, col, message=message) + + +class InvalidDateTimeError(ParseError): + """ + A datetime field was improperly specified. + """ + + def __init__(self, line, col): # type: (int, int) -> None + message = "Invalid datetime" + + super(InvalidDateTimeError, self).__init__(line, col, message=message) + + +class InvalidDateError(ParseError): + """ + A date field was improperly specified. + """ + + def __init__(self, line, col): # type: (int, int) -> None + message = "Invalid date" + + super(InvalidDateError, self).__init__(line, col, message=message) + + +class InvalidTimeError(ParseError): + """ + A date field was improperly specified. + """ + + def __init__(self, line, col): # type: (int, int) -> None + message = "Invalid time" + + super(InvalidTimeError, self).__init__(line, col, message=message) + + class InvalidNumberOrDateError(ParseError): """ A numeric or date field was improperly specified. @@ -46,6 +99,17 @@ class InvalidNumberOrDateError(ParseError): super(InvalidNumberOrDateError, self).__init__(line, col, message=message) +class InvalidUnicodeValueError(ParseError): + """ + A unicode code was improperly specified. + """ + + def __init__(self, line, col): # type: (int, int) -> None + message = "Invalid unicode value" + + super(InvalidUnicodeValueError, self).__init__(line, col, message=message) + + class UnexpectedCharError(ParseError): """ An unexpected character was found during parsing. @@ -85,7 +149,7 @@ class InvalidCharInStringError(ParseError): """ def __init__(self, line, col, char): # type: (int, int, str) -> None - message = "Invalid character '{}' in string".format(char) + message = "Invalid character {} in string".format(repr(char)) super(InvalidCharInStringError, self).__init__(line, col, message=message) @@ -106,7 +170,9 @@ class InternalParserError(ParseError): An error that indicates a bug in the parser. """ - def __init__(self, line, col, message=None): # type: (int, int) -> None + def __init__( + self, line, col, message=None + ): # type: (int, int, Optional[str]) -> None msg = "Internal parser error" if message: msg += " ({})".format(message) diff --git a/pipenv/vendor/tomlkit/items.py b/pipenv/vendor/tomlkit/items.py index 26f24701..cccfd4a1 100644 --- a/pipenv/vendor/tomlkit/items.py +++ b/pipenv/vendor/tomlkit/items.py @@ -6,17 +6,21 @@ import string from datetime import date from datetime import datetime from datetime import time -import sys -if sys.version_info >= (3, 4): - from enum import Enum -else: - from pipenv.vendor.backports.enum import Enum from ._compat import PY2 from ._compat import decode +from ._compat import long from ._compat import unicode from ._utils import escape_string +if PY2: + from pipenv.vendor.backports.enum import Enum + from pipenv.vendor.backports.functools_lru_cache import lru_cache +else: + from enum import Enum + from functools import lru_cache +from toml.decoder import InlineTableDict + def item(value, _parent=None): from .container import Container @@ -31,7 +35,10 @@ def item(value, _parent=None): elif isinstance(value, float): return Float(value, Trivia(), str(value)) elif isinstance(value, dict): - val = Table(Container(), Trivia(), False) + if isinstance(value, InlineTableDict): + val = InlineTable(Container(), Trivia()) + else: + val = Table(Container(), Trivia(), False) for k, v in sorted(value.items(), key=lambda i: (isinstance(i[1], dict), i[0])): val[k] = item(v, _parent=val) @@ -75,18 +82,63 @@ def item(value, _parent=None): class StringType(Enum): - + # Single Line Basic SLB = '"' + # Multi Line Basic MLB = '"""' + # Single Line Literal SLL = "'" + # Multi Line Literal MLL = "'''" + @property + @lru_cache(maxsize=None) + def unit(self): # type: () -> str + return self.value[0] + + @lru_cache(maxsize=None) + def is_basic(self): # type: () -> bool + return self in {StringType.SLB, StringType.MLB} + + @lru_cache(maxsize=None) def is_literal(self): # type: () -> bool return self in {StringType.SLL, StringType.MLL} + @lru_cache(maxsize=None) + def is_singleline(self): # type: () -> bool + return self in {StringType.SLB, StringType.SLL} + + @lru_cache(maxsize=None) def is_multiline(self): # type: () -> bool return self in {StringType.MLB, StringType.MLL} + @lru_cache(maxsize=None) + def toggle(self): # type: () -> StringType + return { + StringType.SLB: StringType.MLB, + StringType.MLB: StringType.SLB, + StringType.SLL: StringType.MLL, + StringType.MLL: StringType.SLL, + }[self] + + +class BoolType(Enum): + TRUE = "true" + FALSE = "false" + + @lru_cache(maxsize=None) + def __bool__(self): + return {BoolType.TRUE: True, BoolType.FALSE: False}[self] + + if PY2: + __nonzero__ = __bool__ # for PY2 + + def __iter__(self): + return iter(self.value) + + def __len__(self): + return len(self.value) + class Trivia: """ @@ -158,7 +210,10 @@ class Key: return hash(self.key) def __eq__(self, other): # type: (Key) -> bool - return self.key == other.key + if isinstance(other, Key): + return self.key == other.key + + return self.key == other def __str__(self): # type: () -> str return self.as_string() @@ -205,6 +260,15 @@ class Item(object): return self + def _getstate(self, protocol=3): + return (self._trivia,) + + def __reduce__(self): + return self.__reduce_ex__(2) + + def __reduce_ex__(self, protocol): + return self.__class__, self._getstate(protocol) + class Whitespace(Item): """ @@ -240,6 +304,9 @@ class Whitespace(Item): def __repr__(self): # type: () -> str return "<{} {}>".format(self.__class__.__name__, repr(self._s)) + def _getstate(self, protocol=3): + return self._s, self._fixed + class Comment(Item): """ @@ -259,7 +326,7 @@ class Comment(Item): return "{}{}".format(self._trivia.indent, decode(self._trivia.comment)) -class Integer(int, Item): +class Integer(long, Item): """ An integer literal. """ @@ -273,7 +340,7 @@ class Integer(int, Item): self._raw = raw self._sign = False - if re.match("^[+\-]\d+$", raw): + if re.match(r"^[+\-]\d+$", raw): self._sign = True @property @@ -322,6 +389,9 @@ class Integer(int, Item): return Integer(result, self._trivia, raw) + def _getstate(self, protocol=3): + return int(self), self._trivia, self._raw + class Float(float, Item): """ @@ -337,7 +407,7 @@ class Float(float, Item): self._raw = raw self._sign = False - if re.match("^[+\-].+$", raw): + if re.match(r"^[+\-].+$", raw): self._sign = True @property @@ -386,16 +456,19 @@ class Float(float, Item): return Float(result, self._trivia, raw) + def _getstate(self, protocol=3): + return float(self), self._trivia, self._raw + class Bool(Item): """ A boolean literal. """ - def __init__(self, value, trivia): # type: (float, Trivia) -> None + def __init__(self, t, trivia): # type: (float, Trivia) -> None super(Bool, self).__init__(trivia) - self._value = value + self._value = bool(t) @property def discriminant(self): # type: () -> int @@ -408,13 +481,16 @@ class Bool(Item): def as_string(self): # type: () -> str return str(self._value).lower() + def _getstate(self, protocol=3): + return self._value, self._trivia -class DateTime(datetime, Item): + +class DateTime(Item, datetime): """ A datetime literal. """ - def __new__(cls, value, *_): # type: (datetime, ...) -> datetime + def __new__(cls, value, *_): # type: (..., datetime, ...) -> datetime return datetime.__new__( cls, value.year, @@ -458,13 +534,29 @@ class DateTime(datetime, Item): return DateTime(result, self._trivia, raw) + def _getstate(self, protocol=3): + return ( + datetime( + self.year, + self.month, + self.day, + self.hour, + self.minute, + self.second, + self.microsecond, + self.tzinfo, + ), + self._trivia, + self._raw, + ) -class Date(date, Item): + +class Date(Item, date): """ A date literal. """ - def __new__(cls, value, *_): # type: (date, ...) -> date + def __new__(cls, value, *_): # type: (..., date, ...) -> date return date.__new__(cls, value.year, value.month, value.day) def __init__(self, _, trivia, raw): # type: (date, Trivia, str) -> None @@ -498,8 +590,11 @@ class Date(date, Item): return Date(result, self._trivia, raw) + def _getstate(self, protocol=3): + return (datetime(self.year, self.month, self.day), self._trivia, self._raw) -class Time(time, Item): + +class Time(Item, time): """ A time literal. """ @@ -525,6 +620,13 @@ class Time(time, Item): def as_string(self): # type: () -> str return self._raw + def _getstate(self, protocol=3): + return ( + time(self.hour, self.minute, self.second, self.microsecond, self.tzinfo), + self._trivia, + self._raw, + ) + class Array(Item, list): """ @@ -623,6 +725,9 @@ class Array(Item, list): def __repr__(self): return str(self) + def _getstate(self, protocol=3): + return self._value, self._trivia + class Table(Item, dict): """ @@ -637,7 +742,7 @@ class Table(Item, dict): is_super_table=False, name=None, display_name=None, - ): # type: (tomlkit.container.Container, Trivia, bool) -> None + ): # type: (tomlkit.container.Container, Trivia, bool, ...) -> None super(Table, self).__init__(trivia) self.name = name @@ -658,10 +763,6 @@ class Table(Item, dict): def discriminant(self): # type: () -> int return 9 - @property - def value(self): # type: () -> tomlkit.container.Container - return self._value - def add(self, key, item=None): # type: (Union[Key, Item, str], Any) -> Item if item is None: if not isinstance(key, (Comment, Whitespace)): @@ -790,6 +891,16 @@ class Table(Item, dict): def __repr__(self): return super(Table, self).__repr__() + def _getstate(self, protocol=3): + return ( + self._value, + self._trivia, + self._is_aot_element, + self._is_super_table, + self.name, + self.display_name, + ) + class InlineTable(Item, dict): """ @@ -825,6 +936,8 @@ class InlineTable(Item, dict): if not isinstance(_item, (Whitespace, Comment)): if not _item.trivia.indent and len(self._value) > 0: _item.trivia.indent = " " + if _item.trivia.comment: + _item.trivia.comment = "" self._value.append(key, _item) @@ -904,6 +1017,8 @@ class InlineTable(Item, dict): if key is not None: super(InlineTable, self).__setitem__(key, value) + if value.trivia.comment: + value.trivia.comment = "" m = re.match("(?s)^[^ ]*([ ]+).*$", self._trivia.indent) if not m: @@ -924,6 +1039,9 @@ class InlineTable(Item, dict): def __repr__(self): return super(InlineTable, self).__repr__() + def _getstate(self, protocol=3): + return (self._value, self._trivia) + class String(unicode, Item): """ @@ -965,6 +1083,9 @@ class String(unicode, Item): def _new(self, result): return String(self._t, result, result, self._trivia) + def _getstate(self, protocol=3): + return self._t, unicode(self), self._original, self._trivia + class AoT(Item, list): """ @@ -974,7 +1095,7 @@ class AoT(Item, list): def __init__( self, body, name=None, parsed=False ): # type: (List[Table], Optional[str]) -> None - self.name = None + self.name = name self._body = [] self._parsed = parsed @@ -1025,6 +1146,9 @@ class AoT(Item, list): def __repr__(self): # type: () -> str return "".format(self.value) + def _getstate(self, protocol=3): + return self._body, self.name, self._parsed + class Null(Item): """ @@ -1044,3 +1168,6 @@ class Null(Item): def as_string(self): # type: () -> str return "" + + def _getstate(self, protocol=3): + return tuple() diff --git a/pipenv/vendor/tomlkit/parser.py b/pipenv/vendor/tomlkit/parser.py index 45c8ee8c..3f507bb4 100644 --- a/pipenv/vendor/tomlkit/parser.py +++ b/pipenv/vendor/tomlkit/parser.py @@ -1,24 +1,24 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals -import datetime -import itertools import re import string -from copy import copy - -from ._compat import PY2 from ._compat import chr from ._compat import decode from ._utils import _escaped +from ._utils import RFC_3339_LOOSE from ._utils import parse_rfc3339 from .container import Container from .exceptions import EmptyKeyError from .exceptions import EmptyTableNameError from .exceptions import InternalParserError from .exceptions import InvalidCharInStringError -from .exceptions import InvalidNumberOrDateError +from .exceptions import InvalidDateTimeError +from .exceptions import InvalidDateError +from .exceptions import InvalidTimeError +from .exceptions import InvalidNumberError +from .exceptions import InvalidUnicodeValueError from .exceptions import MixedArrayTypesError from .exceptions import ParseError from .exceptions import UnexpectedCharError @@ -26,12 +26,14 @@ from .exceptions import UnexpectedEofError from .items import AoT from .items import Array from .items import Bool +from .items import BoolType from .items import Comment from .items import Date from .items import DateTime from .items import Float from .items import InlineTable from .items import Integer +from .items import Item from .items import Key from .items import KeyType from .items import Null @@ -41,6 +43,7 @@ from .items import Table from .items import Time from .items import Trivia from .items import Whitespace +from .source import Source from .toml_char import TOMLChar from .toml_document import TOMLDocument @@ -52,66 +55,69 @@ class Parser: def __init__(self, string): # type: (str) -> None # Input to parse - self._src = decode(string) # type: str - # Iterator used for getting characters from src. - self._chars = iter([(i, TOMLChar(c)) for i, c in enumerate(self._src)]) - # Current byte offset into src. - self._idx = 0 - # Current character - self._current = TOMLChar("") # type: TOMLChar - # Index into src between which and idx slices will be extracted - self._marker = 0 + self._src = Source(decode(string)) self._aot_stack = [] - self.inc() + @property + def _state(self): + return self._src.state + + @property + def _idx(self): + return self._src.idx + + @property + def _current(self): + return self._src.current + + @property + def _marker(self): + return self._src.marker def extract(self): # type: () -> str """ Extracts the value between marker and index """ - if self.end(): - return self._src[self._marker :] - else: - return self._src[self._marker : self._idx] + return self._src.extract() - def inc(self): # type: () -> bool + def inc(self, exception=None): # type: (Optional[ParseError.__class__]) -> bool """ Increments the parser if the end of the input has not been reached. Returns whether or not it was able to advance. """ - try: - self._idx, self._current = next(self._chars) + return self._src.inc(exception=exception) - return True - except StopIteration: - self._idx = len(self._src) - self._current = TOMLChar("\0") - - return False - - def inc_n(self, n): # type: (int) -> bool + def inc_n(self, n, exception=None): # type: (int, Optional[ParseError]) -> bool """ Increments the parser by n characters if the end of the input has not been reached. """ - for _ in range(n): - if not self.inc(): - return False + return self._src.inc_n(n=n, exception=exception) - return True + def consume(self, chars, min=0, max=-1): + """ + Consume chars until min/max is satisfied is valid. + """ + return self._src.consume(chars=chars, min=min, max=max) def end(self): # type: () -> bool """ Returns True if the parser has reached the end of the input. """ - return self._idx >= len(self._src) or self._current == "\0" + return self._src.end() def mark(self): # type: () -> None """ Sets the marker to the index's current position """ - self._marker = self._idx + self._src.mark() + + def parse_error(self, exception=ParseError, *args): + """ + Creates a generic "parse error" at the current position. + """ + return self._src.parse_error(exception, *args) def parse(self): # type: () -> TOMLDocument body = TOMLDocument(True) @@ -171,27 +177,6 @@ class Parser: return True - def parse_error(self, kind=ParseError, args=None): # type: () -> None - """ - Creates a generic "parse error" at the current position. - """ - line, col = self._to_linecol(self._idx) - - if args: - return kind(line, col, *args) - else: - return kind(line, col) - - def _to_linecol(self, offset): # type: (int) -> Tuple[int, int] - cur = 0 - for i, line in enumerate(self._src.splitlines()): - if cur + len(line) + 1 > offset: - return (i + 1, offset - cur) - - cur += len(line) + 1 - - return len(self._src.splitlines()), 0 - def _is_child(self, parent, child): # type: (str, str) -> bool """ Returns whether a key is strictly a child of another key. @@ -254,55 +239,35 @@ class Parser: if the item is value-like. """ self.mark() - saved_idx = self._save_idx() + with self._state as state: + while True: + c = self._current + if c == "\n": + # Found a newline; Return all whitespace found up to this point. + self.inc() - while True: - c = self._current - if c == "\n": - # Found a newline; Return all whitespace found up to this point. - self.inc() + return None, Whitespace(self.extract()) + elif c in " \t\r": + # Skip whitespace. + if not self.inc(): + return None, Whitespace(self.extract()) + elif c == "#": + # Found a comment, parse it + indent = self.extract() + cws, comment, trail = self._parse_comment_trail() - return (None, Whitespace(self.extract())) - elif c in " \t\r": - # Skip whitespace. - if not self.inc(): - return (None, Whitespace(self.extract())) - elif c == "#": - # Found a comment, parse it - indent = self.extract() - cws, comment, trail = self._parse_comment_trail() + return None, Comment(Trivia(indent, cws, comment, trail)) + elif c == "[": + # Found a table, delegate to the calling function. + return + else: + # Begining of a KV pair. + # Return to beginning of whitespace so it gets included + # as indentation for the KV about to be parsed. + state.restore = True + break - return (None, Comment(Trivia(indent, cws, comment, trail))) - elif c == "[": - # Found a table, delegate to the calling function. - return - else: - # Begining of a KV pair. - # Return to beginning of whitespace so it gets included - # as indentation for the KV about to be parsed. - self._restore_idx(*saved_idx) - key, value = self._parse_key_value(True) - - return key, value - - def _save_idx(self): # type: () -> Tuple[Iterator, int, str] - if PY2: - # Python 2.7 does not allow to directly copy - # an iterator, so we have to make tees of the original - # chars iterator. - chars1, chars2 = itertools.tee(self._chars) - - # We can no longer use the original chars iterator. - self._chars = chars1 - - return chars2, self._idx, self._current - - return copy(self._chars), self._idx, self._current - - def _restore_idx(self, chars, idx, current): # type: (Iterator, int, str) -> None - self._chars = chars - self._idx = idx - self._current = current + return self._parse_key_value(True) def _parse_comment_trail(self): # type: () -> Tuple[str, str, str] """ @@ -336,10 +301,10 @@ class Parser: self.mark() break - elif c in " \t\r,": + elif c in " \t\r": self.inc() else: - raise self.parse_error(UnexpectedCharError, (c)) + raise self.parse_error(UnexpectedCharError, c) if self.end(): break @@ -359,9 +324,7 @@ class Parser: return comment_ws, comment, trail - def _parse_key_value( - self, parse_comment=False, inline=True - ): # type: (bool, bool) -> (Key, Item) + def _parse_key_value(self, parse_comment=False): # type: (bool) -> (Key, Item) # Leading indent self.mark() @@ -381,7 +344,7 @@ class Parser: while self._current.is_kv_sep() and self.inc(): if self._current == "=": if found_equals: - raise self.parse_error(UnexpectedCharError, ("=",)) + raise self.parse_error(UnexpectedCharError, "=") else: found_equals = True pass @@ -471,7 +434,7 @@ class Parser: def _handle_dotted_key( self, container, key, value - ): # type: (Container, Key) -> None + ): # type: (Container, Key, Any) -> None names = tuple(self._split_table_name(key.key)) name = names[0] name._dotted = True @@ -508,84 +471,22 @@ class Parser: Attempts to parse a value at the current position. """ self.mark() + c = self._current trivia = Trivia() - c = self._current - if c == '"': + if c == StringType.SLB.value: return self._parse_basic_string() - elif c == "'": + elif c == StringType.SLL.value: return self._parse_literal_string() - elif c == "t" and self._src[self._idx :].startswith("true"): - # Boolean: true - self.inc_n(4) - - return Bool(True, trivia) - elif c == "f" and self._src[self._idx :].startswith("false"): - # Boolean: true - self.inc_n(5) - - return Bool(False, trivia) + elif c == BoolType.TRUE.value[0]: + return self._parse_true() + elif c == BoolType.FALSE.value[0]: + return self._parse_false() elif c == "[": - # Array - elems = [] # type: List[Item] - self.inc() - - while self._current != "]": - self.mark() - while self._current.is_ws() or self._current == ",": - self.inc() - - if self._idx != self._marker: - elems.append(Whitespace(self.extract())) - - if self._current == "]": - break - - if self._current == "#": - cws, comment, trail = self._parse_comment_trail() - - next_ = Comment(Trivia("", cws, comment, trail)) - else: - next_ = self._parse_value() - - elems.append(next_) - - self.inc() - - try: - res = Array(elems, trivia) - except ValueError: - raise self.parse_error(MixedArrayTypesError) - - if res.is_homogeneous(): - return res - - raise self.parse_error(MixedArrayTypesError) + return self._parse_array() elif c == "{": - # Inline table - elems = Container(True) - self.inc() - - while self._current != "}": - self.mark() - while self._current.is_spaces() or self._current == ",": - self.inc() - - if self._idx != self._marker: - ws = self.extract().lstrip(",") - if ws: - elems.append(None, Whitespace(ws)) - - if self._current == "}": - break - - key, val = self._parse_key_value(False, inline=True) - elems.append(key, val) - - self.inc() - - return InlineTable(elems, trivia) - elif c in string.digits + "+-" or self._peek(4) in { + return self._parse_inline_table() + elif c in "+-" or self._peek(4) in { "+inf", "-inf", "inf", @@ -593,7 +494,7 @@ class Parser: "-nan", "nan", }: - # Integer, Float, Date, Time or DateTime + # Number while self._current not in " \t\n\r#,]}" and self.inc(): pass @@ -603,24 +504,166 @@ class Parser: if item is not None: return item - try: - res = parse_rfc3339(raw) - except ValueError: - res = None + raise self.parse_error(InvalidNumberError) + elif c in string.digits: + # Integer, Float, Date, Time or DateTime + while self._current not in " \t\n\r#,]}" and self.inc(): + pass - if res is None: - raise self.parse_error(InvalidNumberOrDateError) + raw = self.extract() - if isinstance(res, datetime.datetime): - return DateTime(res, trivia, raw) - elif isinstance(res, datetime.time): - return Time(res, trivia, raw) - elif isinstance(res, datetime.date): - return Date(res, trivia, raw) - else: - raise self.parse_error(InvalidNumberOrDateError) + m = RFC_3339_LOOSE.match(raw) + if m: + if m.group(1) and m.group(5): + # datetime + try: + return DateTime(parse_rfc3339(raw), trivia, raw) + except ValueError: + raise self.parse_error(InvalidDateTimeError) + + if m.group(1): + try: + return Date(parse_rfc3339(raw), trivia, raw) + except ValueError: + raise self.parse_error(InvalidDateError) + + if m.group(5): + try: + return Time(parse_rfc3339(raw), trivia, raw) + except ValueError: + raise self.parse_error(InvalidTimeError) + + item = self._parse_number(raw, trivia) + if item is not None: + return item + + raise self.parse_error(InvalidNumberError) else: - raise self.parse_error(UnexpectedCharError, (c)) + raise self.parse_error(UnexpectedCharError, c) + + def _parse_true(self): + return self._parse_bool(BoolType.TRUE) + + def _parse_false(self): + return self._parse_bool(BoolType.FALSE) + + def _parse_bool(self, style): # type: (BoolType) -> Bool + with self._state: + style = BoolType(style) + + # only keep parsing for bool if the characters match the style + # try consuming rest of chars in style + for c in style: + self.consume(c, min=1, max=1) + + return Bool(style, Trivia()) + + def _parse_array(self): # type: () -> Array + # Consume opening bracket, EOF here is an issue (middle of array) + self.inc(exception=UnexpectedEofError) + + elems = [] # type: List[Item] + prev_value = None + while True: + # consume whitespace + mark = self._idx + self.consume(TOMLChar.SPACES) + newline = self.consume(TOMLChar.NL) + indent = self._src[mark : self._idx] + if newline: + elems.append(Whitespace(indent)) + continue + + # consume comment + if self._current == "#": + cws, comment, trail = self._parse_comment_trail() + elems.append(Comment(Trivia(indent, cws, comment, trail))) + continue + + # consume indent + if indent: + elems.append(Whitespace(indent)) + continue + + # consume value + if not prev_value: + try: + elems.append(self._parse_value()) + prev_value = True + continue + except UnexpectedCharError: + pass + + # consume comma + if prev_value and self._current == ",": + self.inc(exception=UnexpectedEofError) + elems.append(Whitespace(",")) + prev_value = False + continue + + # consume closing bracket + if self._current == "]": + # consume closing bracket, EOF here doesn't matter + self.inc() + break + + raise self.parse_error(UnexpectedCharError, self._current) + + try: + res = Array(elems, Trivia()) + except ValueError: + pass + else: + if res.is_homogeneous(): + return res + + raise self.parse_error(MixedArrayTypesError) + + def _parse_inline_table(self): # type: () -> InlineTable + # consume opening bracket, EOF here is an issue (middle of array) + self.inc(exception=UnexpectedEofError) + + elems = Container(True) + trailing_comma = None + while True: + # consume leading whitespace + mark = self._idx + self.consume(TOMLChar.SPACES) + raw = self._src[mark : self._idx] + if raw: + elems.add(Whitespace(raw)) + + if not trailing_comma: + # None: empty inline table + # False: previous key-value pair was not followed by a comma + if self._current == "}": + # consume closing bracket, EOF here doesn't matter + self.inc() + break + if trailing_comma is False: + raise self.parse_error(UnexpectedCharError, self._current) + else: + # True: previous key-value pair was followed by a comma + if self._current == "}": + raise self.parse_error(UnexpectedCharError, self._current) + + key, val = self._parse_key_value(False) + elems.add(key, val) + + # consume trailing whitespace + mark = self._idx + self.consume(TOMLChar.SPACES) + raw = self._src[mark : self._idx] + if raw: + elems.add(Whitespace(raw)) + + # consume trailing comma + trailing_comma = self._current == "," + if trailing_comma: + # consume closing bracket, EOF here is an issue (middle of inline table) + self.inc(exception=UnexpectedEofError) + + return InlineTable(elems, Trivia()) def _parse_number(self, raw, trivia): # type: (str, Trivia) -> Optional[Item] # Leading zeros are not allowed @@ -668,141 +711,146 @@ class Parser: except ValueError: return - def _parse_literal_string(self): # type: () -> Item - return self._parse_string("'") + def _parse_literal_string(self): # type: () -> String + with self._state: + return self._parse_string(StringType.SLL) - def _parse_basic_string(self): # type: () -> Item - return self._parse_string('"') + def _parse_basic_string(self): # type: () -> String + with self._state: + return self._parse_string(StringType.SLB) - def _parse_string(self, delim): # type: (str) -> Item - multiline = False + def _parse_escaped_char(self, multiline): + if multiline and self._current.is_ws(): + # When the last non-whitespace character on a line is + # a \, it will be trimmed along with all whitespace + # (including newlines) up to the next non-whitespace + # character or closing delimiter. + # """\ + # hello \ + # world""" + tmp = "" + while self._current.is_ws(): + tmp += self._current + # consume the whitespace, EOF here is an issue + # (middle of string) + self.inc(exception=UnexpectedEofError) + continue + + # the escape followed by whitespace must have a newline + # before any other chars + if "\n" not in tmp: + raise self.parse_error(InvalidCharInStringError, self._current) + + return "" + + if self._current in _escaped: + c = _escaped[self._current] + + # consume this char, EOF here is an issue (middle of string) + self.inc(exception=UnexpectedEofError) + + return c + + if self._current in {"u", "U"}: + # this needs to be a unicode + u, ue = self._peek_unicode(self._current == "U") + if u is not None: + # consume the U char and the unicode value + self.inc_n(len(ue) + 1) + + return u + + raise self.parse_error(InvalidUnicodeValueError) + + raise self.parse_error(InvalidCharInStringError, self._current) + + def _parse_string(self, delim): # type: (StringType) -> String + # only keep parsing for string if the current character matches the delim + if self._current != delim.unit: + raise self.parse_error( + InternalParserError, + "Invalid character for string type {}".format(delim), + ) + + # consume the opening/first delim, EOF here is an issue + # (middle of string or middle of delim) + self.inc(exception=UnexpectedEofError) + + if self._current == delim.unit: + # consume the closing/second delim, we do not care if EOF occurs as + # that would simply imply an empty single line string + if not self.inc() or self._current != delim.unit: + # Empty string + return String(delim, "", "", Trivia()) + + # consume the third delim, EOF here is an issue (middle of string) + self.inc(exception=UnexpectedEofError) + + delim = delim.toggle() # convert delim to multi delim + + self.mark() # to extract the original string with whitespace and all value = "" - if delim == "'": - str_type = StringType.SLL - else: - str_type = StringType.SLB + # A newline immediately following the opening delimiter will be trimmed. + if delim.is_multiline() and self._current == "\n": + # consume the newline, EOF here is an issue (middle of string) + self.inc(exception=UnexpectedEofError) - # Skip opening delim - if not self.inc(): - return self.parse_error(UnexpectedEofError) - - if self._current == delim: - self.inc() - - if self._current == delim: - multiline = True - if delim == "'": - str_type = StringType.MLL - else: - str_type = StringType.MLB - - if not self.inc(): - return self.parse_error(UnexpectedEofError) - else: - # Empty string - return String(str_type, "", "", Trivia()) - - self.mark() - if self._current == "\n": - # The first new line should be discarded - self.inc() - - previous = None - escaped = False + escaped = False # whether the previous key was ESCAPE while True: - if ( - previous != "\\" - or previous == "\\" - and (escaped or str_type.is_literal()) - ) and self._current == delim: - val = self.extract() + if delim.is_singleline() and self._current.is_nl(): + # single line cannot have actual newline characters + raise self.parse_error(InvalidCharInStringError, self._current) + elif not escaped and self._current == delim.unit: + # try to process current as a closing delim + original = self.extract() - if multiline: - stop = True - for _ in range(3): - if self._current != delim: + close = "" + if delim.is_multiline(): + # try consuming three delims as this would mean the end of + # the string + for last in [False, False, True]: + if self._current != delim.unit: # Not a triple quote, leave in result as-is. - stop = False - - # Adding back the quote character - value += delim + # Adding back the characters we already consumed + value += close + close = "" # clear the close break - self.inc() # TODO: Handle EOF + close += delim.unit - if not stop: + # consume this delim, EOF here is only an issue if this + # is not the third (last) delim character + self.inc(exception=UnexpectedEofError if not last else None) + + if not close: # if there is no close characters, keep parsing continue else: + # consume the closing delim, we do not care if EOF occurs as + # that would simply imply the end of self._src self.inc() - return String(str_type, value, val, Trivia()) + return String(delim, value, original, Trivia()) + elif delim.is_basic() and escaped: + # attempt to parse the current char as an escaped value, an exception + # is raised if this fails + value += self._parse_escaped_char(delim.is_multiline()) + + # no longer escaped + escaped = False + elif delim.is_basic() and self._current == "\\": + # the next char is being escaped + escaped = True + + # consume this char, EOF here is an issue (middle of string) + self.inc(exception=UnexpectedEofError) else: - if previous == "\\" and self._current.is_ws() and multiline: - while self._current.is_ws(): - previous = self._current + # this is either a literal string where we keep everything as is, + # or this is not a special escaped char in a basic string + value += self._current - self.inc() - continue - - if self._current == delim: - continue - - if previous == "\\": - if self._current == "\\" and not escaped: - if not str_type.is_literal(): - escaped = True - else: - value += self._current - - previous = self._current - - if not self.inc(): - raise self.parse_error(UnexpectedEofError) - - continue - elif self._current in _escaped and not escaped: - if not str_type.is_literal(): - value = value[:-1] - value += _escaped[self._current] - else: - value += self._current - elif self._current in {"u", "U"} and not escaped: - # Maybe unicode - u, ue = self._peek_unicode(self._current == "U") - if u is not None: - value = value[:-1] - value += u - self.inc_n(len(ue)) - else: - if not escaped and not str_type.is_literal(): - raise self.parse_error( - InvalidCharInStringError, (self._current,) - ) - - value += self._current - else: - if not escaped and not str_type.is_literal(): - raise self.parse_error( - InvalidCharInStringError, (self._current,) - ) - - value += self._current - - if self._current.is_ws() and multiline and not escaped: - continue - else: - value += self._current - - if escaped: - escaped = False - - previous = self._current - if not self.inc(): - raise self.parse_error(UnexpectedEofError) - - if previous == "\\" and self._current.is_ws() and multiline: - value = value[:-1] + # consume this char, EOF here is an issue (middle of string) + self.inc(exception=UnexpectedEofError) def _parse_table( self, parent_name=None @@ -812,8 +860,7 @@ class Parser: """ if self._current != "[": raise self.parse_error( - InternalParserError, - ("_parse_table() called on non-bracket character.",), + InternalParserError, "_parse_table() called on non-bracket character." ) indent = self.extract() @@ -940,7 +987,7 @@ class Parser: else: raise self.parse_error( InternalParserError, - ("_parse_item() returned None on a non-bracket character.",), + "_parse_item() returned None on a non-bracket character.", ) if isinstance(result, Null): @@ -965,32 +1012,27 @@ class Parser: Returns the name of the table about to be parsed, as well as whether it is part of an AoT. """ - # Save initial state - idx = self._save_idx() - marker = self._marker + # we always want to restore after exiting this scope + with self._state(save_marker=True, restore=True): + if self._current != "[": + raise self.parse_error( + InternalParserError, + "_peek_table() entered on non-bracket character", + ) - if self._current != "[": - raise self.parse_error( - InternalParserError, ("_peek_table() entered on non-bracket character",) - ) - - # AoT - self.inc() - is_aot = False - if self._current == "[": + # AoT self.inc() - is_aot = True + is_aot = False + if self._current == "[": + self.inc() + is_aot = True - self.mark() + self.mark() - while self._current != "]" and self.inc(): - table_name = self.extract() + while self._current != "]" and self.inc(): + table_name = self.extract() - # Restore initial state - self._restore_idx(*idx) - self._marker = marker - - return is_aot, table_name + return is_aot, table_name def _parse_aot(self, first, name_first): # type: (Table, str) -> AoT """ @@ -1017,57 +1059,53 @@ class Parser: n is the max number of characters that will be peeked. """ - idx = self._save_idx() - buf = "" - for _ in range(n): - if self._current not in " \t\n\r#,]}": - buf += self._current - self.inc() - continue + # we always want to restore after exiting this scope + with self._state(restore=True): + buf = "" + for _ in range(n): + if self._current not in " \t\n\r#,]}": + buf += self._current + self.inc() + continue - break + break + return buf - self._restore_idx(*idx) - - return buf - - def _peek_unicode(self, is_long): # type: () -> Tuple[bool, str] + def _peek_unicode( + self, is_long + ): # type: (bool) -> Tuple[Optional[str], Optional[str]] """ Peeks ahead non-intrusively by cloning then restoring the initial state of the parser. Returns the unicode value is it's a valid one else None. """ - # Save initial state - idx = self._save_idx() - marker = self._marker + # we always want to restore after exiting this scope + with self._state(save_marker=True, restore=True): + if self._current not in {"u", "U"}: + raise self.parse_error( + InternalParserError, "_peek_unicode() entered on non-unicode value" + ) - if self._current not in {"u", "U"}: - raise self.parse_error( - InternalParserError, ("_peek_unicode() entered on non-unicode value") - ) + self.inc() # Dropping prefix + self.mark() - # AoT - self.inc() # Dropping prefix - self.mark() + if is_long: + chars = 8 + else: + chars = 4 - if is_long: - chars = 8 - else: - chars = 4 + if not self.inc_n(chars): + value, extracted = None, None + else: + extracted = self.extract() - if not self.inc_n(chars): - value, extracted = None, None - else: - extracted = self.extract() + if extracted[0].lower() == "d" and extracted[1].strip("01234567"): + return None, None - try: - value = chr(int(extracted, 16)) - except ValueError: - value = None + try: + value = chr(int(extracted, 16)) + except ValueError: + value = None - # Restore initial state - self._restore_idx(*idx) - self._marker = marker - - return value, extracted + return value, extracted diff --git a/pipenv/vendor/tomlkit/source.py b/pipenv/vendor/tomlkit/source.py new file mode 100644 index 00000000..dcfdafd0 --- /dev/null +++ b/pipenv/vendor/tomlkit/source.py @@ -0,0 +1,193 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +import itertools + +from copy import copy + +from ._compat import PY2 +from ._compat import unicode +from .exceptions import UnexpectedEofError +from .exceptions import UnexpectedCharError +from .exceptions import ParseError +from .toml_char import TOMLChar + + +class _State: + def __init__( + self, source, save_marker=False, restore=False + ): # type: (_Source, Optional[bool], Optional[bool]) -> None + self._source = source + self._save_marker = save_marker + self.restore = restore + + def __enter__(self): # type: () -> None + # Entering this context manager - save the state + if PY2: + # Python 2.7 does not allow to directly copy + # an iterator, so we have to make tees of the original + # chars iterator. + self._source._chars, self._chars = itertools.tee(self._source._chars) + else: + self._chars = copy(self._source._chars) + self._idx = self._source._idx + self._current = self._source._current + self._marker = self._source._marker + + return self + + def __exit__(self, exception_type, exception_val, trace): + # Exiting this context manager - restore the prior state + if self.restore or exception_type: + self._source._chars = self._chars + self._source._idx = self._idx + self._source._current = self._current + if self._save_marker: + self._source._marker = self._marker + + # Restore exceptions are silently consumed, other exceptions need to + # propagate + return exception_type is None + + +class _StateHandler: + """ + State preserver for the Parser. + """ + + def __init__(self, source): # type: (Source) -> None + self._source = source + self._states = [] + + def __call__(self, *args, **kwargs): + return _State(self._source, *args, **kwargs) + + def __enter__(self): # type: () -> None + state = self() + self._states.append(state) + return state.__enter__() + + def __exit__(self, exception_type, exception_val, trace): + state = self._states.pop() + return state.__exit__(exception_type, exception_val, trace) + + +class Source(unicode): + EOF = TOMLChar("\0") + + def __init__(self, _): # type: (unicode) -> None + super(Source, self).__init__() + + # Collection of TOMLChars + self._chars = iter([(i, TOMLChar(c)) for i, c in enumerate(self)]) + + self._idx = 0 + self._marker = 0 + self._current = TOMLChar("") + + self._state = _StateHandler(self) + + self.inc() + + def reset(self): + # initialize both idx and current + self.inc() + + # reset marker + self.mark() + + @property + def state(self): # type: () -> _StateHandler + return self._state + + @property + def idx(self): # type: () -> int + return self._idx + + @property + def current(self): # type: () -> TOMLChar + return self._current + + @property + def marker(self): # type: () -> int + return self._marker + + def extract(self): # type: () -> unicode + """ + Extracts the value between marker and index + """ + return self[self._marker : self._idx] + + def inc(self, exception=None): # type: (Optional[ParseError.__class__]) -> bool + """ + Increments the parser if the end of the input has not been reached. + Returns whether or not it was able to advance. + """ + try: + self._idx, self._current = next(self._chars) + + return True + except StopIteration: + self._idx = len(self) + self._current = self.EOF + if exception: + raise self.parse_error(exception) + + return False + + def inc_n(self, n, exception=None): # type: (int, Exception) -> bool + """ + Increments the parser by n characters + if the end of the input has not been reached. + """ + for _ in range(n): + if not self.inc(exception=exception): + return False + + return True + + def consume(self, chars, min=0, max=-1): + """ + Consume chars until min/max is satisfied is valid. + """ + while self.current in chars and max != 0: + min -= 1 + max -= 1 + if not self.inc(): + break + + # failed to consume minimum number of characters + if min > 0: + self.parse_error(UnexpectedCharError) + + def end(self): # type: () -> bool + """ + Returns True if the parser has reached the end of the input. + """ + return self._current is self.EOF + + def mark(self): # type: () -> None + """ + Sets the marker to the index's current position + """ + self._marker = self._idx + + def parse_error( + self, exception=ParseError, *args + ): # type: (ParseError.__class__, ...) -> ParseError + """ + Creates a generic "parse error" at the current position. + """ + line, col = self._to_linecol() + + return exception(line, col, *args) + + def _to_linecol(self): # type: () -> Tuple[int, int] + cur = 0 + for i, line in enumerate(self.splitlines()): + if cur + len(line) + 1 > self.idx: + return (i + 1, self.idx - cur) + + cur += len(line) + 1 + + return len(self.splitlines()), 0 diff --git a/pipenv/vendor/tomlkit/toml_char.py b/pipenv/vendor/tomlkit/toml_char.py index 8a3bf9e1..02c55172 100644 --- a/pipenv/vendor/tomlkit/toml_char.py +++ b/pipenv/vendor/tomlkit/toml_char.py @@ -1,7 +1,13 @@ import string +from ._compat import PY2 from ._compat import unicode +if PY2: + from pipenv.vendor.backports.functools_lru_cache import lru_cache +else: + from functools import lru_cache + class TOMLChar(unicode): def __init__(self, c): @@ -10,38 +16,51 @@ class TOMLChar(unicode): if len(self) > 1: raise ValueError("A TOML character must be of length 1") + BARE = string.ascii_letters + string.digits + "-_" + KV = "= \t" + NUMBER = string.digits + "+-_.e" + SPACES = " \t" + NL = "\n\r" + WS = SPACES + NL + + @lru_cache(maxsize=None) def is_bare_key_char(self): # type: () -> bool """ Whether the character is a valid bare key name or not. """ - return self in string.ascii_letters + string.digits + "-" + "_" + return self in self.BARE + @lru_cache(maxsize=None) def is_kv_sep(self): # type: () -> bool """ Whether the character is a valid key/value separator ot not. """ - return self in "= \t" + return self in self.KV + @lru_cache(maxsize=None) def is_int_float_char(self): # type: () -> bool """ Whether the character if a valid integer or float value character or not. """ - return self in string.digits + "+" + "-" + "_" + "." + "e" + return self in self.NUMBER + @lru_cache(maxsize=None) def is_ws(self): # type: () -> bool """ Whether the character is a whitespace character or not. """ - return self in " \t\r\n" + return self in self.WS + @lru_cache(maxsize=None) def is_nl(self): # type: () -> bool """ Whether the character is a new line character or not. """ - return self in "\n\r" + return self in self.NL + @lru_cache(maxsize=None) def is_spaces(self): # type: () -> bool """ Whether the character is a space or not """ - return self in " \t" + return self in self.SPACES diff --git a/pipenv/vendor/urllib3/__init__.py b/pipenv/vendor/urllib3/__init__.py old mode 100755 new mode 100644 index 4bd533b5..75725167 --- a/pipenv/vendor/urllib3/__init__.py +++ b/pipenv/vendor/urllib3/__init__.py @@ -23,16 +23,11 @@ from .util.retry import Retry # Set default logging handler to avoid "No handler found" warnings. import logging -try: # Python 2.7+ - from logging import NullHandler -except ImportError: - class NullHandler(logging.Handler): - def emit(self, record): - pass +from logging import NullHandler __author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' __license__ = 'MIT' -__version__ = '1.23' +__version__ = '1.24' __all__ = ( 'HTTPConnectionPool', diff --git a/pipenv/vendor/urllib3/_collections.py b/pipenv/vendor/urllib3/_collections.py old mode 100755 new mode 100644 index 6e36b84e..34f23811 --- a/pipenv/vendor/urllib3/_collections.py +++ b/pipenv/vendor/urllib3/_collections.py @@ -14,10 +14,7 @@ except ImportError: # Platform-specific: No threads available pass -try: # Python 2.7+ - from collections import OrderedDict -except ImportError: - from .packages.ordered_dict import OrderedDict +from collections import OrderedDict from .exceptions import InvalidHeader from .packages.six import iterkeys, itervalues, PY3 diff --git a/pipenv/vendor/urllib3/connection.py b/pipenv/vendor/urllib3/connection.py old mode 100755 new mode 100644 index a03b573f..02b36654 --- a/pipenv/vendor/urllib3/connection.py +++ b/pipenv/vendor/urllib3/connection.py @@ -2,7 +2,6 @@ from __future__ import absolute_import import datetime import logging import os -import sys import socket from socket import error as SocketError, timeout as SocketTimeout import warnings @@ -78,9 +77,6 @@ class HTTPConnection(_HTTPConnection, object): - ``strict``: See the documentation on :class:`urllib3.connectionpool.HTTPConnectionPool` - ``source_address``: Set the source address for the current connection. - - .. note:: This is ignored for Python 2.6. It is only applied for 2.7 and 3.x - - ``socket_options``: Set specific options on the underlying socket. If not specified, then defaults are loaded from ``HTTPConnection.default_socket_options`` which includes disabling Nagle's algorithm (sets TCP_NODELAY to 1) unless the connection is behind a proxy. @@ -108,21 +104,13 @@ class HTTPConnection(_HTTPConnection, object): if six.PY3: # Python 3 kw.pop('strict', None) - # Pre-set source_address in case we have an older Python like 2.6. + # Pre-set source_address. self.source_address = kw.get('source_address') - if sys.version_info < (2, 7): # Python 2.6 - # _HTTPConnection on Python 2.6 will balk at this keyword arg, but - # not newer versions. We can still use it when creating a - # connection though, so we pop it *after* we have saved it as - # self.source_address. - kw.pop('source_address', None) - #: The socket options provided by the user. If no options are #: provided, we use the default options. self.socket_options = kw.pop('socket_options', self.default_socket_options) - # Superclass also sets self.source_address in Python 2.7+. _HTTPConnection.__init__(self, *args, **kw) @property @@ -183,10 +171,7 @@ class HTTPConnection(_HTTPConnection, object): def _prepare_conn(self, conn): self.sock = conn - # the _tunnel_host attribute was added in python 2.6.3 (via - # http://hg.python.org/cpython/rev/0f57b30a152f) so pythons 2.6(0-2) do - # not have them. - if getattr(self, '_tunnel_host', None): + if self._tunnel_host: # TODO: Fix tunnel so it doesn't depend on self.sock state. self._tunnel() # Mark this connection as not reusable @@ -217,13 +202,13 @@ class HTTPConnection(_HTTPConnection, object): self.endheaders() if body is not None: - stringish_types = six.string_types + (six.binary_type,) + stringish_types = six.string_types + (bytes,) if isinstance(body, stringish_types): body = (body,) for chunk in body: if not chunk: continue - if not isinstance(chunk, six.binary_type): + if not isinstance(chunk, bytes): chunk = chunk.encode('utf8') len_str = hex(len(chunk))[2:] self.send(len_str.encode('utf-8')) @@ -242,7 +227,7 @@ class HTTPSConnection(HTTPConnection): def __init__(self, host, port=None, key_file=None, cert_file=None, strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, - ssl_context=None, **kw): + ssl_context=None, server_hostname=None, **kw): HTTPConnection.__init__(self, host, port, strict=strict, timeout=timeout, **kw) @@ -250,6 +235,7 @@ class HTTPSConnection(HTTPConnection): self.key_file = key_file self.cert_file = cert_file self.ssl_context = ssl_context + self.server_hostname = server_hostname # Required property for Google AppEngine 1.9.0 which otherwise causes # HTTPS requests to go out as HTTP. (See Issue #356) @@ -270,6 +256,7 @@ class HTTPSConnection(HTTPConnection): keyfile=self.key_file, certfile=self.cert_file, ssl_context=self.ssl_context, + server_hostname=self.server_hostname ) @@ -312,12 +299,9 @@ class VerifiedHTTPSConnection(HTTPSConnection): def connect(self): # Add certificate verification conn = self._new_conn() - hostname = self.host - if getattr(self, '_tunnel_host', None): - # _tunnel_host was added in Python 2.6.3 - # (See: http://hg.python.org/cpython/rev/0f57b30a152f) + if self._tunnel_host: self.sock = conn # Calls self._set_hostport(), so self.host is # self._tunnel_host below. @@ -328,6 +312,10 @@ class VerifiedHTTPSConnection(HTTPSConnection): # Override the host with the one we're requesting data from. hostname = self._tunnel_host + server_hostname = hostname + if self.server_hostname is not None: + server_hostname = self.server_hostname + is_time_off = datetime.date.today() < RECENT_DATE if is_time_off: warnings.warn(( @@ -352,7 +340,7 @@ class VerifiedHTTPSConnection(HTTPSConnection): certfile=self.cert_file, ca_certs=self.ca_certs, ca_cert_dir=self.ca_cert_dir, - server_hostname=hostname, + server_hostname=server_hostname, ssl_context=context) if self.assert_fingerprint: @@ -373,7 +361,7 @@ class VerifiedHTTPSConnection(HTTPSConnection): 'for details.)'.format(hostname)), SubjectAltNameWarning ) - _match_hostname(cert, self.assert_hostname or hostname) + _match_hostname(cert, self.assert_hostname or server_hostname) self.is_verified = ( context.verify_mode == ssl.CERT_REQUIRED or diff --git a/pipenv/vendor/urllib3/connectionpool.py b/pipenv/vendor/urllib3/connectionpool.py old mode 100755 new mode 100644 index 8fcb0bce..f7a8f193 --- a/pipenv/vendor/urllib3/connectionpool.py +++ b/pipenv/vendor/urllib3/connectionpool.py @@ -89,7 +89,7 @@ class ConnectionPool(object): # This is taken from http://hg.python.org/cpython/file/7aaba721ebc0/Lib/socket.py#l252 -_blocking_errnos = set([errno.EAGAIN, errno.EWOULDBLOCK]) +_blocking_errnos = {errno.EAGAIN, errno.EWOULDBLOCK} class HTTPConnectionPool(ConnectionPool, RequestMethods): @@ -313,7 +313,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # Catch possible read timeouts thrown as SSL errors. If not the # case, rethrow the original. We need to do this because of: # http://bugs.python.org/issue10272 - if 'timed out' in str(err) or 'did not complete (read)' in str(err): # Python 2.6 + if 'timed out' in str(err) or 'did not complete (read)' in str(err): # Python < 2.7.4 raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value) def _make_request(self, conn, method, url, timeout=_Default, chunked=False, @@ -375,7 +375,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): try: try: # Python 2.7, use buffering of HTTP responses httplib_response = conn.getresponse(buffering=True) - except TypeError: # Python 2.6 and older, Python 3 + except TypeError: # Python 3 try: httplib_response = conn.getresponse() except Exception as e: @@ -801,17 +801,7 @@ class HTTPSConnectionPool(HTTPConnectionPool): Establish tunnel connection early, because otherwise httplib would improperly set Host: header to proxy's IP:port. """ - # Python 2.7+ - try: - set_tunnel = conn.set_tunnel - except AttributeError: # Platform-specific: Python 2.6 - set_tunnel = conn._set_tunnel - - if sys.version_info <= (2, 6, 4) and not self.proxy_headers: # Python 2.6.4 and older - set_tunnel(self._proxy_host, self.port) - else: - set_tunnel(self._proxy_host, self.port, self.proxy_headers) - + conn.set_tunnel(self._proxy_host, self.port, self.proxy_headers) conn.connect() def _new_conn(self): diff --git a/pipenv/vendor/urllib3/contrib/__init__.py b/pipenv/vendor/urllib3/contrib/__init__.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/contrib/_appengine_environ.py b/pipenv/vendor/urllib3/contrib/_appengine_environ.py new file mode 100644 index 00000000..f3e00942 --- /dev/null +++ b/pipenv/vendor/urllib3/contrib/_appengine_environ.py @@ -0,0 +1,30 @@ +""" +This module provides means to detect the App Engine environment. +""" + +import os + + +def is_appengine(): + return (is_local_appengine() or + is_prod_appengine() or + is_prod_appengine_mvms()) + + +def is_appengine_sandbox(): + return is_appengine() and not is_prod_appengine_mvms() + + +def is_local_appengine(): + return ('APPENGINE_RUNTIME' in os.environ and + 'Development/' in os.environ['SERVER_SOFTWARE']) + + +def is_prod_appengine(): + return ('APPENGINE_RUNTIME' in os.environ and + 'Google App Engine/' in os.environ['SERVER_SOFTWARE'] and + not is_prod_appengine_mvms()) + + +def is_prod_appengine_mvms(): + return os.environ.get('GAE_VM', False) == 'true' diff --git a/pipenv/vendor/urllib3/contrib/_securetransport/__init__.py b/pipenv/vendor/urllib3/contrib/_securetransport/__init__.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/contrib/_securetransport/bindings.py b/pipenv/vendor/urllib3/contrib/_securetransport/bindings.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/contrib/_securetransport/low_level.py b/pipenv/vendor/urllib3/contrib/_securetransport/low_level.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/contrib/appengine.py b/pipenv/vendor/urllib3/contrib/appengine.py old mode 100755 new mode 100644 index 66922e06..2952f114 --- a/pipenv/vendor/urllib3/contrib/appengine.py +++ b/pipenv/vendor/urllib3/contrib/appengine.py @@ -39,8 +39,8 @@ urllib3 on Google App Engine: """ from __future__ import absolute_import +import io import logging -import os import warnings from ..packages.six.moves.urllib.parse import urljoin @@ -53,11 +53,11 @@ from ..exceptions import ( SSLError ) -from ..packages.six import BytesIO from ..request import RequestMethods from ..response import HTTPResponse from ..util.timeout import Timeout from ..util.retry import Retry +from . import _appengine_environ try: from google.appengine.api import urlfetch @@ -239,7 +239,7 @@ class AppEngineManager(RequestMethods): original_response = HTTPResponse( # In order for decoding to work, we must present the content as # a file-like object. - body=BytesIO(urlfetch_resp.content), + body=io.BytesIO(urlfetch_resp.content), msg=urlfetch_resp.header_msg, headers=urlfetch_resp.headers, status=urlfetch_resp.status_code, @@ -247,7 +247,7 @@ class AppEngineManager(RequestMethods): ) return HTTPResponse( - body=BytesIO(urlfetch_resp.content), + body=io.BytesIO(urlfetch_resp.content), headers=urlfetch_resp.headers, status=urlfetch_resp.status_code, original_response=original_response, @@ -280,26 +280,10 @@ class AppEngineManager(RequestMethods): return retries -def is_appengine(): - return (is_local_appengine() or - is_prod_appengine() or - is_prod_appengine_mvms()) +# Alias methods from _appengine_environ to maintain public API interface. - -def is_appengine_sandbox(): - return is_appengine() and not is_prod_appengine_mvms() - - -def is_local_appengine(): - return ('APPENGINE_RUNTIME' in os.environ and - 'Development/' in os.environ['SERVER_SOFTWARE']) - - -def is_prod_appengine(): - return ('APPENGINE_RUNTIME' in os.environ and - 'Google App Engine/' in os.environ['SERVER_SOFTWARE'] and - not is_prod_appengine_mvms()) - - -def is_prod_appengine_mvms(): - return os.environ.get('GAE_VM', False) == 'true' +is_appengine = _appengine_environ.is_appengine +is_appengine_sandbox = _appengine_environ.is_appengine_sandbox +is_local_appengine = _appengine_environ.is_local_appengine +is_prod_appengine = _appengine_environ.is_prod_appengine +is_prod_appengine_mvms = _appengine_environ.is_prod_appengine_mvms diff --git a/pipenv/vendor/urllib3/contrib/ntlmpool.py b/pipenv/vendor/urllib3/contrib/ntlmpool.py old mode 100755 new mode 100644 index 642e99ed..8ea127c5 --- a/pipenv/vendor/urllib3/contrib/ntlmpool.py +++ b/pipenv/vendor/urllib3/contrib/ntlmpool.py @@ -43,8 +43,7 @@ class NTLMConnectionPool(HTTPSConnectionPool): log.debug('Starting NTLM HTTPS connection no. %d: https://%s%s', self.num_connections, self.host, self.authurl) - headers = {} - headers['Connection'] = 'Keep-Alive' + headers = {'Connection': 'Keep-Alive'} req_header = 'Authorization' resp_header = 'www-authenticate' diff --git a/pipenv/vendor/urllib3/contrib/pyopenssl.py b/pipenv/vendor/urllib3/contrib/pyopenssl.py old mode 100755 new mode 100644 index 4d4b1aff..7c0e9465 --- a/pipenv/vendor/urllib3/contrib/pyopenssl.py +++ b/pipenv/vendor/urllib3/contrib/pyopenssl.py @@ -163,6 +163,9 @@ def _dnsname_to_stdlib(name): from ASCII bytes. We need to idna-encode that string to get it back, and then on Python 3 we also need to convert to unicode via UTF-8 (the stdlib uses PyUnicode_FromStringAndSize on it, which decodes via UTF-8). + + If the name cannot be idna-encoded then we return None signalling that + the name given should be skipped. """ def idna_encode(name): """ @@ -172,14 +175,19 @@ def _dnsname_to_stdlib(name): """ import idna - for prefix in [u'*.', u'.']: - if name.startswith(prefix): - name = name[len(prefix):] - return prefix.encode('ascii') + idna.encode(name) - return idna.encode(name) + try: + for prefix in [u'*.', u'.']: + if name.startswith(prefix): + name = name[len(prefix):] + return prefix.encode('ascii') + idna.encode(name) + return idna.encode(name) + except idna.core.IDNAError: + return None name = idna_encode(name) - if sys.version_info >= (3, 0): + if name is None: + return None + elif sys.version_info >= (3, 0): name = name.decode('utf-8') return name @@ -223,9 +231,10 @@ def get_subj_alt_name(peer_cert): # Sadly the DNS names need to be idna encoded and then, on Python 3, UTF-8 # decoded. This is pretty frustrating, but that's what the standard library # does with certificates, and so we need to attempt to do the same. + # We also want to skip over names which cannot be idna encoded. names = [ - ('DNS', _dnsname_to_stdlib(name)) - for name in ext.get_values_for_type(x509.DNSName) + ('DNS', name) for name in map(_dnsname_to_stdlib, ext.get_values_for_type(x509.DNSName)) + if name is not None ] names.extend( ('IP Address', str(name)) diff --git a/pipenv/vendor/urllib3/contrib/securetransport.py b/pipenv/vendor/urllib3/contrib/securetransport.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/contrib/socks.py b/pipenv/vendor/urllib3/contrib/socks.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/exceptions.py b/pipenv/vendor/urllib3/exceptions.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/fields.py b/pipenv/vendor/urllib3/fields.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/filepost.py b/pipenv/vendor/urllib3/filepost.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/packages/__init__.py b/pipenv/vendor/urllib3/packages/__init__.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/packages/backports/__init__.py b/pipenv/vendor/urllib3/packages/backports/__init__.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/packages/backports/makefile.py b/pipenv/vendor/urllib3/packages/backports/makefile.py old mode 100755 new mode 100644 index 75b80dcf..740db377 --- a/pipenv/vendor/urllib3/packages/backports/makefile.py +++ b/pipenv/vendor/urllib3/packages/backports/makefile.py @@ -16,7 +16,7 @@ def backport_makefile(self, mode="r", buffering=None, encoding=None, """ Backport of ``socket.makefile`` from Python 3.5. """ - if not set(mode) <= set(["r", "w", "b"]): + if not set(mode) <= {"r", "w", "b"}: raise ValueError( "invalid mode %r (only r, w, b allowed)" % (mode,) ) diff --git a/pipenv/vendor/urllib3/packages/ordered_dict.py b/pipenv/vendor/urllib3/packages/ordered_dict.py deleted file mode 100755 index 4479363c..00000000 --- a/pipenv/vendor/urllib3/packages/ordered_dict.py +++ /dev/null @@ -1,259 +0,0 @@ -# Backport of OrderedDict() class that runs on Python 2.4, 2.5, 2.6, 2.7 and pypy. -# Passes Python2.7's test suite and incorporates all the latest updates. -# Copyright 2009 Raymond Hettinger, released under the MIT License. -# http://code.activestate.com/recipes/576693/ -try: - from thread import get_ident as _get_ident -except ImportError: - from dummy_thread import get_ident as _get_ident - -try: - from _abcoll import KeysView, ValuesView, ItemsView -except ImportError: - pass - - -class OrderedDict(dict): - 'Dictionary that remembers insertion order' - # An inherited dict maps keys to values. - # The inherited dict provides __getitem__, __len__, __contains__, and get. - # The remaining methods are order-aware. - # Big-O running times for all methods are the same as for regular dictionaries. - - # The internal self.__map dictionary maps keys to links in a doubly linked list. - # The circular doubly linked list starts and ends with a sentinel element. - # The sentinel element never gets deleted (this simplifies the algorithm). - # Each link is stored as a list of length three: [PREV, NEXT, KEY]. - - def __init__(self, *args, **kwds): - '''Initialize an ordered dictionary. Signature is the same as for - regular dictionaries, but keyword arguments are not recommended - because their insertion order is arbitrary. - - ''' - if len(args) > 1: - raise TypeError('expected at most 1 arguments, got %d' % len(args)) - try: - self.__root - except AttributeError: - self.__root = root = [] # sentinel node - root[:] = [root, root, None] - self.__map = {} - self.__update(*args, **kwds) - - def __setitem__(self, key, value, dict_setitem=dict.__setitem__): - 'od.__setitem__(i, y) <==> od[i]=y' - # Setting a new item creates a new link which goes at the end of the linked - # list, and the inherited dictionary is updated with the new key/value pair. - if key not in self: - root = self.__root - last = root[0] - last[1] = root[0] = self.__map[key] = [last, root, key] - dict_setitem(self, key, value) - - def __delitem__(self, key, dict_delitem=dict.__delitem__): - 'od.__delitem__(y) <==> del od[y]' - # Deleting an existing item uses self.__map to find the link which is - # then removed by updating the links in the predecessor and successor nodes. - dict_delitem(self, key) - link_prev, link_next, key = self.__map.pop(key) - link_prev[1] = link_next - link_next[0] = link_prev - - def __iter__(self): - 'od.__iter__() <==> iter(od)' - root = self.__root - curr = root[1] - while curr is not root: - yield curr[2] - curr = curr[1] - - def __reversed__(self): - 'od.__reversed__() <==> reversed(od)' - root = self.__root - curr = root[0] - while curr is not root: - yield curr[2] - curr = curr[0] - - def clear(self): - 'od.clear() -> None. Remove all items from od.' - try: - for node in self.__map.itervalues(): - del node[:] - root = self.__root - root[:] = [root, root, None] - self.__map.clear() - except AttributeError: - pass - dict.clear(self) - - def popitem(self, last=True): - '''od.popitem() -> (k, v), return and remove a (key, value) pair. - Pairs are returned in LIFO order if last is true or FIFO order if false. - - ''' - if not self: - raise KeyError('dictionary is empty') - root = self.__root - if last: - link = root[0] - link_prev = link[0] - link_prev[1] = root - root[0] = link_prev - else: - link = root[1] - link_next = link[1] - root[1] = link_next - link_next[0] = root - key = link[2] - del self.__map[key] - value = dict.pop(self, key) - return key, value - - # -- the following methods do not depend on the internal structure -- - - def keys(self): - 'od.keys() -> list of keys in od' - return list(self) - - def values(self): - 'od.values() -> list of values in od' - return [self[key] for key in self] - - def items(self): - 'od.items() -> list of (key, value) pairs in od' - return [(key, self[key]) for key in self] - - def iterkeys(self): - 'od.iterkeys() -> an iterator over the keys in od' - return iter(self) - - def itervalues(self): - 'od.itervalues -> an iterator over the values in od' - for k in self: - yield self[k] - - def iteritems(self): - 'od.iteritems -> an iterator over the (key, value) items in od' - for k in self: - yield (k, self[k]) - - def update(*args, **kwds): - '''od.update(E, **F) -> None. Update od from dict/iterable E and F. - - If E is a dict instance, does: for k in E: od[k] = E[k] - If E has a .keys() method, does: for k in E.keys(): od[k] = E[k] - Or if E is an iterable of items, does: for k, v in E: od[k] = v - In either case, this is followed by: for k, v in F.items(): od[k] = v - - ''' - if len(args) > 2: - raise TypeError('update() takes at most 2 positional ' - 'arguments (%d given)' % (len(args),)) - elif not args: - raise TypeError('update() takes at least 1 argument (0 given)') - self = args[0] - # Make progressively weaker assumptions about "other" - other = () - if len(args) == 2: - other = args[1] - if isinstance(other, dict): - for key in other: - self[key] = other[key] - elif hasattr(other, 'keys'): - for key in other.keys(): - self[key] = other[key] - else: - for key, value in other: - self[key] = value - for key, value in kwds.items(): - self[key] = value - - __update = update # let subclasses override update without breaking __init__ - - __marker = object() - - def pop(self, key, default=__marker): - '''od.pop(k[,d]) -> v, remove specified key and return the corresponding value. - If key is not found, d is returned if given, otherwise KeyError is raised. - - ''' - if key in self: - result = self[key] - del self[key] - return result - if default is self.__marker: - raise KeyError(key) - return default - - def setdefault(self, key, default=None): - 'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od' - if key in self: - return self[key] - self[key] = default - return default - - def __repr__(self, _repr_running={}): - 'od.__repr__() <==> repr(od)' - call_key = id(self), _get_ident() - if call_key in _repr_running: - return '...' - _repr_running[call_key] = 1 - try: - if not self: - return '%s()' % (self.__class__.__name__,) - return '%s(%r)' % (self.__class__.__name__, self.items()) - finally: - del _repr_running[call_key] - - def __reduce__(self): - 'Return state information for pickling' - items = [[k, self[k]] for k in self] - inst_dict = vars(self).copy() - for k in vars(OrderedDict()): - inst_dict.pop(k, None) - if inst_dict: - return (self.__class__, (items,), inst_dict) - return self.__class__, (items,) - - def copy(self): - 'od.copy() -> a shallow copy of od' - return self.__class__(self) - - @classmethod - def fromkeys(cls, iterable, value=None): - '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S - and values equal to v (which defaults to None). - - ''' - d = cls() - for key in iterable: - d[key] = value - return d - - def __eq__(self, other): - '''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive - while comparison to a regular mapping is order-insensitive. - - ''' - if isinstance(other, OrderedDict): - return len(self)==len(other) and self.items() == other.items() - return dict.__eq__(self, other) - - def __ne__(self, other): - return not self == other - - # -- the following methods are only used in Python 2.7 -- - - def viewkeys(self): - "od.viewkeys() -> a set-like object providing a view on od's keys" - return KeysView(self) - - def viewvalues(self): - "od.viewvalues() -> an object providing a view on od's values" - return ValuesView(self) - - def viewitems(self): - "od.viewitems() -> a set-like object providing a view on od's items" - return ItemsView(self) diff --git a/pipenv/vendor/urllib3/packages/six.py b/pipenv/vendor/urllib3/packages/six.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/packages/ssl_match_hostname/__init__.py b/pipenv/vendor/urllib3/packages/ssl_match_hostname/__init__.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/packages/ssl_match_hostname/_implementation.py b/pipenv/vendor/urllib3/packages/ssl_match_hostname/_implementation.py old mode 100755 new mode 100644 index 1fd42f38..d6e66c01 --- a/pipenv/vendor/urllib3/packages/ssl_match_hostname/_implementation.py +++ b/pipenv/vendor/urllib3/packages/ssl_match_hostname/_implementation.py @@ -9,8 +9,7 @@ import sys # ipaddress has been backported to 2.6+ in pypi. If it is installed on the # system, use it to handle IPAddress ServerAltnames (this was added in # python-3.5) otherwise only do DNS matching. This allows -# backports.ssl_match_hostname to continue to be used all the way back to -# python-2.4. +# backports.ssl_match_hostname to continue to be used in Python 2.7. try: import ipaddress except ImportError: diff --git a/pipenv/vendor/urllib3/poolmanager.py b/pipenv/vendor/urllib3/poolmanager.py old mode 100755 new mode 100644 index 506a3c9b..fe5491cf --- a/pipenv/vendor/urllib3/poolmanager.py +++ b/pipenv/vendor/urllib3/poolmanager.py @@ -47,6 +47,7 @@ _key_fields = ( 'key__socks_options', # dict 'key_assert_hostname', # bool or string 'key_assert_fingerprint', # str + 'key_server_hostname', #str ) #: The namedtuple class used to construct keys for the connection pool. diff --git a/pipenv/vendor/urllib3/request.py b/pipenv/vendor/urllib3/request.py old mode 100755 new mode 100644 index 1be33341..8f2f44bb --- a/pipenv/vendor/urllib3/request.py +++ b/pipenv/vendor/urllib3/request.py @@ -36,7 +36,7 @@ class RequestMethods(object): explicitly. """ - _encode_url_methods = set(['DELETE', 'GET', 'HEAD', 'OPTIONS']) + _encode_url_methods = {'DELETE', 'GET', 'HEAD', 'OPTIONS'} def __init__(self, headers=None): self.headers = headers or {} diff --git a/pipenv/vendor/urllib3/response.py b/pipenv/vendor/urllib3/response.py old mode 100755 new mode 100644 index 9873cb94..f0cfbb54 --- a/pipenv/vendor/urllib3/response.py +++ b/pipenv/vendor/urllib3/response.py @@ -11,7 +11,7 @@ from .exceptions import ( BodyNotHttplibCompatible, ProtocolError, DecodeError, ReadTimeoutError, ResponseNotChunked, IncompleteRead, InvalidHeader ) -from .packages.six import string_types as basestring, binary_type, PY3 +from .packages.six import string_types as basestring, PY3 from .packages.six.moves import http_client as httplib from .connection import HTTPException, BaseSSLError from .util.response import is_fp_closed, is_response_to_head @@ -23,7 +23,7 @@ class DeflateDecoder(object): def __init__(self): self._first_try = True - self._data = binary_type() + self._data = b'' self._obj = zlib.decompressobj() def __getattr__(self, name): @@ -69,7 +69,7 @@ class GzipDecoder(object): return getattr(self._obj, name) def decompress(self, data): - ret = binary_type() + ret = b'' if self._state == GzipDecoderState.SWALLOW_DATA or not data: return ret while True: @@ -90,7 +90,31 @@ class GzipDecoder(object): self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS) +class MultiDecoder(object): + """ + From RFC7231: + If one or more encodings have been applied to a representation, the + sender that applied the encodings MUST generate a Content-Encoding + header field that lists the content codings in the order in which + they were applied. + """ + + def __init__(self, modes): + self._decoders = [_get_decoder(m.strip()) for m in modes.split(',')] + + def flush(self): + return self._decoders[0].flush() + + def decompress(self, data): + for d in reversed(self._decoders): + data = d.decompress(data) + return data + + def _get_decoder(mode): + if ',' in mode: + return MultiDecoder(mode) + if mode == 'gzip': return GzipDecoder() @@ -159,7 +183,7 @@ class HTTPResponse(io.IOBase): self.msg = msg self._request_url = request_url - if body and isinstance(body, (basestring, binary_type)): + if body and isinstance(body, (basestring, bytes)): self._body = body self._pool = pool @@ -283,8 +307,13 @@ class HTTPResponse(io.IOBase): # Note: content-encoding value should be case-insensitive, per RFC 7230 # Section 3.2 content_encoding = self.headers.get('content-encoding', '').lower() - if self._decoder is None and content_encoding in self.CONTENT_DECODERS: - self._decoder = _get_decoder(content_encoding) + if self._decoder is None: + if content_encoding in self.CONTENT_DECODERS: + self._decoder = _get_decoder(content_encoding) + elif ',' in content_encoding: + encodings = [e.strip() for e in content_encoding.split(',') if e.strip() in self.CONTENT_DECODERS] + if len(encodings): + self._decoder = _get_decoder(content_encoding) def _decode(self, data, decode_content, flush_decoder): """ diff --git a/pipenv/vendor/urllib3/util/__init__.py b/pipenv/vendor/urllib3/util/__init__.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/util/connection.py b/pipenv/vendor/urllib3/util/connection.py old mode 100755 new mode 100644 index 5cf488f4..5ad70b2f --- a/pipenv/vendor/urllib3/util/connection.py +++ b/pipenv/vendor/urllib3/util/connection.py @@ -1,6 +1,7 @@ from __future__ import absolute_import import socket from .wait import NoWayToWaitForSocketError, wait_for_read +from ..contrib import _appengine_environ def is_connection_dropped(conn): # Platform-specific @@ -105,6 +106,13 @@ def _has_ipv6(host): sock = None has_ipv6 = False + # App Engine doesn't support IPV6 sockets and actually has a quota on the + # number of sockets that can be used, so just early out here instead of + # creating a socket needlessly. + # See https://github.com/urllib3/urllib3/issues/1446 + if _appengine_environ.is_appengine_sandbox(): + return False + if socket.has_ipv6: # has_ipv6 returns true if cPython was compiled with IPv6 support. # It does not tell us if the system has IPv6 support enabled. To diff --git a/pipenv/vendor/urllib3/util/queue.py b/pipenv/vendor/urllib3/util/queue.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/util/request.py b/pipenv/vendor/urllib3/util/request.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/util/response.py b/pipenv/vendor/urllib3/util/response.py old mode 100755 new mode 100644 index 67cf730a..3d548648 --- a/pipenv/vendor/urllib3/util/response.py +++ b/pipenv/vendor/urllib3/util/response.py @@ -59,8 +59,14 @@ def assert_header_parsing(headers): get_payload = getattr(headers, 'get_payload', None) unparsed_data = None - if get_payload: # Platform-specific: Python 3. - unparsed_data = get_payload() + if get_payload: + # get_payload is actually email.message.Message.get_payload; + # we're only interested in the result if it's not a multipart message + if not headers.is_multipart(): + payload = get_payload() + + if isinstance(payload, (bytes, str)): + unparsed_data = payload if defects or unparsed_data: raise HeaderParsingError(defects=defects, unparsed_data=unparsed_data) diff --git a/pipenv/vendor/urllib3/util/retry.py b/pipenv/vendor/urllib3/util/retry.py old mode 100755 new mode 100644 index 7ad3dc66..e7d0abd6 --- a/pipenv/vendor/urllib3/util/retry.py +++ b/pipenv/vendor/urllib3/util/retry.py @@ -115,7 +115,7 @@ class Retry(object): (most errors are resolved immediately by a second try without a delay). urllib3 will sleep for:: - {backoff factor} * (2 ^ ({number of total retries} - 1)) + {backoff factor} * (2 ** ({number of total retries} - 1)) seconds. If the backoff_factor is 0.1, then :func:`.sleep` will sleep for [0.0s, 0.2s, 0.4s, ...] between retries. It will never be longer diff --git a/pipenv/vendor/urllib3/util/ssl_.py b/pipenv/vendor/urllib3/util/ssl_.py old mode 100755 new mode 100644 index 2893752a..24ee26d6 --- a/pipenv/vendor/urllib3/util/ssl_.py +++ b/pipenv/vendor/urllib3/util/ssl_.py @@ -56,9 +56,8 @@ except ImportError: OP_NO_COMPRESSION = 0x20000 -# Python 2.7 and earlier didn't have inet_pton on non-Linux -# so we fallback on inet_aton in those cases. This means that -# we can only detect IPv4 addresses in this case. +# Python 2.7 doesn't have inet_pton on non-Linux so we fallback on inet_aton in +# those cases. This means that we can only detect IPv4 addresses in this case. if hasattr(socket, 'inet_pton'): inet_pton = socket.inet_pton else: @@ -67,7 +66,7 @@ else: import ipaddress def inet_pton(_, host): - if isinstance(host, six.binary_type): + if isinstance(host, bytes): host = host.decode('ascii') return ipaddress.ip_address(host) @@ -115,10 +114,7 @@ try: except ImportError: import sys - class SSLContext(object): # Platform-specific: Python 2 & 3.1 - supports_set_ciphers = ((2, 7) <= sys.version_info < (3,) or - (3, 2) <= sys.version_info) - + class SSLContext(object): # Platform-specific: Python 2 def __init__(self, protocol_version): self.protocol = protocol_version # Use default values from a real SSLContext @@ -141,12 +137,6 @@ except ImportError: raise SSLError("CA directories not supported in older Pythons") def set_ciphers(self, cipher_suite): - if not self.supports_set_ciphers: - raise TypeError( - 'Your version of Python does not support setting ' - 'a custom cipher suite. Please upgrade to Python ' - '2.7, 3.2, or later if you need this functionality.' - ) self.ciphers = cipher_suite def wrap_socket(self, socket, server_hostname=None, server_side=False): @@ -167,10 +157,7 @@ except ImportError: 'ssl_version': self.protocol, 'server_side': server_side, } - if self.supports_set_ciphers: # Platform-specific: Python 2.7+ - return wrap_socket(socket, ciphers=self.ciphers, **kwargs) - else: # Platform-specific: Python 2.6 - return wrap_socket(socket, **kwargs) + return wrap_socket(socket, ciphers=self.ciphers, **kwargs) def assert_fingerprint(cert, fingerprint): @@ -291,9 +278,6 @@ def create_urllib3_context(ssl_version=None, cert_reqs=None, context.options |= options - if getattr(context, 'supports_set_ciphers', True): # Platform-specific: Python 2.6 - context.set_ciphers(ciphers or DEFAULT_CIPHERS) - context.verify_mode = cert_reqs if getattr(context, 'check_hostname', None) is not None: # Platform-specific: Python 3.2 # We do our own verification, including fingerprints and alternative @@ -316,8 +300,7 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, A pre-made :class:`SSLContext` object. If none is provided, one will be created using :func:`create_urllib3_context`. :param ciphers: - A string of ciphers we wish the client to support. This is not - supported on Python 2.6 as the ssl module does not support it. + A string of ciphers we wish the client to support. :param ca_cert_dir: A directory containing CA certificates in multiple separate files, as supported by OpenSSL's -CApath flag or the capath argument to @@ -334,7 +317,7 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, if ca_certs or ca_cert_dir: try: context.load_verify_locations(ca_certs, ca_cert_dir) - except IOError as e: # Platform-specific: Python 2.6, 2.7, 3.2 + except IOError as e: # Platform-specific: Python 2.7 raise SSLError(e) # Py33 raises FileNotFoundError which subclasses OSError # These are not equivalent unless we check the errno attribute @@ -378,7 +361,7 @@ def is_ipaddress(hostname): :param str hostname: Hostname to examine. :return: True if the hostname is an IP address, False otherwise. """ - if six.PY3 and isinstance(hostname, six.binary_type): + if six.PY3 and isinstance(hostname, bytes): # IDN A-label bytes are ASCII compatible. hostname = hostname.decode('ascii') diff --git a/pipenv/vendor/urllib3/util/timeout.py b/pipenv/vendor/urllib3/util/timeout.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/util/url.py b/pipenv/vendor/urllib3/util/url.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/urllib3/util/wait.py b/pipenv/vendor/urllib3/util/wait.py old mode 100755 new mode 100644 index fa686eff..4db71baf --- a/pipenv/vendor/urllib3/util/wait.py +++ b/pipenv/vendor/urllib3/util/wait.py @@ -43,9 +43,6 @@ if sys.version_info >= (3, 5): else: # Old and broken Pythons. def _retry_on_intr(fn, timeout): - if timeout is not None and timeout <= 0: - return fn(timeout) - if timeout is None: deadline = float("inf") else: @@ -117,7 +114,7 @@ def _have_working_poll(): # from libraries like eventlet/greenlet. try: poll_obj = select.poll() - poll_obj.poll(0) + _retry_on_intr(poll_obj.poll, 0) except (AttributeError, OSError): return False else: diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index 7a574b9f..ff7226b2 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -3,7 +3,7 @@ backports.shutil_get_terminal_size==1.0.0 backports.weakref==1.0.post1 blindspin==2.0.1 click==7.0 -click-completion==0.4.1 +click-completion==0.5.0 click-didyoumean==0.0.3 colorama==0.3.9 delegator.py==0.1.1 @@ -14,37 +14,39 @@ first==2.0.1 iso8601==0.1.12 jinja2==2.10 markupsafe==1.0 -parse==1.8.4 +parse==1.9.0 pathlib2==2.3.2 scandir==1.9 pipdeptree==0.13.0 pipreqs==0.4.9 docopt==0.6.2 yarg==0.1.9 -pythonfinder==1.1.2 -requests==2.19.1 +pythonfinder==1.1.9.post1 +requests==2.20.1 chardet==3.0.4 idna==2.7 - urllib3==1.23 - certifi==2018.8.24 -requirementslib==1.1.9 + urllib3==1.24 + certifi==2018.10.15 +requirementslib==1.3.1.post1 attrs==18.2.0 distlib==0.2.8 packaging==18.0 pyparsing==2.2.2 - pytoml==0.1.19 plette==0.2.2 - tomlkit==0.4.4 -shellingham==1.2.6 + tomlkit==0.5.2 +shellingham==1.2.7 six==1.11.0 semver==2.8.1 shutilwhich==1.1.0 toml==0.10.0 cached-property==1.4.3 -vistir==0.1.7 -pip-shims==0.3.1 +vistir==0.2.4 +pip-shims==0.3.2 ptyprocess==0.6.0 enum34==1.1.6 yaspin==0.14.0 cerberus==1.2 git+https://github.com/sarugaku/passa.git@master#egg=passa +cursor==1.2.0 +resolvelib==0.2.2 +backports.functools_lru_cache==1.5 diff --git a/pipenv/vendor/vendor_pip.txt b/pipenv/vendor/vendor_pip.txt index b9854e9a..9389dd94 100644 --- a/pipenv/vendor/vendor_pip.txt +++ b/pipenv/vendor/vendor_pip.txt @@ -9,14 +9,15 @@ msgpack-python==0.5.6 lockfile==0.12.2 progress==1.4 ipaddress==1.0.22 # Only needed on 2.6 and 2.7 -packaging==17.1 -pyparsing==2.2.0 -pytoml==0.1.16 +packaging==18.0 +pep517==0.2 +pyparsing==2.2.1 +pytoml==0.1.19 retrying==1.3.3 requests==2.19.1 chardet==3.0.4 idna==2.7 urllib3==1.23 - certifi==2018.4.16 -setuptools==39.2.0 + certifi==2018.8.24 +setuptools==40.4.3 webencodings==0.5.1 diff --git a/pipenv/vendor/vistir/__init__.py b/pipenv/vendor/vistir/__init__.py index fe6f884c..809c973c 100644 --- a/pipenv/vendor/vistir/__init__.py +++ b/pipenv/vendor/vistir/__init__.py @@ -1,19 +1,37 @@ # -*- coding=utf-8 -*- from __future__ import absolute_import, unicode_literals -from .compat import NamedTemporaryFile, TemporaryDirectory, partialmethod +from .compat import ( + NamedTemporaryFile, + TemporaryDirectory, + partialmethod, + to_native_string, +) from .contextmanagers import ( atomic_open_for_write, cd, open_file, temp_environ, temp_path, + spinner, ) -from .misc import load_path, partialclass, run, shell_escape -from .path import mkdir_p, rmtree +from .misc import ( + load_path, + partialclass, + run, + shell_escape, + decode_for_output, + to_text, + to_bytes, + take, + chunked, + divide, +) +from .path import mkdir_p, rmtree, create_tracked_tempdir, create_tracked_tempfile +from .spin import VistirSpinner, create_spinner -__version__ = '0.1.7' +__version__ = '0.2.4' __all__ = [ @@ -31,4 +49,16 @@ __all__ = [ "TemporaryDirectory", "NamedTemporaryFile", "partialmethod", + "spinner", + "VistirSpinner", + "create_spinner", + "create_tracked_tempdir", + "create_tracked_tempfile", + "to_native_string", + "decode_for_output", + "to_text", + "to_bytes", + "take", + "chunked", + "divide", ] diff --git a/pipenv/vendor/vistir/backports/tempfile.py b/pipenv/vendor/vistir/backports/tempfile.py index 43470a6e..fb044acf 100644 --- a/pipenv/vendor/vistir/backports/tempfile.py +++ b/pipenv/vendor/vistir/backports/tempfile.py @@ -175,6 +175,7 @@ def NamedTemporaryFile( prefix=None, dir=None, delete=True, + wrapper_class_override=None ): """Create and return a temporary file. Arguments: @@ -193,6 +194,8 @@ def NamedTemporaryFile( flags = _bin_openflags # Setting O_TEMPORARY in the flags causes the OS to delete # the file when it is closed. This is only supported by Windows. + if not wrapper_class_override: + wrapper_class_override = _TemporaryFileWrapper if os.name == "nt" and delete: flags |= os.O_TEMPORARY if sys.version_info < (3, 5): @@ -203,7 +206,12 @@ def NamedTemporaryFile( file = io.open( fd, mode, buffering=buffering, newline=newline, encoding=encoding ) - return _TemporaryFileWrapper(file, name, delete) + if wrapper_class_override is not None: + return type( + str("_TempFileWrapper"), (wrapper_class_override, object), {} + )(file, name, delete) + else: + return _TemporaryFileWrapper(file, name, delete) except BaseException: os.unlink(name) diff --git a/pipenv/vendor/vistir/compat.py b/pipenv/vendor/vistir/compat.py index 0c865fe6..83226481 100644 --- a/pipenv/vendor/vistir/compat.py +++ b/pipenv/vendor/vistir/compat.py @@ -1,6 +1,7 @@ # -*- coding=utf-8 -*- from __future__ import absolute_import, unicode_literals +import errno import os import sys import warnings @@ -16,22 +17,29 @@ __all__ = [ "finalize", "partialmethod", "JSONDecodeError", + "FileNotFoundError", "ResourceWarning", "FileNotFoundError", + "PermissionError", + "IsADirectoryError", "fs_str", + "lru_cache", "TemporaryDirectory", "NamedTemporaryFile", + "to_native_string", ] if sys.version_info >= (3, 5): from pathlib import Path - + from functools import lru_cache else: from pathlib2 import Path + from pipenv.vendor.backports.functools_lru_cache import lru_cache +from .backports.tempfile import NamedTemporaryFile as _NamedTemporaryFile if sys.version_info < (3, 3): from pipenv.vendor.backports.shutil_get_terminal_size import get_terminal_size - from .backports.tempfile import NamedTemporaryFile + NamedTemporaryFile = _NamedTemporaryFile else: from tempfile import NamedTemporaryFile from shutil import get_terminal_size @@ -57,16 +65,27 @@ if six.PY2: pass class FileNotFoundError(IOError): + """No such file or directory""" + + def __init__(self, *args, **kwargs): + self.errno = errno.ENOENT + super(FileNotFoundError, self).__init__(*args, **kwargs) + + class PermissionError(OSError): + def __init__(self, *args, **kwargs): + self.errno = errno.EACCES + super(PermissionError, self).__init__(*args, **kwargs) + + class IsADirectoryError(OSError): + """The command does not work on directories""" pass else: - from builtins import ResourceWarning, FileNotFoundError + from builtins import ResourceWarning, FileNotFoundError, PermissionError, IsADirectoryError - class ResourceWarning(ResourceWarning): - pass - class FileNotFoundError(FileNotFoundError): - pass +if not sys.warnoptions: + warnings.simplefilter("default", ResourceWarning) class TemporaryDirectory(object): @@ -103,9 +122,39 @@ class TemporaryDirectory(object): ) @classmethod - def _cleanup(cls, name, warn_message): + def _rmtree(cls, name): from .path import rmtree - rmtree(name) + + def onerror(func, path, exc_info): + if issubclass(exc_info[0], (PermissionError, OSError)): + try: + try: + if path != name: + os.chflags(os.path.dirname(path), 0) + os.chflags(path, 0) + except AttributeError: + pass + if path != name: + os.chmod(os.path.dirname(path), 0o70) + os.chmod(path, 0o700) + + try: + os.unlink(path) + # PermissionError is raised on FreeBSD for directories + except (IsADirectoryError, PermissionError, OSError): + cls._rmtree(path) + except FileNotFoundError: + pass + elif issubclass(exc_info[0], FileNotFoundError): + pass + else: + raise + + rmtree(name, onerror=onerror) + + @classmethod + def _cleanup(cls, name, warn_message): + cls._rmtree(name) warnings.warn(warn_message, ResourceWarning) def __repr__(self): @@ -118,9 +167,8 @@ class TemporaryDirectory(object): self.cleanup() def cleanup(self): - from .path import rmtree if self._finalizer.detach(): - rmtree(self.name) + self._rmtree(self.name) def fs_str(string): @@ -135,3 +183,10 @@ def fs_str(string): _fs_encoding = sys.getfilesystemencoding() or sys.getdefaultencoding() + + +def to_native_string(string): + from .misc import to_text, to_bytes + if six.PY2: + return to_bytes(string) + return to_text(string) diff --git a/pipenv/vendor/vistir/contextmanagers.py b/pipenv/vendor/vistir/contextmanagers.py index 80f1f897..0920a9c3 100644 --- a/pipenv/vendor/vistir/contextmanagers.py +++ b/pipenv/vendor/vistir/contextmanagers.py @@ -1,6 +1,7 @@ # -*- coding=utf-8 -*- from __future__ import absolute_import, unicode_literals +import io import os import stat import sys @@ -13,7 +14,9 @@ from .compat import NamedTemporaryFile, Path from .path import is_file_url, is_valid_url, path_to_url, url_to_path -__all__ = ["temp_environ", "temp_path", "cd", "atomic_open_for_write", "open_file"] +__all__ = [ + "temp_environ", "temp_path", "cd", "atomic_open_for_write", "open_file", "spinner" +] # Borrowed from Pew. @@ -60,7 +63,7 @@ def cd(path): >>> print(os.path.abspath(os.curdir)) '/home/user/code/myrepo' >>> with cd("/home/user/code/otherdir/subdir"): - print("Changed directory: %s" % os.path.abspath(os.curdir)) + ... print("Changed directory: %s" % os.path.abspath(os.curdir)) Changed directory: /home/user/code/otherdir/subdir >>> print(os.path.abspath(os.curdir)) '/home/user/code/myrepo' @@ -77,6 +80,74 @@ def cd(path): os.chdir(prev_cwd) +@contextmanager +def dummy_spinner(spin_type, text, **kwargs): + class FakeClass(object): + def __init__(self, text=""): + self.text = text + + def fail(self, exitcode=1, text=None): + if text: + print(text) + raise SystemExit(exitcode, text) + + def ok(self, text): + print(text) + return 0 + + def write(self, text): + print(text) + + myobj = FakeClass(text) + yield myobj + + +@contextmanager +def spinner(spinner_name=None, start_text=None, handler_map=None, nospin=False): + """Get a spinner object or a dummy spinner to wrap a context. + + :param str spinner_name: A spinner type e.g. "dots" or "bouncingBar" (default: {"bouncingBar"}) + :param str start_text: Text to start off the spinner with (default: {None}) + :param dict handler_map: Handler map for signals to be handled gracefully (default: {None}) + :param bool nospin: If true, use the dummy spinner (default: {False}) + :return: A spinner object which can be manipulated while alive + :rtype: :class:`~vistir.spin.VistirSpinner` + + Raises: + RuntimeError -- Raised if the spinner extra is not installed + """ + + from .spin import create_spinner + has_yaspin = None + try: + import yaspin + except ImportError: + has_yaspin = False + if not nospin: + raise RuntimeError( + "Failed to import spinner! Reinstall vistir with command:" + " pip install --upgrade vistir[spinner]" + ) + else: + spinner_name = "" + else: + has_yaspin = True + spinner_name = "" + use_yaspin = (has_yaspin is False) or (nospin is True) + if has_yaspin is None or has_yaspin is True and not nospin: + use_yaspin = True + if not start_text and nospin is False: + start_text = "Running..." + with create_spinner( + spinner_name=spinner_name, + text=start_text, + handler_map=handler_map, + nospin=nospin, + use_yaspin=use_yaspin + ) as _spinner: + yield _spinner + + @contextmanager def atomic_open_for_write(target, binary=False, newline=None, encoding=None): """Atomically open `target` for writing. @@ -168,12 +239,13 @@ def atomic_open_for_write(target, binary=False, newline=None, encoding=None): @contextmanager -def open_file(link, session=None): +def open_file(link, session=None, stream=True): """ Open local or remote file for reading. :type link: pip._internal.index.Link or str :type session: requests.Session + :param bool stream: Try to stream if remote, default True :raises ValueError: If link points to a local directory. :return: a context manager to the opened file-like object """ @@ -192,8 +264,8 @@ def open_file(link, session=None): if os.path.isdir(local_path): raise ValueError("Cannot open directory for read: {}".format(link)) else: - with open(local_path, "rb") as local_file: - yield local_file + with io.open(local_path, "rb") as local_file: + yield local_file else: # Remote URL headers = {"Accept-Encoding": "identity"} @@ -201,8 +273,14 @@ def open_file(link, session=None): from requests import Session session = Session() - response = session.get(link, headers=headers, stream=True) - try: - yield response.raw - finally: - response.close() + with session.get(link, headers=headers, stream=stream) as resp: + try: + raw = getattr(resp, "raw", None) + result = raw if raw else resp + yield result + finally: + if raw: + conn = getattr(raw, "_connection") + if conn is not None: + conn.close() + result.close() diff --git a/pipenv/vendor/vistir/misc.py b/pipenv/vendor/vistir/misc.py index 44607a98..a9a127d8 100644 --- a/pipenv/vendor/vistir/misc.py +++ b/pipenv/vendor/vistir/misc.py @@ -2,19 +2,25 @@ from __future__ import absolute_import, unicode_literals import json +import logging import locale import os import subprocess import sys from collections import OrderedDict -from contextlib import contextmanager from functools import partial +from itertools import islice import six from .cmdparse import Script -from .compat import Path, fs_str, partialmethod +from .compat import Path, fs_str, partialmethod, to_native_string +from .contextmanagers import spinner as spinner + +if os.name != "nt": + class WindowsError(OSError): + pass __all__ = [ @@ -27,9 +33,30 @@ __all__ = [ "to_text", "to_bytes", "locale_encoding", + "chunked", + "take", + "divide", + "getpreferredencoding", + "decode_for_output", ] +def _get_logger(name=None, level="ERROR"): + if not name: + name = __name__ + if isinstance(level, six.string_types): + level = getattr(logging, level.upper()) + logger = logging.getLogger(name) + logger.setLevel(level) + formatter = logging.Formatter( + "%(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s" + ) + handler = logging.StreamHandler(stream=sys.stderr) + handler.setFormatter(formatter) + logger.addHandler(handler) + return logger + + def shell_escape(cmd): """Escape strings for use in :func:`~subprocess.Popen` and :func:`run`. @@ -75,9 +102,11 @@ def dedup(iterable): return iter(OrderedDict.fromkeys(iterable)) -def _spawn_subprocess(script, env={}, block=True, cwd=None, combine_stderr=True): +def _spawn_subprocess(script, env=None, block=True, cwd=None, combine_stderr=True): from distutils.spawn import find_executable + if not env: + env = {} command = find_executable(script.command) options = { "env": env, @@ -102,7 +131,7 @@ def _spawn_subprocess(script, env={}, block=True, cwd=None, combine_stderr=True) try: return subprocess.Popen(cmd, **options) except WindowsError as e: - if e.winerror != 193: + if getattr(e, "winerror", 9999) != 193: raise options["shell"] = True # Try shell mode to use Windows's file association for file launch. @@ -111,15 +140,18 @@ def _spawn_subprocess(script, env={}, block=True, cwd=None, combine_stderr=True) def _create_subprocess( cmd, - env={}, + env=None, block=True, return_object=False, cwd=os.curdir, verbose=False, spinner=None, combine_stderr=False, - display_limit=200 + display_limit=200, + start_text="" ): + if not env: + env = {} try: c = _spawn_subprocess(cmd, env=env, block=block, cwd=cwd, combine_stderr=combine_stderr) @@ -130,9 +162,11 @@ def _create_subprocess( c.stdin.close() output = [] err = [] - spinner_orig_text = "" + spinner_orig_text = None if spinner: - spinner_orig_text = spinner.text + spinner_orig_text = getattr(spinner, "text", None) + if spinner_orig_text is None: + spinner_orig_text = start_text if start_text is not None else "" streams = { "stdout": c.stdout, "stderr": c.stderr @@ -147,23 +181,39 @@ def _create_subprocess( line = to_text(stream.readline()) if not line: continue - line = line.rstrip() + line = to_text("{0}".format(line.rstrip())) if outstream == "stderr": stderr_line = line else: stdout_line = line if not (stdout_line or stderr_line): break - if stderr_line: - err.append(line) - if stdout_line: + if stderr_line is not None: + err.append(stderr_line) + err_line = fs_str("{0}".format(stderr_line)) + if verbose and err_line is not None: + if spinner: + spinner._hide_cursor() + spinner.write_err(err_line) + spinner._show_cursor() + else: + sys.stderr.write(err_line) + sys.stderr.flush() + if stdout_line is not None: output.append(stdout_line) - display_line = stdout_line + display_line = fs_str("{0}".format(stdout_line)) if len(stdout_line) > display_limit: display_line = "{0}...".format(stdout_line[:display_limit]) - if verbose: - spinner.write(display_line) - spinner.text = "{0} {1}".format(spinner_orig_text, display_line) + if verbose and display_line is not None: + if spinner: + spinner._hide_cursor() + spinner.write_err(display_line) + spinner._show_cursor() + else: + sys.stderr.write(display_line) + sys.stderr.flush() + if spinner: + spinner.text = to_native_string("{0} {1}".format(spinner_orig_text, display_line)) continue try: c.wait() @@ -174,19 +224,21 @@ def _create_subprocess( c.stderr.close() if spinner: if c.returncode > 0: - spinner.fail("Failed...cleaning up...") - spinner.text = "Complete!" - spinner.ok("✔") - c.out = "\n".join(output) + spinner.fail(to_native_string("Failed...cleaning up...")) + if not os.name == "nt": + spinner.ok(to_native_string("✔ Complete")) + else: + spinner.ok(to_native_string("Complete")) + c.out = "\n".join(output) if output else "" c.err = "\n".join(err) if err else "" else: c.out, c.err = c.communicate() + if not block: + c.wait() + c.out = to_text("{0}".format(c.out)) if c.out else fs_str("") + c.err = to_text("{0}".format(c.err)) if c.err else fs_str("") if not return_object: - if not block: - c.wait() - out = c.out if c.out else "" - err = c.err if c.err else "" - return out.strip(), err.strip() + return c.out.strip(), c.err.strip() return c @@ -198,7 +250,7 @@ def run( cwd=None, verbose=False, nospin=False, - spinner=None, + spinner_name=None, combine_stderr=True, display_limit=200 ): @@ -211,7 +263,7 @@ def run( :param str cwd: Current working directory contect to use for spawning the subprocess. :param bool verbose: Whether to print stdout in real time when non-blocking. :param bool nospin: Whether to disable the cli spinner. - :param str spinner: The name of the spinner to use if enabled, defaults to bouncingBar + :param str spinner_name: The name of the spinner to use if enabled, defaults to bouncingBar :param bool combine_stderr: Optionally merge stdout and stderr in the subprocess, false if nonblocking. :param int dispay_limit: The max width of output lines to display when using a spinner. :returns: A 2-tuple of (output, error) or a :class:`subprocess.Popen` object. @@ -221,8 +273,10 @@ def run( this functionality. """ - if not env: - env = os.environ.copy() + _env = os.environ.copy() + if env: + _env.update(env) + env = _env if six.PY2: fs_encode = partial(to_bytes, encoding=locale_encoding) _env = {fs_encode(k): fs_encode(v) for k, v in os.environ.items()} @@ -230,8 +284,8 @@ def run( _env[fs_encode(key)] = fs_encode(val) else: _env = {k: fs_str(v) for k, v in os.environ.items()} - if not spinner: - spinner = "bouncingBar" + if not spinner_name: + spinner_name = "bouncingBar" if six.PY2: if isinstance(cmd, six.string_types): cmd = cmd.encode("utf-8") @@ -241,48 +295,8 @@ def run( cmd = Script.parse(cmd) if block or not return_object: combine_stderr = False - sigmap = {} - if nospin is False: - try: - import signal - from yaspin import yaspin - from yaspin import spinners - from yaspin.signal_handlers import fancy_handler - except ImportError: - raise RuntimeError( - "Failed to import spinner! Reinstall vistir with command:" - " pip install --upgrade vistir[spinner]" - ) - else: - animation = getattr(spinners.Spinners, spinner) - sigmap = { - signal.SIGINT: fancy_handler - } - if os.name == "nt": - sigmap.update({ - signal.CTRL_C_EVENT: fancy_handler, - signal.CTRL_BREAK_EVENT: fancy_handler - }) - spinner_func = yaspin - else: - - @contextmanager - def spinner_func(spin_type, text, **kwargs): - class FakeClass(object): - def __init__(self, text=""): - self.text = text - - def ok(self, text): - return - - def write(self, text): - print(text) - - myobj = FakeClass(text) - yield myobj - - animation = None - with spinner_func(animation, sigmap=sigmap, text="Running...") as sp: + start_text = "" + with spinner(spinner_name=spinner_name, start_text=start_text, nospin=nospin) as sp: return _create_subprocess( cmd, env=_env, @@ -291,7 +305,8 @@ def run( cwd=cwd, verbose=verbose, spinner=sp, - combine_stderr=combine_stderr + combine_stderr=combine_stderr, + start_text=start_text ) @@ -426,7 +441,87 @@ def to_text(string, encoding="utf-8", errors=None): return string +def divide(n, iterable): + """ + split an iterable into n groups, per https://more-itertools.readthedocs.io/en/latest/api.html#grouping + + :param int n: Number of unique groups + :param iter iterable: An iterable to split up + :return: a list of new iterables derived from the original iterable + :rtype: list + """ + + seq = tuple(iterable) + q, r = divmod(len(seq), n) + + ret = [] + for i in range(n): + start = (i * q) + (i if i < r else r) + stop = ((i + 1) * q) + (i + 1 if i + 1 < r else r) + ret.append(iter(seq[start:stop])) + + return ret + + +def take(n, iterable): + """Take n elements from the supplied iterable without consuming it. + + :param int n: Number of unique groups + :param iter iterable: An iterable to split up + + from https://github.com/erikrose/more-itertools/blob/master/more_itertools/recipes.py + """ + + return list(islice(iterable, n)) + + +def chunked(n, iterable): + """Split an iterable into lists of length *n*. + + :param int n: Number of unique groups + :param iter iterable: An iterable to split up + + from https://github.com/erikrose/more-itertools/blob/master/more_itertools/more.py + """ + + return iter(partial(take, n, iter(iterable)), []) + + try: locale_encoding = locale.getdefaultencoding()[1] or "ascii" except Exception: locale_encoding = "ascii" + + +def getpreferredencoding(): + """Determine the proper output encoding for terminal rendering""" + + # Borrowed from Invoke + # (see https://github.com/pyinvoke/invoke/blob/93af29d/invoke/runners.py#L881) + _encoding = locale.getpreferredencoding(False) + if six.PY2 and not sys.platform == "win32": + _default_encoding = locale.getdefaultlocale()[1] + if _default_encoding is not None: + _encoding = _default_encoding + return _encoding + + +PREFERRED_ENCODING = getpreferredencoding() + + +def decode_for_output(output): + """Given a string, decode it for output to a terminal + + :param str output: A string to print to a terminal + :return: A re-encoded string using the preferred encoding + :rtype: str + """ + + if not isinstance(output, six.string_types): + return output + try: + output = output.encode(PREFERRED_ENCODING) + except AttributeError: + pass + output = output.decode(PREFERRED_ENCODING) + return output diff --git a/pipenv/vendor/vistir/path.py b/pipenv/vendor/vistir/path.py index 166282e8..febaddbc 100644 --- a/pipenv/vendor/vistir/path.py +++ b/pipenv/vendor/vistir/path.py @@ -1,5 +1,5 @@ # -*- coding=utf-8 -*- -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, unicode_literals, print_function import atexit import errno @@ -15,8 +15,15 @@ import six from six.moves import urllib_parse from six.moves.urllib import request as urllib_request -from .compat import Path, _fs_encoding, TemporaryDirectory -from .misc import locale_encoding, to_bytes, to_text +from .backports.tempfile import _TemporaryFileWrapper +from .compat import ( + _NamedTemporaryFile, + Path, + ResourceWarning, + TemporaryDirectory, + _fs_encoding, + finalize, +) __all__ = [ @@ -29,6 +36,7 @@ __all__ = [ "mkdir_p", "ensure_mkdir_p", "create_tracked_tempdir", + "create_tracked_tempfile", "path_to_url", "rmtree", "safe_expandvars", @@ -38,6 +46,10 @@ __all__ = [ ] +if os.name == "nt": + warnings.filterwarnings("ignore", category=DeprecationWarning, message="The Windows bytes API has been deprecated.*") + + def unicode_path(path): # Paths are supposed to be represented as unicode here if six.PY2 and not isinstance(path, six.text_type): @@ -53,9 +65,10 @@ def native_path(path): # once again thank you django... # https://github.com/django/django/blob/fc6b90b/django/utils/_os.py -if six.PY3 or os.name == 'nt': +if six.PY3 or os.name == "nt": abspathu = os.path.abspath else: + def abspathu(path): """ Version of os.path.abspath that uses the unicode representation @@ -74,6 +87,8 @@ def normalize_drive(path): identified with either upper or lower cased drive names. The case is always converted to uppercase because it seems to be preferred. """ + from .misc import to_text + if os.name != "nt" or not isinstance(path, six.string_types): return path @@ -95,6 +110,7 @@ def path_to_url(path): >>> path_to_url("/home/user/code/myrepo/myfile.zip") 'file:///home/user/code/myrepo/myfile.zip' """ + from .misc import to_text, to_bytes if not path: return path @@ -108,6 +124,8 @@ def url_to_path(url): Follows logic taken from pip's equivalent function """ + from .misc import to_bytes + assert is_file_url(url), "Only file: urls can be converted to local paths" _, netloc, path, _, _ = urllib_parse.urlsplit(url) # Netlocs are UNC paths @@ -120,14 +138,18 @@ def url_to_path(url): def is_valid_url(url): """Checks if a given string is an url""" + from .misc import to_text + if not url: return url - pieces = urllib_parse.urlparse(url) + pieces = urllib_parse.urlparse(to_text(url)) return all([pieces.scheme, pieces.netloc]) def is_file_url(url): """Returns true if the given url is a file url""" + from .misc import to_text + if not url: return False if not isinstance(url, six.string_types): @@ -144,9 +166,12 @@ def is_readonly_path(fn): Permissions check is `bool(path.stat & stat.S_IREAD)` or `not os.access(path, os.W_OK)` """ - fn = to_bytes(fn, encoding="utf-8") + from .compat import to_native_string + + fn = to_native_string(fn) if os.path.exists(fn): - return bool(os.stat(fn).st_mode & stat.S_IREAD) and not os.access(fn, os.W_OK) + file_stat = os.stat(fn).st_mode + return not bool(file_stat & stat.S_IWRITE) or not os.access(fn, os.W_OK) return False @@ -158,7 +183,9 @@ def mkdir_p(newdir, mode=0o777): :raises: OSError if a file is encountered along the way """ # http://code.activestate.com/recipes/82465-a-friendly-mkdir/ - newdir = abspathu(to_bytes(newdir, "utf-8")) + from .misc import to_bytes, to_text + + newdir = to_bytes(newdir, "utf-8") if os.path.exists(newdir): if not os.path.isdir(newdir): raise OSError( @@ -166,17 +193,19 @@ def mkdir_p(newdir, mode=0o777): newdir ) ) - pass else: - head, tail = os.path.split(newdir) + head, tail = os.path.split(to_bytes(newdir, encoding="utf-8")) # Make sure the tail doesn't point to the asame place as the head - tail_and_head_match = os.path.relpath(tail, start=os.path.basename(head)) == "." + curdir = to_bytes(".", encoding="utf-8") + tail_and_head_match = ( + os.path.relpath(tail, start=os.path.basename(head)) == curdir + ) if tail and not tail_and_head_match and not os.path.isdir(newdir): target = os.path.join(head, tail) if os.path.exists(target) and os.path.isfile(target): raise OSError( "A file with the same name as the desired dir, '{0}', already exists.".format( - newdir + to_text(newdir, encoding="utf-8") ) ) os.makedirs(os.path.join(head, tail), mode) @@ -185,8 +214,8 @@ def mkdir_p(newdir, mode=0o777): def ensure_mkdir_p(mode=0o777): """Decorator to ensure `mkdir_p` is called to the function's return value. """ - def decorator(f): + def decorator(f): @functools.wraps(f) def decorated(*args, **kwargs): path = f(*args, **kwargs) @@ -210,12 +239,27 @@ def create_tracked_tempdir(*args, **kwargs): The return value is the path to the created directory. """ + tempdir = TemporaryDirectory(*args, **kwargs) TRACKED_TEMPORARY_DIRECTORIES.append(tempdir) atexit.register(tempdir.cleanup) + warnings.simplefilter("ignore", ResourceWarning) return tempdir.name +def create_tracked_tempfile(*args, **kwargs): + """Create a tracked temporary file. + + This uses the `NamedTemporaryFile` construct, but does not remove the file + until the interpreter exits. + + The return value is the file object. + """ + + kwargs["wrapper_class_override"] = _TrackedTempfileWrapper + return _NamedTemporaryFile(*args, **kwargs) + + def set_write_bit(fn): """Set read-write permissions for the current user on the target path. Fail silently if the path doesn't exist. @@ -223,13 +267,26 @@ def set_write_bit(fn): :param str fn: The target filename or path """ - fn = to_bytes(fn, encoding=locale_encoding) + from .compat import to_native_string + + fn = to_native_string(fn) if not os.path.exists(fn): return - os.chmod(fn, stat.S_IWRITE | stat.S_IWUSR | stat.S_IRUSR) + file_stat = os.stat(fn).st_mode + os.chmod(fn, file_stat | stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + if not os.path.isdir(fn): + try: + os.chflags(fn, 0) + except AttributeError: + pass + for root, dirs, files in os.walk(fn, topdown=False): + for dir_ in [os.path.join(root,d) for d in dirs]: + set_write_bit(dir_) + for file_ in [os.path.join(root, f) for f in files]: + set_write_bit(file_) -def rmtree(directory, ignore_errors=False): +def rmtree(directory, ignore_errors=False, onerror=None): """Stand-in for :func:`~shutil.rmtree` with additional error-handling. This version of `rmtree` handles read-only paths, especially in the case of index @@ -237,16 +294,27 @@ def rmtree(directory, ignore_errors=False): :param str directory: The target directory to remove :param bool ignore_errors: Whether to ignore errors, defaults to False + :param func onerror: An error handling function, defaults to :func:`handle_remove_readonly` .. note:: Setting `ignore_errors=True` may cause this to silently fail to delete the path """ - directory = to_bytes(directory, encoding=locale_encoding) - shutil.rmtree( - directory, ignore_errors=ignore_errors, onerror=handle_remove_readonly - ) + from .compat import to_native_string + + directory = to_native_string(directory) + if onerror is None: + onerror = handle_remove_readonly + try: + shutil.rmtree( + directory, ignore_errors=ignore_errors, onerror=onerror + ) + except (IOError, OSError, FileNotFoundError) as exc: + # Ignore removal failures where the file doesn't exist + if exc.errno == errno.ENOENT: + pass + raise def handle_remove_readonly(func, path, exc): @@ -263,35 +331,47 @@ def handle_remove_readonly(func, path, exc): :func:`set_write_bit` on the target path and try again. """ # Check for read-only attribute - from .compat import ResourceWarning + from .compat import ( + ResourceWarning, FileNotFoundError, PermissionError, to_native_string + ) + + PERM_ERRORS = (errno.EACCES, errno.EPERM, errno.ENOENT) default_warning_message = ( "Unable to remove file due to permissions restriction: {!r}" ) # split the initial exception out into its type, exception, and traceback exc_type, exc_exception, exc_tb = exc - path = to_bytes(path) + path = to_native_string(path) if is_readonly_path(path): # Apply write permission and call original function set_write_bit(path) try: func(path) - except (OSError, IOError) as e: - if e.errno in [errno.EACCES, errno.EPERM]: - warnings.warn( - default_warning_message.format( - to_text(path, encoding=locale_encoding) - ), ResourceWarning - ) + except (OSError, IOError, FileNotFoundError, PermissionError) as e: + if e.errno == errno.ENOENT: + return + elif e.errno in PERM_ERRORS: + warnings.warn(default_warning_message.format(path), ResourceWarning) return - if exc_exception.errno in [errno.EACCES, errno.EPERM]: - warnings.warn( - default_warning_message.format(to_text(path)), - ResourceWarning - ) - return - - raise + if exc_exception.errno in PERM_ERRORS: + set_write_bit(path) + try: + func(path) + except (OSError, IOError, FileNotFoundError, PermissionError) as e: + if e.errno in PERM_ERRORS: + warnings.warn(default_warning_message.format(path), ResourceWarning) + pass + elif e.errno == errno.ENOENT: # File already gone + pass + else: + raise + else: + return + elif exc_exception.errno == errno.ENOENT: + pass + else: + raise exc_exception def walk_up(bottom): @@ -356,6 +436,7 @@ def get_converted_relative_path(path, relative_to=None): >>> vistir.path.get_converted_relative_path('/home/user/code/myrepo/myfolder') '.' """ + from .misc import to_text, to_bytes # noqa if not relative_to: relative_to = os.getcwdu() if six.PY2 else os.getcwd() @@ -395,3 +476,28 @@ def safe_expandvars(value): if isinstance(value, six.string_types): return os.path.expandvars(value) return value + + +class _TrackedTempfileWrapper(_TemporaryFileWrapper, object): + def __init__(self, *args, **kwargs): + super(_TrackedTempfileWrapper, self).__init__(*args, **kwargs) + self._finalizer = finalize(self, self.cleanup) + + @classmethod + def _cleanup(cls, fileobj): + try: + fileobj.close() + finally: + os.unlink(fileobj.name) + + def cleanup(self): + if self._finalizer.detach(): + try: + self.close() + finally: + os.unlink(self.name) + else: + try: + self.close() + except OSError: + pass diff --git a/pipenv/vendor/vistir/spin.py b/pipenv/vendor/vistir/spin.py new file mode 100644 index 00000000..e7311555 --- /dev/null +++ b/pipenv/vendor/vistir/spin.py @@ -0,0 +1,297 @@ +# -*- coding=utf-8 -*- + +import functools +import os +import signal +import sys + +import colorama +import cursor +import six + +from .compat import to_native_string +from .termcolors import COLOR_MAP, COLORS, colored +from io import StringIO + +try: + import yaspin +except ImportError: + yaspin = None + Spinners = None +else: + from yaspin.spinners import Spinners + +handler = None +if yaspin and os.name == "nt": + handler = yaspin.signal_handlers.default_handler +elif yaspin and os.name != "nt": + handler = yaspin.signal_handlers.fancy_handler + +CLEAR_LINE = chr(27) + "[K" + + +class DummySpinner(object): + def __init__(self, text="", **kwargs): + colorama.init() + from .misc import decode_for_output + self.text = to_native_string(decode_for_output(text)) + self.stdout = kwargs.get("stdout", sys.stdout) + self.stderr = kwargs.get("stderr", sys.stderr) + self.out_buff = StringIO() + + def __enter__(self): + if self.text and self.text != "None": + self.write_err(self.text) + return self + + def __exit__(self, exc_type, exc_val, traceback): + if exc_type: + import traceback + from .misc import decode_for_output + self.write_err(decode_for_output(traceback.format_exception(*sys.exc_info()))) + self._close_output_buffer() + return False + + def __getattr__(self, k): + try: + retval = super(DummySpinner, self).__getattribute__(k) + except AttributeError: + if k in COLOR_MAP.keys() or k.upper() in COLORS: + return self + raise + else: + return retval + + def _close_output_buffer(self): + if self.out_buff and not self.out_buff.closed: + try: + self.out_buff.close() + except Exception: + pass + + def fail(self, exitcode=1, text="FAIL"): + from .misc import decode_for_output + if text and text != "None": + self.write_err(decode_for_output(text)) + self._close_output_buffer() + + def ok(self, text="OK"): + if text and text != "None": + self.stderr.write(self.text) + self._close_output_buffer() + return 0 + + def write(self, text=None): + from .misc import decode_for_output + if text is None or isinstance(text, six.string_types) and text == "None": + pass + text = decode_for_output(text) + self.stdout.write(decode_for_output("\r")) + line = decode_for_output("{0}\n".format(text)) + self.stdout.write(line) + self.stdout.write(CLEAR_LINE) + + def write_err(self, text=None): + from .misc import decode_for_output + if text is None or isinstance(text, six.string_types) and text == "None": + pass + text = decode_for_output(text) + self.stderr.write(decode_for_output("\r")) + line = decode_for_output("{0}\n".format(text)) + self.stderr.write(line) + self.stderr.write(CLEAR_LINE) + + @staticmethod + def _hide_cursor(): + pass + + @staticmethod + def _show_cursor(): + pass + + +base_obj = yaspin.core.Yaspin if yaspin is not None else DummySpinner + + +class VistirSpinner(base_obj): + def __init__(self, *args, **kwargs): + """Get a spinner object or a dummy spinner to wrap a context. + + Keyword Arguments: + :param str spinner_name: A spinner type e.g. "dots" or "bouncingBar" (default: {"bouncingBar"}) + :param str start_text: Text to start off the spinner with (default: {None}) + :param dict handler_map: Handler map for signals to be handled gracefully (default: {None}) + :param bool nospin: If true, use the dummy spinner (default: {False}) + """ + + self.handler = handler + colorama.init() + sigmap = {} + if handler: + sigmap.update({ + signal.SIGINT: handler, + signal.SIGTERM: handler + }) + handler_map = kwargs.pop("handler_map", {}) + if os.name == "nt": + sigmap[signal.SIGBREAK] = handler + else: + sigmap[signal.SIGALRM] = handler + if handler_map: + sigmap.update(handler_map) + spinner_name = kwargs.pop("spinner_name", "bouncingBar") + start_text = kwargs.pop("start_text", None) + _text = kwargs.pop("text", "Running...") + kwargs["text"] = start_text if start_text is not None else _text + kwargs["sigmap"] = sigmap + kwargs["spinner"] = getattr(Spinners, spinner_name, "") + self.stdout = kwargs.pop("stdout", sys.stdout) + self.stderr = kwargs.pop("stderr", sys.stderr) + self.out_buff = StringIO() + super(VistirSpinner, self).__init__(*args, **kwargs) + self.is_dummy = bool(yaspin is None) + + def ok(self, text="OK"): + """Set Ok (success) finalizer to a spinner.""" + # Do not display spin text for ok state + self._text = None + + _text = text if text else "OK" + self._freeze(_text) + + def fail(self, text="FAIL"): + """Set fail finalizer to a spinner.""" + # Do not display spin text for fail state + self._text = None + + _text = text if text else "FAIL" + self._freeze(_text) + + def write(self, text): + from .misc import to_text + sys.stdout.write("\r") + self.stdout.write(CLEAR_LINE) + if text is None: + text = "" + text = to_native_string("{0}\n".format(text)) + sys.stdout.write(text) + self.out_buff.write(to_text(text)) + + def write_err(self, text): + """Write error text in the terminal without breaking the spinner.""" + from .misc import to_text + + self.stderr.write("\r") + self.stderr.write(CLEAR_LINE) + if text is None: + text = "" + text = to_native_string("{0}\n".format(text)) + self.stderr.write(text) + self.out_buff.write(to_text(text)) + + def _freeze(self, final_text): + """Stop spinner, compose last frame and 'freeze' it.""" + if not final_text: + final_text = "" + text = to_native_string(final_text) + self._last_frame = self._compose_out(text, mode="last") + + # Should be stopped here, otherwise prints after + # self._freeze call will mess up the spinner + self.stop() + self.stdout.write(self._last_frame) + + def stop(self, *args, **kwargs): + if self.stderr and self.stderr != sys.stderr: + self.stderr.close() + if self.stdout and self.stdout != sys.stdout: + self.stdout.close() + self.out_buff.close() + super(VistirSpinner, self).stop(*args, **kwargs) + + def _compose_color_func(self): + fn = functools.partial( + colored, + color=self._color, + on_color=self._on_color, + attrs=list(self._attrs), + ) + return fn + + def _compose_out(self, frame, mode=None): + # Ensure Unicode input + + frame = to_native_string(frame) + if self._text is None: + self._text = "" + text = to_native_string(self._text) + if self._color_func is not None: + frame = self._color_func(frame) + if self._side == "right": + frame, text = text, frame + # Mode + if not mode: + out = to_native_string("\r{0} {1}".format(frame, text)) + else: + out = to_native_string("{0} {1}\n".format(frame, text)) + return out + + def _register_signal_handlers(self): + # SIGKILL cannot be caught or ignored, and the receiving + # process cannot perform any clean-up upon receiving this + # signal. + try: + if signal.SIGKILL in self._sigmap.keys(): + raise ValueError( + "Trying to set handler for SIGKILL signal. " + "SIGKILL cannot be cought or ignored in POSIX systems." + ) + except AttributeError: + pass + + for sig, sig_handler in self._sigmap.items(): + # A handler for a particular signal, once set, remains + # installed until it is explicitly reset. Store default + # signal handlers for subsequent reset at cleanup phase. + dfl_handler = signal.getsignal(sig) + self._dfl_sigmap[sig] = dfl_handler + + # ``signal.SIG_DFL`` and ``signal.SIG_IGN`` are also valid + # signal handlers and are not callables. + if callable(sig_handler): + # ``signal.signal`` accepts handler function which is + # called with two arguments: signal number and the + # interrupted stack frame. ``functools.partial`` solves + # the problem of passing spinner instance into the handler + # function. + sig_handler = functools.partial(sig_handler, spinner=self) + + signal.signal(sig, sig_handler) + + def _reset_signal_handlers(self): + for sig, sig_handler in self._dfl_sigmap.items(): + signal.signal(sig, sig_handler) + + @staticmethod + def _hide_cursor(): + cursor.hide() + + @staticmethod + def _show_cursor(): + cursor.show() + + @staticmethod + def _clear_err(): + sys.stderr.write(CLEAR_LINE) + + @staticmethod + def _clear_line(): + sys.stdout.write(CLEAR_LINE) + + +def create_spinner(*args, **kwargs): + nospin = kwargs.pop("nospin", False) + use_yaspin = kwargs.pop("use_yaspin", not nospin) + if nospin or not use_yaspin: + return DummySpinner(*args, **kwargs) + return VistirSpinner(*args, **kwargs) diff --git a/pipenv/vendor/vistir/termcolors.py b/pipenv/vendor/vistir/termcolors.py new file mode 100644 index 00000000..8395d97d --- /dev/null +++ b/pipenv/vendor/vistir/termcolors.py @@ -0,0 +1,146 @@ +# -*- coding=utf-8 -*- +from __future__ import absolute_import, print_function, unicode_literals +import colorama +import os +from .compat import to_native_string + + +ATTRIBUTES = dict( + list(zip([ + 'bold', + 'dark', + '', + 'underline', + 'blink', + '', + 'reverse', + 'concealed' + ], + list(range(1, 9)) + )) + ) +del ATTRIBUTES[''] + + +HIGHLIGHTS = dict( + list(zip([ + 'on_grey', + 'on_red', + 'on_green', + 'on_yellow', + 'on_blue', + 'on_magenta', + 'on_cyan', + 'on_white' + ], + list(range(40, 48)) + )) + ) + + +COLORS = dict( + list(zip([ + 'grey', + 'red', + 'green', + 'yellow', + 'blue', + 'magenta', + 'cyan', + 'white', + ], + list(range(30, 38)) + )) + ) + + +COLOR_MAP = { + # name: type + "blink": "attrs", + "bold": "attrs", + "concealed": "attrs", + "dark": "attrs", + "reverse": "attrs", + "underline": "attrs", + "blue": "color", + "cyan": "color", + "green": "color", + "magenta": "color", + "red": "color", + "white": "color", + "yellow": "color", + "on_blue": "on_color", + "on_cyan": "on_color", + "on_green": "on_color", + "on_grey": "on_color", + "on_magenta": "on_color", + "on_red": "on_color", + "on_white": "on_color", + "on_yellow": "on_color", +} +COLOR_ATTRS = COLOR_MAP.keys() + + +RESET = colorama.Style.RESET_ALL + + +def colored(text, color=None, on_color=None, attrs=None): + """Colorize text using a reimplementation of the colorizer from + https://github.com/pavdmyt/yaspin so that it works on windows. + + Available text colors: + red, green, yellow, blue, magenta, cyan, white. + + Available text highlights: + on_red, on_green, on_yellow, on_blue, on_magenta, on_cyan, on_white. + + Available attributes: + bold, dark, underline, blink, reverse, concealed. + + Example: + colored('Hello, World!', 'red', 'on_grey', ['blue', 'blink']) + colored('Hello, World!', 'green') + """ + if os.getenv('ANSI_COLORS_DISABLED') is None: + style = "NORMAL" + if 'bold' in attrs: + style = "BRIGHT" + attrs.remove('bold') + if color is not None: + color = color.upper() + text = to_native_string("%s%s%s%s%s") % ( + to_native_string(getattr(colorama.Fore, color)), + to_native_string(getattr(colorama.Style, style)), + to_native_string(text), + to_native_string(colorama.Fore.RESET), + to_native_string(colorama.Style.NORMAL), + ) + + if on_color is not None: + on_color = on_color.upper() + text = to_native_string("%s%s%s%s") % ( + to_native_string(getattr(colorama.Back, on_color)), + to_native_string(text), + to_native_string(colorama.Back.RESET), + to_native_string(colorama.Style.NORMAL), + ) + + if attrs is not None: + fmt_str = to_native_string("%s[%%dm%%s%s[9m") % ( + chr(27), + chr(27) + ) + for attr in attrs: + text = fmt_str % (ATTRIBUTES[attr], text) + + text += RESET + return text + + +def cprint(text, color=None, on_color=None, attrs=None, **kwargs): + """Print colorize text. + + It accepts arguments of print function. + """ + + print((colored(text, color, on_color, attrs)), **kwargs) diff --git a/pipenv/vendor/yaspin/core.py b/pipenv/vendor/yaspin/core.py index d01fb98e..06b8b621 100644 --- a/pipenv/vendor/yaspin/core.py +++ b/pipenv/vendor/yaspin/core.py @@ -16,6 +16,9 @@ import sys import threading import time +import colorama +import cursor + from .base_spinner import default_spinner from .compat import PY2, basestring, builtin_str, bytes, iteritems, str from .constants import COLOR_ATTRS, COLOR_MAP, ENCODING, SPINNER_ATTRS @@ -23,6 +26,9 @@ from .helpers import to_unicode from .termcolor import colored +colorama.init() + + class Yaspin(object): """Implements a context manager that spawns a thread to write spinner frames into a tty (stdout) during @@ -369,11 +375,14 @@ class Yaspin(object): # SIGKILL cannot be caught or ignored, and the receiving # process cannot perform any clean-up upon receiving this # signal. - if signal.SIGKILL in self._sigmap.keys(): - raise ValueError( - "Trying to set handler for SIGKILL signal. " - "SIGKILL cannot be cought or ignored in POSIX systems." - ) + try: + if signal.SIGKILL in self._sigmap.keys(): + raise ValueError( + "Trying to set handler for SIGKILL signal. " + "SIGKILL cannot be cought or ignored in POSIX systems." + ) + except AttributeError: + pass for sig, sig_handler in iteritems(self._sigmap): # A handler for a particular signal, once set, remains @@ -521,14 +530,12 @@ class Yaspin(object): @staticmethod def _hide_cursor(): - sys.stdout.write("\033[?25l") - sys.stdout.flush() + cursor.hide() @staticmethod def _show_cursor(): - sys.stdout.write("\033[?25h") - sys.stdout.flush() + cursor.show() @staticmethod def _clear_line(): - sys.stdout.write("\033[K") + sys.stdout.write(chr(27) + "[K") diff --git a/pytest.ini b/pytest.ini index 48dfab02..2bfca079 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,6 +1,6 @@ [pytest] addopts = -ra -n auto -testpaths = tests/ +testpaths = tests ; Add vendor and patched in addition to the default list of ignored dirs norecursedirs = .* build dist CVS _darcs {arch} *.egg vendor patched news tasks docs filterwarnings = diff --git a/tasks/release.py b/tasks/release.py index 4a242ba5..da13291e 100644 --- a/tasks/release.py +++ b/tasks/release.py @@ -62,6 +62,17 @@ def generate_markdown(ctx): ctx.run('pandoc CHANGELOG.rst -f rst -t markdown -o CHANGELOG.md') +@invoke.task +def generate_manual(ctx, commit=False): + log('Generating manual from reStructured source...') + ctx.run('make man -C docs') + ctx.run('cp docs/_build/man/pipenv.1 pipenv/') + if commit: + log('Commiting...') + ctx.run('git add pipenv/pipenv.1') + ctx.run('git commit -m "Update manual page."') + + @invoke.task def generate_changelog(ctx, commit=False, draft=False): log('Generating changelog...') diff --git a/tasks/vendoring/__init__.py b/tasks/vendoring/__init__.py index 3198c2d4..ea0d038c 100644 --- a/tasks/vendoring/__init__.py +++ b/tasks/vendoring/__init__.py @@ -20,6 +20,7 @@ LIBRARY_DIRNAMES = { 'requirements-parser': 'requirements', 'backports.shutil_get_terminal_size': 'backports/shutil_get_terminal_size', 'backports.weakref': 'backports/weakref', + 'backports.functools_lru_cache': 'backports/functools_lru_cache', 'shutil_backports': 'backports/shutil_get_terminal_size', 'python-dotenv': 'dotenv', 'pip-tools': 'piptools', @@ -34,6 +35,7 @@ PY2_DOWNLOAD = ['enum34',] # from time to time, remove the no longer needed ones HARDCODED_LICENSE_URLS = { 'pytoml': 'https://github.com/avakar/pytoml/raw/master/LICENSE', + 'cursor': 'https://raw.githubusercontent.com/GijsTimmers/cursor/master/LICENSE', 'delegator.py': 'https://raw.githubusercontent.com/kennethreitz/delegator.py/master/LICENSE', 'click-didyoumean': 'https://raw.githubusercontent.com/click-contrib/click-didyoumean/master/LICENSE', 'click-completion': 'https://raw.githubusercontent.com/click-contrib/click-completion/master/LICENSE', @@ -48,7 +50,9 @@ HARDCODED_LICENSE_URLS = { 'master/LICENSE', 'requirementslib': 'https://github.com/techalchemy/requirementslib/raw/master/LICENSE', 'distlib': 'https://github.com/vsajip/distlib/raw/master/LICENSE.txt', - 'pythonfinder': 'https://raw.githubusercontent.com/techalchemy/pythonfinder/master/LICENSE.txt' + 'pythonfinder': 'https://raw.githubusercontent.com/techalchemy/pythonfinder/master/LICENSE.txt', + 'pyparsing': 'https://raw.githubusercontent.com/pyparsing/pyparsing/master/LICENSE', + 'resolvelib': 'https://raw.githubusercontent.com/sarugaku/resolvelib/master/LICENSE' } FILE_WHITE_LIST = ( @@ -70,13 +74,14 @@ PATCHED_RENAMES = { LIBRARY_RENAMES = { 'pip': 'pipenv.patched.notpip', + "functools32": "pipenv.vendor.backports.functools_lru_cache", 'enum34': 'enum', } def drop_dir(path): if path.exists() and path.is_dir(): - shutil.rmtree(str(path)) + shutil.rmtree(str(path), ignore_errors=True) def remove_all(paths): @@ -110,8 +115,6 @@ def clean_vendor(ctx, vendor_dir): for item in vendor_dir.iterdir(): if item.is_dir(): shutil.rmtree(str(item)) - elif "LICENSE" in item.name or "COPYING" in item.name: - continue elif item.name not in FILE_WHITE_LIST: item.unlink() else: @@ -330,6 +333,7 @@ def post_install_cleanup(ctx, vendor_dir): # Cleanup setuptools unneeded parts drop_dir(vendor_dir / 'bin') drop_dir(vendor_dir / 'tests') + remove_all(vendor_dir.glob('toml.py')) def vendor(ctx, vendor_dir, package=None, rewrite=True): @@ -353,6 +357,10 @@ def vendor(ctx, vendor_dir, package=None, rewrite=True): log("Removing scandir library files...") remove_all(vendor_dir.glob('*.so')) + drop_dir(vendor_dir / 'setuptools') + drop_dir(vendor_dir / 'pkg_resources' / '_vendor') + drop_dir(vendor_dir / 'pkg_resources' / 'extern') + drop_dir(vendor_dir / 'bin') # Global import rewrites log('Renaming specified libs...') diff --git a/tasks/vendoring/patches/patched/_post-pip-update-pep425tags.patch b/tasks/vendoring/patches/patched/_post-pip-update-pep425tags.patch index 654a3da9..fce6ae89 100644 --- a/tasks/vendoring/patches/patched/_post-pip-update-pep425tags.patch +++ b/tasks/vendoring/patches/patched/_post-pip-update-pep425tags.patch @@ -1,20 +1,8 @@ diff --git a/pipenv/patched/notpip/_internal/pep425tags.py b/pipenv/patched/notpip/_internal/pep425tags.py -index c9804ab5..89c95c45 100644 +index f3b9b5b4..182c1c88 100644 --- a/pipenv/patched/notpip/_internal/pep425tags.py +++ b/pipenv/patched/notpip/_internal/pep425tags.py -@@ -10,10 +10,7 @@ import sysconfig - import warnings - from collections import OrderedDict - --try: -- import pipenv.patched.notpip._internal.utils.glibc --except ImportError: -- import pipenv.patched.notpip.utils.glibc -+import pipenv.patched.notpip._internal.utils.glibc - - logger = logging.getLogger(__name__) - -@@ -157,7 +154,7 @@ def is_manylinux1_compatible(): +@@ -159,7 +159,7 @@ def is_manylinux1_compatible(): pass # Check glibc version. CentOS 5 uses glibc 2.5. diff --git a/tasks/vendoring/patches/patched/contoml.patch b/tasks/vendoring/patches/patched/contoml.patch deleted file mode 100644 index b9b2e9d7..00000000 --- a/tasks/vendoring/patches/patched/contoml.patch +++ /dev/null @@ -1,28 +0,0 @@ -diff --git a/pipenv/patched/contoml/file/file.py b/pipenv/patched/contoml/file/file.py -index 5033a7b..99ce148 100644 ---- a/pipenv/patched/contoml/file/file.py -+++ b/pipenv/patched/contoml/file/file.py -@@ -30,6 +30,14 @@ class TOMLFile: - except KeyError: - return FreshTable(parent=self, name=item, is_array=False) - -+ def get(self, item, default=None): -+ """This was not here for who knows why.""" -+ -+ if item not in self: -+ return default -+ else: -+ return self.__getitem__(item) -+ - def __contains__(self, item): - return item in self.keys() - -@@ -223,7 +231,7 @@ class TOMLFile: - if has_anonymous_entry(): - return items - else: -- return items + [('', self[''])] -+ return list(items) + [('', self[''])] - - @property - def primitive(self): diff --git a/tasks/vendoring/patches/patched/pip18.patch b/tasks/vendoring/patches/patched/pip18.patch index 558c28e8..f4e607c1 100644 --- a/tasks/vendoring/patches/patched/pip18.patch +++ b/tasks/vendoring/patches/patched/pip18.patch @@ -1,5 +1,5 @@ diff --git a/pipenv/patched/pip/_internal/download.py b/pipenv/patched/pip/_internal/download.py -index 96f3b65c..3fb4ebef 100644 +index 96f3b65c..cc5b3d15 100644 --- a/pipenv/patched/pip/_internal/download.py +++ b/pipenv/patched/pip/_internal/download.py @@ -19,6 +19,7 @@ from pip._vendor.lockfile import LockError @@ -19,43 +19,78 @@ index 96f3b65c..3fb4ebef 100644 "python": platform.python_version(), "implementation": { "name": platform.python_implementation(), -@@ -322,7 +323,7 @@ class InsecureHTTPAdapter(HTTPAdapter): - conn.ca_certs = None +diff --git a/pipenv/patched/pip/_internal/utils/temp_dir.py b/pipenv/patched/pip/_internal/utils/temp_dir.py +index edc506bf..84d57dac 100644 +--- a/pipenv/patched/pip/_internal/utils/temp_dir.py ++++ b/pipenv/patched/pip/_internal/utils/temp_dir.py +@@ -3,8 +3,10 @@ from __future__ import absolute_import + import logging + import os.path + import tempfile ++import warnings + from pip._internal.utils.misc import rmtree ++from pipenv.vendor.vistir.compat import finalize, ResourceWarning --class PipSession(requests.Session): -+class PipSession(Session): + logger = logging.getLogger(__name__) - timeout = None - -@@ -752,7 +753,7 @@ def _copy_dist_from_dir(link_path, location): - - # build an sdist - setup_py = 'setup.py' -- sdist_args = [sys.executable] -+ sdist_args = [os.environ.get('PIP_PYTHON_PATH', sys.executable)] - sdist_args.append('-c') - sdist_args.append(SETUPTOOLS_SHIM % setup_py) - sdist_args.append('sdist') -diff --git a/pipenv/patched/pip/_internal/index.py b/pipenv/patched/pip/_internal/index.py -index 8c0ec82c..ad00ba04 100644 ---- a/pipenv/patched/pip/_internal/index.py -+++ b/pipenv/patched/pip/_internal/index.py -@@ -58,11 +58,12 @@ logger = logging.getLogger(__name__) - - class InstallationCandidate(object): - -- def __init__(self, project, version, location): -+ def __init__(self, project, version, location, requires_python=None): - self.project = project - self.version = parse_version(version) - self.location = location - self._key = (self.project, self.version, self.location) -+ self.requires_python = requires_python +@@ -45,6 +47,20 @@ class TempDirectory(object): + self.path = path + self.delete = delete + self.kind = kind ++ self._finalizer = None ++ if path: ++ self._register_finalizer() ++ ++ def _register_finalizer(self): ++ if self.delete and self.path: ++ self._finalizer = finalize( ++ self, ++ self._cleanup, ++ self.path, ++ warn_message=None ++ ) ++ else: ++ self._finalizer = None def __repr__(self): - return "".format( -@@ -168,6 +169,9 @@ class PackageFinder(object): + return "<{} {!r}>".format(self.__class__.__name__, self.path) +@@ -72,11 +88,27 @@ class TempDirectory(object): + self.path = os.path.realpath( + tempfile.mkdtemp(prefix="pip-{}-".format(self.kind)) + ) ++ self._register_finalizer() + logger.debug("Created temporary directory: {}".format(self.path)) + ++ @classmethod ++ def _cleanup(cls, name, warn_message=None): ++ try: ++ rmtree(name) ++ except OSError: ++ pass ++ else: ++ if warn_message: ++ warnings.warn(warn_message, ResourceWarning) ++ + def cleanup(self): + """Remove the temporary directory created and reset state + """ +- if self.path is not None and os.path.exists(self.path): +- rmtree(self.path) +- self.path = None ++ if getattr(self._finalizer, "detach", None) and self._finalizer.detach(): ++ if os.path.exists(self.path): ++ try: ++ rmtree(self.path) ++ except OSError: ++ pass ++ else: ++ self.path = None +diff --git a/pipenv/patched/pip/_internal/index.py b/pipenv/patched/pip/_internal/index.py +index 8c2f24f1..cdd48874 100644 +--- a/pipenv/patched/pip/_internal/index.py ++++ b/pipenv/patched/pip/_internal/index.py +@@ -246,6 +246,9 @@ class PackageFinder(object): # The Session we'll use to make requests self.session = session @@ -65,7 +100,7 @@ index 8c0ec82c..ad00ba04 100644 # The valid tags to check potential found wheel candidates against self.valid_tags = get_supported( versions=versions, -@@ -220,6 +224,24 @@ class PackageFinder(object): +@@ -298,6 +301,25 @@ class PackageFinder(object): ) self.dependency_links.extend(links) @@ -86,11 +121,12 @@ index 8c0ec82c..ad00ba04 100644 + current_list.append(link) + + return extras ++ + @staticmethod def _sort_locations(locations, expand_dir=False): """ -@@ -272,7 +294,7 @@ class PackageFinder(object): +@@ -350,7 +372,7 @@ class PackageFinder(object): return files, urls @@ -99,7 +135,7 @@ index 8c0ec82c..ad00ba04 100644 """ Function used to generate link sort key for link tuples. The greater the return value, the more preferred it is. -@@ -292,14 +314,19 @@ class PackageFinder(object): +@@ -370,14 +392,19 @@ class PackageFinder(object): if candidate.location.is_wheel: # can raise InvalidWheelFilename wheel = Wheel(candidate.location.filename) @@ -121,7 +157,19 @@ index 8c0ec82c..ad00ba04 100644 if wheel.build_tag is not None: match = re.match(r'^(\d+)(.*)$', wheel.build_tag) build_tag_groups = match.groups() -@@ -484,7 +511,7 @@ class PackageFinder(object): +@@ -528,7 +555,10 @@ class PackageFinder(object): + + page_versions = [] + for page in self._get_pages(url_locations, project_name): +- logger.debug('Analyzing links from page %s', page.url) ++ try: ++ logger.debug('Analyzing links from page %s', page.url) ++ except AttributeError: ++ continue + with indent_log(): + page_versions.extend( + self._package_versions(page.iter_links(), search) +@@ -562,7 +592,7 @@ class PackageFinder(object): dependency_versions ) @@ -130,19 +178,19 @@ index 8c0ec82c..ad00ba04 100644 """Try to find a Link matching req Expects req, an InstallRequirement and upgrade, a boolean -@@ -594,8 +621,9 @@ class PackageFinder(object): +@@ -672,7 +702,10 @@ class PackageFinder(object): continue seen.add(location) - page = self._get_page(location) -- if page is None: + try: + page = self._get_page(location) -+ except requests.HTTPError as e: ++ except requests.HTTPError: ++ continue + if page is None: continue - yield page -@@ -631,7 +659,7 @@ class PackageFinder(object): +@@ -709,7 +742,7 @@ class PackageFinder(object): logger.debug('Skipping link %s; %s', link, reason) self.logged_links.add(link) @@ -151,7 +199,7 @@ index 8c0ec82c..ad00ba04 100644 """Return an InstallationCandidate or None""" version = None if link.egg_fragment: -@@ -647,12 +675,12 @@ class PackageFinder(object): +@@ -725,12 +758,12 @@ class PackageFinder(object): link, 'unsupported archive format: %s' % ext, ) return @@ -166,7 +214,7 @@ index 8c0ec82c..ad00ba04 100644 self._log_skipped_link(link, 'macosx10 one') return if ext == wheel_ext: -@@ -666,7 +694,7 @@ class PackageFinder(object): +@@ -744,7 +777,7 @@ class PackageFinder(object): link, 'wrong project name (not %s)' % search.supplied) return @@ -175,7 +223,7 @@ index 8c0ec82c..ad00ba04 100644 self._log_skipped_link( link, 'it is not compatible with this Python') return -@@ -702,14 +730,14 @@ class PackageFinder(object): +@@ -780,14 +813,14 @@ class PackageFinder(object): link.filename, link.requires_python) support_this_python = True @@ -191,13 +239,30 @@ index 8c0ec82c..ad00ba04 100644 + return InstallationCandidate(search.supplied, version, link, link.requires_python) def _get_page(self, link): - return HTMLPage.get_page(link, session=self.session) + return _get_html_page(link, session=self.session) +diff --git a/pipenv/patched/pip/_internal/models/candidate.py b/pipenv/patched/pip/_internal/models/candidate.py +index c736de6c..a78566c1 100644 +--- a/pipenv/patched/pip/_internal/models/candidate.py ++++ b/pipenv/patched/pip/_internal/models/candidate.py +@@ -7,10 +7,11 @@ class InstallationCandidate(KeyBasedCompareMixin): + """Represents a potential "candidate" for installation. + """ + +- def __init__(self, project, version, location): ++ def __init__(self, project, version, location, requires_python=None): + self.project = project + self.version = parse_version(version) + self.location = location ++ self.requires_python = requires_python + + super(InstallationCandidate, self).__init__( + key=(self.project, self.version, self.location), diff --git a/pipenv/patched/pip/_internal/operations/prepare.py b/pipenv/patched/pip/_internal/operations/prepare.py -index 7740c284..b6e946d8 100644 +index 104bea33..ecf78b9a 100644 --- a/pipenv/patched/pip/_internal/operations/prepare.py +++ b/pipenv/patched/pip/_internal/operations/prepare.py @@ -17,7 +17,7 @@ from pip._internal.exceptions import ( - ) + from pip._internal.utils.compat import expanduser from pip._internal.utils.hashes import MissingHashes from pip._internal.utils.logging import indent_log -from pip._internal.utils.misc import display_path, normalize_path @@ -206,8 +271,8 @@ index 7740c284..b6e946d8 100644 logger = logging.getLogger(__name__) @@ -123,7 +123,11 @@ class IsSDist(DistAbstraction): - "Installing build dependencies" - ) + " and ".join(map(repr, sorted(missing))) + ) - self.req.run_egg_info() + try: @@ -237,10 +302,10 @@ index 7740c284..b6e946d8 100644 # We can't hit this spot and have populate_link return None. diff --git a/pipenv/patched/pip/_internal/pep425tags.py b/pipenv/patched/pip/_internal/pep425tags.py -index 0b5c7832..bea31585 100644 +index ab1a0298..763c0a24 100644 --- a/pipenv/patched/pip/_internal/pep425tags.py +++ b/pipenv/patched/pip/_internal/pep425tags.py -@@ -10,7 +10,10 @@ import sysconfig +@@ -10,7 +10,11 @@ import sysconfig import warnings from collections import OrderedDict @@ -249,81 +314,80 @@ index 0b5c7832..bea31585 100644 + import pip._internal.utils.glibc +except ImportError: + import pip.utils.glibc ++ + from pip._internal.utils.compat import get_extension_suffixes logger = logging.getLogger(__name__) - diff --git a/pipenv/patched/pip/_internal/req/req_install.py b/pipenv/patched/pip/_internal/req/req_install.py -index 462c80aa..d039adc8 100644 +index c2624fee..ee75acd6 100644 --- a/pipenv/patched/pip/_internal/req/req_install.py +++ b/pipenv/patched/pip/_internal/req/req_install.py -@@ -615,7 +615,7 @@ class InstallRequirement(object): +@@ -452,7 +452,8 @@ class InstallRequirement(object): with indent_log(): script = SETUPTOOLS_SHIM % self.setup_py - base_cmd = [sys.executable, '-c', script] -+ base_cmd = [os.environ.get('PIP_PYTHON_PATH', sys.executable), '-c', script] ++ sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) ++ base_cmd = [sys_executable, '-c', script] if self.isolated: base_cmd += ["--no-user-cfg"] egg_info_cmd = base_cmd + ['egg_info'] -@@ -797,7 +797,7 @@ class InstallRequirement(object): +@@ -613,10 +614,11 @@ class InstallRequirement(object): + + with indent_log(): + # FIXME: should we do --install-headers here too? ++ sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) with self.build_env: call_subprocess( [ - sys.executable, -+ os.environ.get('PIP_PYTHON_PATH', sys.executable), ++ sys_executable, '-c', SETUPTOOLS_SHIM % self.setup_py ] + -@@ -1015,7 +1015,7 @@ class InstallRequirement(object): +@@ -834,7 +836,8 @@ class InstallRequirement(object): def get_install_args(self, global_options, record_filename, root, prefix, pycompile): - install_args = [sys.executable, "-u"] -+ install_args = [os.environ.get('PIP_PYTHON_PATH', sys.executable), "-u"] ++ sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) ++ install_args = [sys_executable, "-u"] install_args.append('-c') install_args.append(SETUPTOOLS_SHIM % self.setup_py) install_args += list(global_options) + \ diff --git a/pipenv/patched/pip/_internal/req/req_set.py b/pipenv/patched/pip/_internal/req/req_set.py -index 2bc6b745..e552afc1 100644 +index b1983171..0bab231d 100644 --- a/pipenv/patched/pip/_internal/req/req_set.py +++ b/pipenv/patched/pip/_internal/req/req_set.py -@@ -12,7 +12,7 @@ logger = logging.getLogger(__name__) +@@ -12,13 +12,16 @@ logger = logging.getLogger(__name__) class RequirementSet(object): -- def __init__(self, require_hashes=False): -+ def __init__(self, require_hashes=False, ignore_compatibility=True): +- def __init__(self, require_hashes=False, check_supported_wheels=True): ++ def __init__(self, require_hashes=False, check_supported_wheels=True, ignore_compatibility=True): """Create a RequirementSet. """ -@@ -24,6 +24,7 @@ class RequirementSet(object): - self.unnamed_requirements = [] - self.successfully_downloaded = [] - self.reqs_to_cleanup = [] -+ self.ignore_compatibility = ignore_compatibility + self.requirements = OrderedDict() + self.require_hashes = require_hashes + self.check_supported_wheels = check_supported_wheels ++ if ignore_compatibility: ++ self.check_supported_wheels = False ++ self.ignore_compatibility = True if (check_supported_wheels is False or ignore_compatibility is True) else False - def __str__(self): - reqs = [req for req in self.requirements.values() -@@ -65,7 +66,7 @@ class RequirementSet(object): - # environment markers. - if install_req.link and install_req.link.is_wheel: - wheel = Wheel(install_req.link.filename) -- if not wheel.supported(): -+ if not wheel.supported() and not self.ignore_compatibility: - raise InstallationError( - "%s is not a supported wheel on this platform." % - wheel.filename -@@ -151,7 +152,7 @@ class RequirementSet(object): + # Mapping of alias: real_name + self.requirement_aliases = {} +@@ -171,7 +174,7 @@ class RequirementSet(object): return self.requirements[name] if name in self.requirement_aliases: return self.requirements[self.requirement_aliases[name]] - raise KeyError("No project with the name %r" % project_name) -+ # raise KeyError("No project with the name %r" % project_name) ++ pass def cleanup_files(self): """Clean up files, remove builds.""" diff --git a/pipenv/patched/pip/_internal/resolve.py b/pipenv/patched/pip/_internal/resolve.py -index 8480e48c..ffc4aa7d 100644 +index 2d9f1c56..bedc2582 100644 --- a/pipenv/patched/pip/_internal/resolve.py +++ b/pipenv/patched/pip/_internal/resolve.py @@ -35,7 +35,7 @@ class Resolver(object): @@ -356,22 +420,12 @@ index 8480e48c..ffc4aa7d 100644 """Prepare a single requirements file. :return: A list of additional InstallRequirements to also install. -@@ -245,6 +249,9 @@ class Resolver(object): - # Tell user what we are doing for this requirement: - # obtain (editable), skipping, processing (local url), collecting - # (remote url or package name) -+ if ignore_requires_python or self.ignore_requires_python: -+ self.ignore_compatibility = True -+ - if req_to_install.constraint or req_to_install.prepared: - return [] - -@@ -260,11 +267,17 @@ class Resolver(object): +@@ -260,11 +264,17 @@ class Resolver(object): try: check_dist_requires_python(dist) except UnsupportedPythonVersion as err: - if self.ignore_requires_python: -+ if self.ignore_compatibility: ++ if self.ignore_requires_python or self.ignore_compatibility: logger.warning(err.args[0]) else: raise @@ -385,14 +439,13 @@ index 8480e48c..ffc4aa7d 100644 more_reqs = [] def add_req(subreq, extras_requested): -@@ -290,10 +303,14 @@ class Resolver(object): - # We add req_to_install before its dependencies, so that we +@@ -291,9 +301,13 @@ class Resolver(object): # can refer to it when adding dependencies. if not requirement_set.has_requirement(req_to_install.name): + # 'unnamed' requirements will get added here + available_requested = sorted( + set(dist.extras) & set(req_to_install.extras) + ) - # 'unnamed' requirements will get added here req_to_install.is_direct = True requirement_set.add_requirement( req_to_install, parent_req_name=None, @@ -400,7 +453,7 @@ index 8480e48c..ffc4aa7d 100644 ) if not self.ignore_dependencies: -@@ -317,6 +334,19 @@ class Resolver(object): +@@ -317,6 +331,19 @@ class Resolver(object): for subreq in dist.requires(available_requested): add_req(subreq, extras_requested=available_requested) @@ -408,7 +461,7 @@ index 8480e48c..ffc4aa7d 100644 + for available in available_requested: + if hasattr(dist, '_DistInfoDistribution__dep_map'): + for req in dist._DistInfoDistribution__dep_map[available]: -+ req = InstallRequirement.from_req( ++ req = install_req_from_req( + str(req), + req_to_install, + isolated=self.isolated, @@ -420,24 +473,11 @@ index 8480e48c..ffc4aa7d 100644 if not req_to_install.editable and not req_to_install.satisfied_by: # XXX: --no-install leads this to report 'Successfully # downloaded' for only non-editable reqs, even though we took -diff --git a/pipenv/patched/pip/_internal/utils/misc.py b/pipenv/patched/pip/_internal/utils/misc.py -index 3236af63..439a831d 100644 ---- a/pipenv/patched/pip/_internal/utils/misc.py -+++ b/pipenv/patched/pip/_internal/utils/misc.py -@@ -96,7 +96,7 @@ def get_prog(): - try: - prog = os.path.basename(sys.argv[0]) - if prog in ('__main__.py', '-c'): -- return "%s -m pip" % sys.executable -+ return "%s -m pip" % os.environ.get('PIP_PYTHON_PATH', sys.executable) - else: - return prog - except (AttributeError, TypeError, IndexError): diff --git a/pipenv/patched/pip/_internal/utils/packaging.py b/pipenv/patched/pip/_internal/utils/packaging.py -index 5f9bb93d..276a9ccc 100644 +index c43142f0..f241cce0 100644 --- a/pipenv/patched/pip/_internal/utils/packaging.py +++ b/pipenv/patched/pip/_internal/utils/packaging.py -@@ -28,7 +28,7 @@ def check_requires_python(requires_python): +@@ -29,7 +29,7 @@ def check_requires_python(requires_python): requires_python_specifier = specifiers.SpecifierSet(requires_python) # We only use major.minor.micro @@ -446,33 +486,21 @@ index 5f9bb93d..276a9ccc 100644 return python_version in requires_python_specifier -@@ -40,20 +40,17 @@ def get_metadata(dist): - return dist.get_metadata('PKG-INFO') +@@ -48,9 +48,11 @@ def get_metadata(dist): + return feed_parser.close() -def check_dist_requires_python(dist): +def check_dist_requires_python(dist, absorb=True): - metadata = get_metadata(dist) - feed_parser = FeedParser() - feed_parser.feed(metadata) - pkg_info_dict = feed_parser.close() + pkg_info_dict = get_metadata(dist) requires_python = pkg_info_dict.get('Requires-Python') -+ if not absorb: ++ if absorb: + return requires_python try: if not check_requires_python(requires_python): -- raise exceptions.UnsupportedPythonVersion( -- "%s requires Python '%s' but the running Python is %s" % ( -- dist.project_name, -- requires_python, -- '.'.join(map(str, sys.version_info[:3])),) -- ) -+ return requires_python - except specifiers.InvalidSpecifier as e: - logger.warning( - "Package %s has an invalid Requires-Python entry %s - %s", + raise exceptions.UnsupportedPythonVersion( diff --git a/pipenv/patched/pip/_internal/wheel.py b/pipenv/patched/pip/_internal/wheel.py -index fcf9d3d3..d8aff848 100644 +index 5ce890eb..46c0181c 100644 --- a/pipenv/patched/pip/_internal/wheel.py +++ b/pipenv/patched/pip/_internal/wheel.py @@ -83,7 +83,7 @@ def fix_script(path): @@ -484,28 +512,24 @@ index fcf9d3d3..d8aff848 100644 firstline = b'#!' + exename + os.linesep.encode("ascii") rest = script.read() with open(path, 'wb') as script: -@@ -665,7 +665,7 @@ class WheelBuilder(object): +@@ -167,7 +167,8 @@ def message_about_scripts_not_on_PATH(scripts): + ] + # If an executable sits with sys.executable, we don't warn for it. + # This covers the case of venv invocations without activating the venv. +- not_warn_dirs.append(os.path.normcase(os.path.dirname(sys.executable))) ++ executable_loc = os.environ.get("PIP_PYTHON_PATH", sys.executable) ++ not_warn_dirs.append(os.path.normcase(os.path.dirname(executable_loc))) + warn_for = { + parent_dir: scripts for parent_dir, scripts in grouped_by_dir.items() + if os.path.normcase(parent_dir) not in not_warn_dirs +@@ -667,8 +668,9 @@ class WheelBuilder(object): + # isolating. Currently, it breaks Python in virtualenvs, because it # relies on site.py to find parts of the standard library outside the # virtualenv. ++ executable_loc = os.environ.get('PIP_PYTHON_PATH', sys.executable) return [ - sys.executable, '-u', '-c', -+ os.environ.get('PIP_PYTHON_PATH', sys.executable), '-u', '-c', ++ executable_loc, '-u', '-c', SETUPTOOLS_SHIM % req.setup_py ] + list(self.global_options) -diff --git a/pipenv/patched/pip/_internal/index.py b/pipenv/patched/pip/_internal/index.py -index 793dd1cb..426880e9 100644 ---- a/pipenv/patched/pip/_internal/index.py -+++ b/pipenv/patched/pip/_internal/index.py -@@ -477,7 +477,10 @@ class PackageFinder(object): - - page_versions = [] - for page in self._get_pages(url_locations, project_name): -- logger.debug('Analyzing links from page %s', page.url) -+ try: -+ logger.debug('Analyzing links from page %s', page.url) -+ except AttributeError: -+ continue - with indent_log(): - page_versions.extend( - self._package_versions(page.links, search) diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index 1db5ef44..3799ccf4 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -12,30 +12,56 @@ index 1fa3805..c0ecec8 100644 install_req_from_editable, ) diff --git a/pipenv/patched/piptools/_compat/pip_compat.py b/pipenv/patched/piptools/_compat/pip_compat.py -index 28da51f..de9b435 100644 +index 28da51f..c466ef0 100644 --- a/pipenv/patched/piptools/_compat/pip_compat.py +++ b/pipenv/patched/piptools/_compat/pip_compat.py -@@ -1,12 +1,13 @@ +@@ -1,45 +1,55 @@ # -*- coding=utf-8 -*- - import importlib +-import importlib -import pip -+from pip_shims import pip_version - import pkg_resources +-import pkg_resources ++__all__ = [ ++ "InstallRequirement", ++ "parse_requirements", ++ "RequirementSet", ++ "user_cache_dir", ++ "FAVORITE_HASH", ++ "is_file_url", ++ "url_to_path", ++ "PackageFinder", ++ "FormatControl", ++ "Wheel", ++ "Command", ++ "cmdoptions", ++ "get_installed_distributions", ++ "PyPI", ++ "SafeFileCache", ++ "InstallationError", ++ "parse_version", ++ "pip_version", ++ "install_req_from_editable", ++ "install_req_from_line", ++ "user_cache_dir" ++] -def do_import(module_path, subimport=None, old_path=None): -+def do_import(module_path, subimport=None, old_path=None, vendored_name=None): - old_path = old_path or module_path +- old_path = old_path or module_path - prefixes = ["pip._internal", "pip"] -+ prefix = vendored_name if vendored_name else "pip" -+ prefixes = ["{0}._internal".format(prefix), "{0}".format(prefix)] - paths = [module_path, old_path] - search_order = ["{0}.{1}".format(p, pth) for p in prefixes for pth in paths if pth is not None] - package = subimport if subimport else None -@@ -21,25 +22,28 @@ def do_import(module_path, subimport=None, old_path=None): - return getattr(imported, package) - - +- paths = [module_path, old_path] +- search_order = ["{0}.{1}".format(p, pth) for p in prefixes for pth in paths if pth is not None] +- package = subimport if subimport else None +- for to_import in search_order: +- if not subimport: +- to_import, _, package = to_import.rpartition(".") +- try: +- imported = importlib.import_module(to_import) +- except ImportError: +- continue +- else: +- return getattr(imported, package) +- +- -InstallRequirement = do_import('req.req_install', 'InstallRequirement') -parse_requirements = do_import('req.req_file', 'parse_requirements') -RequirementSet = do_import('req.req_set', 'RequirementSet') @@ -50,34 +76,38 @@ index 28da51f..de9b435 100644 -cmdoptions = do_import('cli.cmdoptions', old_path='cmdoptions') -get_installed_distributions = do_import('utils.misc', 'get_installed_distributions', old_path='utils') -PyPI = do_import('models.index', 'PyPI') -+InstallRequirement = do_import('req.req_install', 'InstallRequirement', vendored_name="notpip") -+parse_requirements = do_import('req.req_file', 'parse_requirements', vendored_name="notpip") -+RequirementSet = do_import('req.req_set', 'RequirementSet', vendored_name="notpip") -+user_cache_dir = do_import('utils.appdirs', 'user_cache_dir', vendored_name="notpip") -+FAVORITE_HASH = do_import('utils.hashes', 'FAVORITE_HASH', vendored_name="notpip") -+is_file_url = do_import('download', 'is_file_url', vendored_name="notpip") -+url_to_path = do_import('download', 'url_to_path', vendored_name="notpip") -+PackageFinder = do_import('index', 'PackageFinder', vendored_name="notpip") -+FormatControl = do_import('index', 'FormatControl', vendored_name="notpip") -+Wheel = do_import('wheel', 'Wheel', vendored_name="notpip") -+Command = do_import('cli.base_command', 'Command', old_path='basecommand', vendored_name="notpip") -+cmdoptions = do_import('cli.cmdoptions', old_path='cmdoptions', vendored_name="notpip") -+get_installed_distributions = do_import('utils.misc', 'get_installed_distributions', old_path='utils', vendored_name="notpip") -+PyPI = do_import('models.index', 'PyPI', vendored_name='notpip') -+SafeFileCache = do_import('download', 'SafeFileCache', vendored_name='notpip') -+InstallationError = do_import('exceptions', 'InstallationError', vendored_name='notpip') ++from pipenv.vendor.appdirs import user_cache_dir ++from pip_shims.shims import ( ++ InstallRequirement, ++ parse_requirements, ++ RequirementSet, ++ FAVORITE_HASH, ++ is_file_url, ++ url_to_path, ++ PackageFinder, ++ FormatControl, ++ Wheel, ++ Command, ++ cmdoptions, ++ get_installed_distributions, ++ PyPI, ++ SafeFileCache, ++ InstallationError, ++ parse_version, ++ pip_version, ++) # pip 18.1 has refactored InstallRequirement constructors use by pip-tools. -if pkg_resources.parse_version(pip.__version__) < pkg_resources.parse_version('18.1'): -+if pkg_resources.parse_version(pip_version) < pkg_resources.parse_version('18.1'): ++if parse_version(pip_version) < parse_version('18.1'): install_req_from_line = InstallRequirement.from_line install_req_from_editable = InstallRequirement.from_editable else: - install_req_from_line = do_import('req.constructors', 'install_req_from_line') - install_req_from_editable = do_import('req.constructors', 'install_req_from_editable') -+ install_req_from_line = do_import('req.constructors', 'install_req_from_line', vendored_name="notpip") -+ install_req_from_editable = do_import('req.constructors', 'install_req_from_editable', vendored_name="notpip") -+ ++ from pip_shims.shims import ( ++ install_req_from_editable, install_req_from_line ++ ) diff --git a/pipenv/patched/piptools/repositories/local.py b/pipenv/patched/piptools/repositories/local.py index 08dabe1..480ad1e 100644 --- a/pipenv/patched/piptools/repositories/local.py @@ -92,7 +122,7 @@ index 08dabe1..480ad1e 100644 else: return self.repository.find_best_match(ireq, prereleases) diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py -index bf69803..a1a3906 100644 +index bf69803..31b85b9 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -1,7 +1,7 @@ @@ -104,7 +134,7 @@ index bf69803..a1a3906 100644 import hashlib import os from contextlib import contextmanager -@@ -15,13 +15,23 @@ from .._compat import ( +@@ -15,13 +15,22 @@ from .._compat import ( Wheel, FAVORITE_HASH, TemporaryDirectory, @@ -113,11 +143,10 @@ index bf69803..a1a3906 100644 + InstallRequirement, + SafeFileCache ) -+os.environ["PIP_SHIMS_BASE_MODULE"] = str("notpip") ++os.environ["PIP_SHIMS_BASE_MODULE"] = str("pip") +from pip_shims.shims import do_import, VcsSupport, WheelCache +from packaging.requirements import Requirement +from packaging.specifiers import SpecifierSet, Specifier -+from packaging.markers import Op, Value, Variable, Marker +InstallationError = do_import(("exceptions.InstallationError", "7.0", "9999")) +from pip._internal.resolve import Resolver as PipResolver + @@ -128,11 +157,11 @@ index bf69803..a1a3906 100644 -from ..utils import (fs_str, is_pinned_requirement, lookup_table, - make_install_requirement) +from ..utils import (fs_str, is_pinned_requirement, lookup_table, dedup, -+ make_install_requirement, clean_requires_python) ++ make_install_requirement, clean_requires_python) from .base import BaseRepository try: -@@ -31,10 +41,44 @@ except ImportError: +@@ -31,10 +40,44 @@ except ImportError: def RequirementTracker(): yield @@ -181,7 +210,7 @@ index bf69803..a1a3906 100644 class PyPIRepository(BaseRepository): -@@ -46,8 +90,9 @@ class PyPIRepository(BaseRepository): +@@ -46,8 +89,9 @@ class PyPIRepository(BaseRepository): config), but any other PyPI mirror can be used if index_urls is changed/configured on the Finder. """ @@ -192,7 +221,7 @@ index bf69803..a1a3906 100644 self.pip_options = pip_options index_urls = [pip_options.index_url] + pip_options.extra_index_urls -@@ -73,6 +118,10 @@ class PyPIRepository(BaseRepository): +@@ -73,6 +117,10 @@ class PyPIRepository(BaseRepository): # of all secondary dependencies for the given requirement, so we # only have to go to disk once for each requirement self._dependencies_cache = {} @@ -203,7 +232,7 @@ index bf69803..a1a3906 100644 # Setup file paths self.freshen_build_caches() -@@ -113,10 +162,13 @@ class PyPIRepository(BaseRepository): +@@ -113,10 +161,13 @@ class PyPIRepository(BaseRepository): if ireq.editable: return ireq # return itself as the best match @@ -219,7 +248,7 @@ index bf69803..a1a3906 100644 # Reuses pip's internal candidate sort key to sort matching_candidates = [candidates_by_version[ver] for ver in matching_versions] -@@ -126,25 +178,86 @@ class PyPIRepository(BaseRepository): +@@ -126,25 +177,87 @@ class PyPIRepository(BaseRepository): # Turn the candidate into a pinned InstallRequirement return make_install_requirement( @@ -236,8 +265,7 @@ index bf69803..a1a3906 100644 + def gen(ireq): + if self.DEFAULT_INDEX_URL not in self.finder.index_urls: + return - -- def resolve_reqs(self, download_dir, ireq, wheel_cache): ++ + url = 'https://pypi.org/pypi/{0}/json'.format(ireq.req.name) + releases = self.session.get(url).json()['releases'] + @@ -266,7 +294,8 @@ index bf69803..a1a3906 100644 + try: + if ireq not in self._json_dep_cache: + self._json_dep_cache[ireq] = [g for g in gen(ireq)] -+ + +- def resolve_reqs(self, download_dir, ireq, wheel_cache): + return set(self._json_dep_cache[ireq]) + except Exception: + return set() @@ -291,6 +320,7 @@ index bf69803..a1a3906 100644 + dist = None + ireq.isolated = False + ireq._wheel_cache = wheel_cache ++ try: from pip._internal.operations.prepare import RequirementPreparer - from pip._internal.resolve import Resolver as PipResolver @@ -330,7 +360,7 @@ index bf69803..a1a3906 100644 } resolver = None preparer = None -@@ -177,15 +291,98 @@ class PyPIRepository(BaseRepository): +@@ -177,15 +291,109 @@ class PyPIRepository(BaseRepository): resolver_kwargs['preparer'] = preparer reqset = RequirementSet() ireq.is_direct = True @@ -339,9 +369,21 @@ index bf69803..a1a3906 100644 resolver = PipResolver(**resolver_kwargs) resolver.require_hashes = False results = resolver._resolve_one(reqset, ireq) - reqset.cleanup_files() +- reqset.cleanup_files() - return set(results) ++ cleanup_fn = getattr(reqset, "cleanup_files", None) ++ if cleanup_fn is not None: ++ try: ++ cleanup_fn() ++ except OSError: ++ pass ++ ++ if ireq.editable and (not ireq.source_dir or not os.path.exists(ireq.source_dir)): ++ if ireq.editable: ++ self._source_dir = TemporaryDirectory(fs_str("source")) ++ ireq.ensure_has_source_dir(self.source_dir) ++ + if ireq.editable and (ireq.source_dir and os.path.exists(ireq.source_dir)): + # Collect setup_requires info from local eggs. + # Do this after we call the preparer on these reqs to make sure their @@ -432,7 +474,7 @@ index bf69803..a1a3906 100644 """ Given a pinned or an editable InstallRequirement, returns a set of dependencies (also InstallRequirements, but not necessarily pinned). -@@ -200,6 +397,7 @@ class PyPIRepository(BaseRepository): +@@ -200,6 +408,7 @@ class PyPIRepository(BaseRepository): # If a download_dir is passed, pip will unnecessarely # archive the entire source directory download_dir = None @@ -440,7 +482,7 @@ index bf69803..a1a3906 100644 elif ireq.link and not ireq.link.is_artifact: # No download_dir for VCS sources. This also works around pip # using git-checkout-index, which gets rid of the .git dir. -@@ -214,7 +412,8 @@ class PyPIRepository(BaseRepository): +@@ -214,7 +423,8 @@ class PyPIRepository(BaseRepository): wheel_cache = WheelCache(CACHE_DIR, self.pip_options.format_control) prev_tracker = os.environ.get('PIP_REQ_TRACKER') try: @@ -450,7 +492,7 @@ index bf69803..a1a3906 100644 finally: if 'PIP_REQ_TRACKER' in os.environ: if prev_tracker: -@@ -236,6 +435,10 @@ class PyPIRepository(BaseRepository): +@@ -236,6 +446,10 @@ class PyPIRepository(BaseRepository): if ireq.editable: return set() @@ -461,7 +503,7 @@ index bf69803..a1a3906 100644 if not is_pinned_requirement(ireq): raise TypeError( "Expected pinned requirement, got {}".format(ireq)) -@@ -243,24 +446,22 @@ class PyPIRepository(BaseRepository): +@@ -243,24 +457,22 @@ class PyPIRepository(BaseRepository): # We need to get all of the candidates that match our current version # pin, these will represent all of the files that could possibly # satisfy this constraint. diff --git a/tasks/vendoring/patches/patched/prettytoml-newlinefix.patch b/tasks/vendoring/patches/patched/prettytoml-newlinefix.patch deleted file mode 100644 index 2b1066a1..00000000 --- a/tasks/vendoring/patches/patched/prettytoml-newlinefix.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/pipenv/patched/prettytoml/elements/traversal/__init__.py b/pipenv/patched/prettytoml/elements/traversal/__init__.py -index 5b98045..c93506e 100644 ---- a/pipenv/patched/prettytoml/elements/traversal/__init__.py -+++ b/pipenv/patched/prettytoml/elements/traversal/__init__.py -@@ -78,7 +78,7 @@ class TraversalMixin: - - if following_comment == float('-inf'): - return following_newline -- if following_newline == float('inf'): -+ if following_newline == float('-inf'): - return following_comment - - if following_newline < following_comment: diff --git a/tasks/vendoring/patches/patched/prettytoml-python37.patch b/tasks/vendoring/patches/patched/prettytoml-python37.patch deleted file mode 100644 index 5039a1c7..00000000 --- a/tasks/vendoring/patches/patched/prettytoml-python37.patch +++ /dev/null @@ -1,32 +0,0 @@ -From c44f2126fb5c75a5f5afd9d320c9f6cfc4ce3384 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= -Date: Tue, 26 Jun 2018 21:02:45 +0200 -Subject: [PATCH] Catch StopIteration in AbstractTable._enumerate_items - -This makes PEP 479 enabled Pythons (such as 3.7) work again. - -Otherwise you get: - - RuntimeError: generator raised StopIteration - -Fixes https://github.com/pypa/pipenv/issues/2426 ---- - pipenv/patched/prettytoml/elements/abstracttable.py | 5 ++++- - 1 file changed, 4 insertions(+), 1 deletion(-) - -diff --git a/pipenv/patched/prettytoml/elements/abstracttable.py b/pipenv/patched/prettytoml/elements/abstracttable.py -index 59fd574..627da0e 100644 ---- a/pipenv/patched/prettytoml/elements/abstracttable.py -+++ b/pipenv/patched/prettytoml/elements/abstracttable.py -@@ -19,7 +19,10 @@ def _enumerate_items(self): - """ - non_metadata = self._enumerate_non_metadata_sub_elements() - while True: -- yield next(non_metadata), next(non_metadata) -+ try: -+ yield next(non_metadata), next(non_metadata) -+ except StopIteration: -+ return - - def items(self): - for (key_i, key), (value_i, value) in self._enumerate_items(): diff --git a/tasks/vendoring/patches/patched/prettytoml-table-iter.patch b/tasks/vendoring/patches/patched/prettytoml-table-iter.patch deleted file mode 100644 index 9ec52633..00000000 --- a/tasks/vendoring/patches/patched/prettytoml-table-iter.patch +++ /dev/null @@ -1,29 +0,0 @@ -diff --git a/pipenv/patched/prettytoml/elements/abstracttable.py b/pipenv/patched/prettytoml/elements/abstracttable.py -index 59fd5748..48663aed 100644 ---- a/pipenv/patched/prettytoml/elements/abstracttable.py -+++ b/pipenv/patched/prettytoml/elements/abstracttable.py -@@ -1,8 +1,13 @@ -+try: -+ from collections.abc import Mapping -+except ImportError: -+ from collections import Mapping -+ - from prettytoml.elements.common import ContainerElement - from prettytoml.elements import traversal - - --class AbstractTable(ContainerElement, traversal.TraversalMixin): -+class AbstractTable(ContainerElement, traversal.TraversalMixin, Mapping): - """ - Common code for handling tables as key-value pairs with metadata elements sprinkled all over. - -@@ -37,6 +42,9 @@ class AbstractTable(ContainerElement, traversal.TraversalMixin): - def __len__(self): - return len(tuple(self._enumerate_items())) - -+ def __iter__(self): -+ return (key for key, _ in self.items()) -+ - def __contains__(self, item): - return item in self.keys() - diff --git a/tasks/vendoring/patches/patched/prettytoml.patch b/tasks/vendoring/patches/patched/prettytoml.patch deleted file mode 100644 index 85dfb791..00000000 --- a/tasks/vendoring/patches/patched/prettytoml.patch +++ /dev/null @@ -1,78 +0,0 @@ -diff --git a/pipenv/patched/prettytoml/_version.py b/pipenv/patched/prettytoml/_version.py -index 4f146e6..e0f1547 100644 ---- a/pipenv/patched/prettytoml/_version.py -+++ b/pipenv/patched/prettytoml/_version.py -@@ -1 +1 @@ --VERSION = '0.03' -+VERSION = 'master' -diff --git a/pipenv/patched/prettytoml/elements/table.py b/pipenv/patched/prettytoml/elements/table.py -index f78a6d1..cdc3ed4 100644 ---- a/pipenv/patched/prettytoml/elements/table.py -+++ b/pipenv/patched/prettytoml/elements/table.py -@@ -94,9 +94,9 @@ class TableElement(abstracttable.AbstractTable): - value_element, - factory.create_newline_element(), - ] -- -+ - insertion_index = self._find_insertion_index() -- -+ - self._sub_elements = \ - self.sub_elements[:insertion_index] + inserted_elements + self.sub_elements[insertion_index:] - -@@ -105,11 +105,16 @@ class TableElement(abstracttable.AbstractTable): - preceding_newline = self._find_preceding_newline(begin) - if preceding_newline >= 0: - begin = preceding_newline -- end = self._find_following_newline(begin) -+ end = self._find_following_line_terminator(begin) - if end < 0: - end = len(tuple(self._sub_elements)) - self._sub_elements = self.sub_elements[:begin] + self.sub_elements[end:] - -+ def pop(self, key): -+ v = self[key] -+ del self[key] -+ return v -+ - def value(self): - return self - -diff --git a/pipenv/patched/prettytoml/tokens/py2toml.py b/pipenv/patched/prettytoml/tokens/py2toml.py -index 3db97b4..8299195 100644 ---- a/pipenv/patched/prettytoml/tokens/py2toml.py -+++ b/pipenv/patched/prettytoml/tokens/py2toml.py -@@ -5,11 +5,8 @@ A converter of python values to TOML Token instances. - import codecs - import datetime - import six --import strict_rfc3339 --import timestamp - from prettytoml import tokens - import re --from prettytoml.elements.metadata import NewlineElement - from prettytoml.errors import TOMLError - from prettytoml.tokens import Token - from prettytoml.util import chunkate_string -@@ -49,15 +46,17 @@ def create_primitive_token(value, multiline_strings_allowed=True): - elif isinstance(value, float): - return tokens.Token(tokens.TYPE_FLOAT, u'{}'.format(value)) - elif isinstance(value, (datetime.datetime, datetime.date, datetime.time)): -- ts = timestamp(value) // 1000 -- return tokens.Token(tokens.TYPE_DATE, strict_rfc3339.timestamp_to_rfc3339_utcoffset(ts)) -+ s = value.isoformat() -+ if s.endswith('+00:00'): -+ s = s[:-6] + 'Z' -+ return tokens.Token(tokens.TYPE_DATE, s) - elif isinstance(value, six.string_types): - return create_string_token(value, multiline_strings_allowed=multiline_strings_allowed) - - raise NotPrimitiveError("{} of type {}".format(value, type(value))) - - --_bare_string_regex = re.compile('^[a-zA-Z0-9_-]*$') -+_bare_string_regex = re.compile('^[a-zA-Z_-]*$') - - - def create_string_token(text, bare_string_allowed=False, multiline_strings_allowed=True): diff --git a/tasks/vendoring/patches/vendor/delegator-close-filehandles.patch b/tasks/vendoring/patches/vendor/delegator-close-filehandles.patch new file mode 100644 index 00000000..092a6994 --- /dev/null +++ b/tasks/vendoring/patches/vendor/delegator-close-filehandles.patch @@ -0,0 +1,88 @@ +diff --git a/pipenv/vendor/delegator.py b/pipenv/vendor/delegator.py +index d15aeb97..cf6f91c8 100644 +--- a/pipenv/vendor/delegator.py ++++ b/pipenv/vendor/delegator.py +@@ -7,6 +7,8 @@ import locale + import errno + + from pexpect.popen_spawn import PopenSpawn ++import pexpect ++pexpect.EOF.__module__ = "pexpect.exceptions" + + # Include `unicode` in STR_TYPES for Python 2.X + try: +@@ -110,8 +112,11 @@ class Command(object): + if self.subprocess.before: + result += self.subprocess.before + +- if self.subprocess.after: +- result += self.subprocess.after ++ if self.subprocess.after and self.subprocess.after not in (pexpect.EOF, pexpect.TIMEOUT): ++ try: ++ result += self.subprocess.after ++ except (pexpect.EOF, pexpect.TIMEOUT): ++ pass + + result += self.subprocess.read() + return result +@@ -178,6 +183,7 @@ class Command(object): + # Use subprocess. + if self.blocking: + popen_kwargs = self._default_popen_kwargs.copy() ++ del popen_kwargs["stdin"] + popen_kwargs["universal_newlines"] = not binary + if cwd: + popen_kwargs["cwd"] = cwd +@@ -205,7 +211,10 @@ class Command(object): + if self.blocking: + raise RuntimeError("expect can only be used on non-blocking commands.") + +- self.subprocess.expect(pattern=pattern, timeout=timeout) ++ try: ++ self.subprocess.expect(pattern=pattern, timeout=timeout) ++ except pexpect.EOF: ++ pass + + def send(self, s, end=os.linesep, signal=False): + """Sends the given string or signal to std_in.""" +@@ -234,14 +243,25 @@ class Command(object): + """Blocks until process is complete.""" + if self._uses_subprocess: + # consume stdout and stderr +- try: +- stdout, stderr = self.subprocess.communicate() +- self.__out = stdout +- self.__err = stderr +- except ValueError: +- pass # Don't read from finished subprocesses. ++ if self.blocking: ++ try: ++ stdout, stderr = self.subprocess.communicate() ++ self.__out = stdout ++ self.__err = stderr ++ except ValueError: ++ pass # Don't read from finished subprocesses. ++ else: ++ self.subprocess.stdin.close() ++ self.std_out.close() ++ self.std_err.close() ++ self.subprocess.wait() + else: +- self.subprocess.wait() ++ self.subprocess.sendeof() ++ try: ++ self.subprocess.wait() ++ finally: ++ if self.subprocess.proc.stdout: ++ self.subprocess.proc.stdout.close() + + def pipe(self, command, timeout=None, cwd=None): + """Runs the current command and passes its output to the next +@@ -263,7 +283,6 @@ class Command(object): + c.run(block=False, cwd=cwd) + if data: + c.send(data) +- c.subprocess.sendeof() + c.block() + return c + diff --git a/tasks/vendoring/patches/vendor/passa-close-session.patch b/tasks/vendoring/patches/vendor/passa-close-session.patch new file mode 100644 index 00000000..38846bef --- /dev/null +++ b/tasks/vendoring/patches/vendor/passa-close-session.patch @@ -0,0 +1,12 @@ +diff --git a/pipenv/vendor/passa/internals/dependencies.py b/pipenv/vendor/passa/internals/dependencies.py +index 53b19b17..358cc33b 100644 +--- a/pipenv/vendor/passa/internals/dependencies.py ++++ b/pipenv/vendor/passa/internals/dependencies.py +@@ -154,6 +154,7 @@ def _get_dependencies_from_json(ireq, sources): + return dependencies + except Exception as e: + print("unable to read dependencies via {0} ({1})".format(url, e)) ++ session.close() + return + + diff --git a/tasks/vendoring/patches/vendor/tomlkit-typing-imports.patch b/tasks/vendoring/patches/vendor/tomlkit-fix.patch similarity index 50% rename from tasks/vendoring/patches/vendor/tomlkit-typing-imports.patch rename to tasks/vendoring/patches/vendor/tomlkit-fix.patch index 2288b513..36e2f808 100644 --- a/tasks/vendoring/patches/vendor/tomlkit-typing-imports.patch +++ b/tasks/vendoring/patches/vendor/tomlkit-fix.patch @@ -11,7 +11,7 @@ index e541c20c..0ac26752 100644 from .container import Container from .items import AoT diff --git a/pipenv/vendor/tomlkit/container.py b/pipenv/vendor/tomlkit/container.py -index c1d2d7c6..a7876ff1 100644 +index cb8af1d5..9b5db5cb 100644 --- a/pipenv/vendor/tomlkit/container.py +++ b/pipenv/vendor/tomlkit/container.py @@ -1,13 +1,5 @@ @@ -28,22 +28,43 @@ index c1d2d7c6..a7876ff1 100644 from ._compat import decode from .exceptions import KeyAlreadyPresent from .exceptions import NonExistentKey +@@ -17,6 +9,7 @@ from .items import Item + from .items import Key + from .items import Null + from .items import Table ++from .items import Trivia + from .items import Whitespace + from .items import item as _item + +@@ -221,7 +214,12 @@ class Container(dict): + for i in idx: + self._body[i] = (None, Null()) + else: +- self._body[idx] = (None, Null()) ++ old_data = self._body[idx][1] ++ trivia = getattr(old_data, "trivia", None) ++ if trivia and trivia.comment: ++ self._body[idx] = (None, Comment(Trivia(comment_ws="", comment=trivia.comment))) ++ else: ++ self._body[idx] = (None, Null()) + + super(Container, self).__delitem__(key.key) + diff --git a/pipenv/vendor/tomlkit/exceptions.py b/pipenv/vendor/tomlkit/exceptions.py -index 8d48bf19..d889a924 100644 +index 4fbc667b..c1a4e620 100644 --- a/pipenv/vendor/tomlkit/exceptions.py +++ b/pipenv/vendor/tomlkit/exceptions.py -@@ -1,6 +1,3 @@ +@@ -1,5 +1,3 @@ -from typing import Optional - -- + class TOMLKitError(Exception): - pass diff --git a/pipenv/vendor/tomlkit/items.py b/pipenv/vendor/tomlkit/items.py -index 747dbd50..8807f4b3 100644 +index 375b5f02..cccfd4a1 100644 --- a/pipenv/vendor/tomlkit/items.py +++ b/pipenv/vendor/tomlkit/items.py -@@ -6,14 +6,11 @@ import string +@@ -6,14 +6,6 @@ import string from datetime import date from datetime import datetime from datetime import time @@ -55,29 +76,65 @@ index 747dbd50..8807f4b3 100644 -from typing import Optional -from typing import Union - -+import sys -+if sys.version_info >= (3, 4): -+ from enum import Enum -+else: -+ from pipenv.vendor.backports.enum import Enum from ._compat import PY2 from ._compat import decode +@@ -22,9 +14,12 @@ from ._compat import unicode + from ._utils import escape_string + + if PY2: ++ from pipenv.vendor.backports.enum import Enum + from pipenv.vendor.backports.functools_lru_cache import lru_cache + else: ++ from enum import Enum + from functools import lru_cache ++from toml.decoder import InlineTableDict + + + def item(value, _parent=None): +@@ -40,7 +35,10 @@ def item(value, _parent=None): + elif isinstance(value, float): + return Float(value, Trivia(), str(value)) + elif isinstance(value, dict): +- val = Table(Container(), Trivia(), False) ++ if isinstance(value, InlineTableDict): ++ val = InlineTable(Container(), Trivia()) ++ else: ++ val = Table(Container(), Trivia(), False) + for k, v in sorted(value.items(), key=lambda i: (isinstance(i[1], dict), i[0])): + val[k] = item(v, _parent=val) + diff --git a/pipenv/vendor/tomlkit/parser.py b/pipenv/vendor/tomlkit/parser.py -index b55a3fe4..3d4984d1 100644 +index 7b948331..3f507bb4 100644 --- a/pipenv/vendor/tomlkit/parser.py +++ b/pipenv/vendor/tomlkit/parser.py -@@ -7,10 +7,6 @@ import re +@@ -4,13 +4,6 @@ from __future__ import unicode_literals + import re import string - from copy import copy --from typing import Iterator +-from typing import Any +-from typing import Generator +-from typing import List -from typing import Optional -from typing import Tuple -from typing import Union +- + from ._compat import chr + from ._compat import decode + from ._utils import _escaped +diff --git a/pipenv/vendor/tomlkit/source.py b/pipenv/vendor/tomlkit/source.py +index 1a96e058..dcfdafd0 100644 +--- a/pipenv/vendor/tomlkit/source.py ++++ b/pipenv/vendor/tomlkit/source.py +@@ -4,8 +4,6 @@ from __future__ import unicode_literals + import itertools + + from copy import copy +-from typing import Optional +-from typing import Tuple from ._compat import PY2 - from ._compat import chr + from ._compat import unicode diff --git a/pipenv/vendor/tomlkit/toml_file.py b/pipenv/vendor/tomlkit/toml_file.py index 3b416664..631e9959 100644 --- a/pipenv/vendor/tomlkit/toml_file.py diff --git a/tasks/vendoring/patches/vendor/vistir-imports.patch b/tasks/vendoring/patches/vendor/vistir-imports.patch index f93e7959..673efad8 100644 --- a/tasks/vendoring/patches/vendor/vistir-imports.patch +++ b/tasks/vendoring/patches/vendor/vistir-imports.patch @@ -1,25 +1,3 @@ -diff --git a/pipenv/vendor/vistir/compat.py b/pipenv/vendor/vistir/compat.py -index 1f1b7a96..0c865fe6 100644 ---- a/pipenv/vendor/vistir/compat.py -+++ b/pipenv/vendor/vistir/compat.py -@@ -30,7 +30,7 @@ else: - from pathlib2 import Path - - if sys.version_info < (3, 3): -- from backports.shutil_get_terminal_size import get_terminal_size -+ from pipenv.vendor.backports.shutil_get_terminal_size import get_terminal_size - from .backports.tempfile import NamedTemporaryFile - else: - from tempfile import NamedTemporaryFile -@@ -39,7 +39,7 @@ else: - try: - from weakref import finalize - except ImportError: -- from backports.weakref import finalize -+ from pipenv.vendor.backports.weakref import finalize - - try: - from functools import partialmethod diff --git a/pipenv/vendor/vistir/backports/tempfile.py b/pipenv/vendor/vistir/backports/tempfile.py index 483a479a..43470a6e 100644 --- a/pipenv/vendor/vistir/backports/tempfile.py @@ -33,3 +11,30 @@ index 483a479a..43470a6e 100644 __all__ = ["finalize", "NamedTemporaryFile"] +diff --git a/pipenv/vendor/vistir/compat.py b/pipenv/vendor/vistir/compat.py +index 9ae33fdc..ec3b65cb 100644 +--- a/pipenv/vendor/vistir/compat.py ++++ b/pipenv/vendor/vistir/compat.py +@@ -31,11 +31,11 @@ if sys.version_info >= (3, 5): + from functools import lru_cache + else: + from pathlib2 import Path +- from backports.functools_lru_cache import lru_cache ++ from pipenv.vendor.backports.functools_lru_cache import lru_cache + + from .backports.tempfile import NamedTemporaryFile as _NamedTemporaryFile + if sys.version_info < (3, 3): +- from backports.shutil_get_terminal_size import get_terminal_size ++ from pipenv.vendor.backports.shutil_get_terminal_size import get_terminal_size + NamedTemporaryFile = _NamedTemporaryFile + else: + from tempfile import NamedTemporaryFile +@@ -44,7 +44,7 @@ else: + try: + from weakref import finalize + except ImportError: +- from backports.weakref import finalize ++ from pipenv.vendor.backports.weakref import finalize + + try: + from functools import partialmethod diff --git a/tasks/vendoring/patches/vendor/yaspin-signal-handling.patch b/tasks/vendoring/patches/vendor/yaspin-signal-handling.patch new file mode 100644 index 00000000..a1d27cd3 --- /dev/null +++ b/tasks/vendoring/patches/vendor/yaspin-signal-handling.patch @@ -0,0 +1,62 @@ +diff --git a/pipenv/vendor/yaspin/core.py b/pipenv/vendor/yaspin/core.py +index d01fb98e..06b8b621 100644 +--- a/pipenv/vendor/yaspin/core.py ++++ b/pipenv/vendor/yaspin/core.py +@@ -16,6 +16,9 @@ import sys + import threading + import time + ++import colorama ++import cursor ++ + from .base_spinner import default_spinner + from .compat import PY2, basestring, builtin_str, bytes, iteritems, str + from .constants import COLOR_ATTRS, COLOR_MAP, ENCODING, SPINNER_ATTRS +@@ -23,6 +26,9 @@ from .helpers import to_unicode + from .termcolor import colored + + ++colorama.init() ++ ++ + class Yaspin(object): + """Implements a context manager that spawns a thread + to write spinner frames into a tty (stdout) during +@@ -369,11 +375,14 @@ class Yaspin(object): + # SIGKILL cannot be caught or ignored, and the receiving + # process cannot perform any clean-up upon receiving this + # signal. +- if signal.SIGKILL in self._sigmap.keys(): +- raise ValueError( +- "Trying to set handler for SIGKILL signal. " +- "SIGKILL cannot be cought or ignored in POSIX systems." +- ) ++ try: ++ if signal.SIGKILL in self._sigmap.keys(): ++ raise ValueError( ++ "Trying to set handler for SIGKILL signal. " ++ "SIGKILL cannot be cought or ignored in POSIX systems." ++ ) ++ except AttributeError: ++ pass + + for sig, sig_handler in iteritems(self._sigmap): + # A handler for a particular signal, once set, remains +@@ -521,14 +530,12 @@ class Yaspin(object): + + @staticmethod + def _hide_cursor(): +- sys.stdout.write("\033[?25l") +- sys.stdout.flush() ++ cursor.hide() + + @staticmethod + def _show_cursor(): +- sys.stdout.write("\033[?25h") +- sys.stdout.flush() ++ cursor.show() + + @staticmethod + def _clear_line(): +- sys.stdout.write("\033[K") ++ sys.stdout.write(chr(27) + "[K") diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 12f27342..6ed95b3e 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,5 +1,6 @@ import json import os +import sys import warnings import pytest @@ -7,13 +8,13 @@ import pytest from pipenv._compat import TemporaryDirectory, Path from pipenv.vendor import delegator from pipenv.vendor import requests -from pipenv.vendor import six from pipenv.vendor import toml from pytest_pypi.app import prepare_packages as prepare_pypi_packages +from vistir.compat import ResourceWarning, fs_str +from vistir.path import mkdir_p -if six.PY2: - class ResourceWarning(Warning): - pass + +warnings.simplefilter("default", category=ResourceWarning) HAS_WARNED_GITHUB = False @@ -25,8 +26,8 @@ def check_internet(): resp = requests.get('http://httpbin.org/ip', timeout=1.0) resp.raise_for_status() except Exception: - warnings.warn('Cannot connect to HTTPBin...', ResourceWarning) - warnings.warn('Will skip tests requiring Internet', ResourceWarning) + warnings.warn('Cannot connect to HTTPBin...', RuntimeWarning) + warnings.warn('Will skip tests requiring Internet', RuntimeWarning) return False return True @@ -46,20 +47,26 @@ def check_github_ssh(): global HAS_WARNED_GITHUB if not res and not HAS_WARNED_GITHUB: warnings.warn( - 'Cannot connect to GitHub via SSH', ResourceWarning + 'Cannot connect to GitHub via SSH', RuntimeWarning ) warnings.warn( - 'Will skip tests requiring SSH access to GitHub', ResourceWarning + 'Will skip tests requiring SSH access to GitHub', RuntimeWarning ) HAS_WARNED_GITHUB = True return res -WE_HAVE_INTERNET = check_internet() -WE_HAVE_GITHUB_SSH_KEYS = check_github_ssh() +def check_for_mercurial(): + c = delegator.run("hg --help") + if c.return_code != 0: + return False + else: + return True + TESTS_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) PYPI_VENDOR_DIR = os.path.join(TESTS_ROOT, 'pypi') +WE_HAVE_HG = check_for_mercurial() prepare_pypi_packages(PYPI_VENDOR_DIR) @@ -68,26 +75,77 @@ def pytest_runtest_setup(item): pytest.skip('requires internet') if item.get_marker('needs_github_ssh') is not None and not WE_HAVE_GITHUB_SSH_KEYS: pytest.skip('requires github ssh') + if item.get_marker('needs_hg') is not None and not WE_HAVE_HG: + pytest.skip('requires mercurial') + + +@pytest.fixture +def pathlib_tmpdir(request, tmpdir): + yield Path(str(tmpdir)) + try: + tmpdir.remove(ignore_errors=True) + except Exception: + pass + + +# Borrowed from pip's test runner filesystem isolation +@pytest.fixture(autouse=True) +def isolate(pathlib_tmpdir): + """ + Isolate our tests so that things like global configuration files and the + like do not affect our test results. + We use an autouse function scoped fixture because we want to ensure that + every test has it's own isolated home directory. + """ + + # Create a directory to use as our home location. + home_dir = os.path.join(str(pathlib_tmpdir), "home") + os.makedirs(home_dir) + mkdir_p(os.path.join(home_dir, ".config", "git")) + with open(os.path.join(home_dir, ".config", "git", "config"), "wb") as fp: + fp.write( + b"[user]\n\tname = pipenv\n\temail = pipenv@pipenv.org\n" + ) + os.environ["GIT_CONFIG_NOSYSTEM"] = fs_str("1") + os.environ["GIT_AUTHOR_NAME"] = fs_str("pipenv") + os.environ["GIT_AUTHOR_EMAIL"] = fs_str("pipenv@pipenv.org") + mkdir_p(os.path.join(home_dir, ".virtualenvs")) + os.environ["WORKON_HOME"] = fs_str(os.path.join(home_dir, ".virtualenvs")) + global WE_HAVE_GITHUB_SSH_KEYS + WE_HAVE_GITHUB_SSH_KEYS = check_github_ssh() + + +WE_HAVE_INTERNET = check_internet() +WE_HAVE_GITHUB_SSH_KEYS = check_github_ssh() class _PipenvInstance(object): """An instance of a Pipenv Project...""" - def __init__(self, pypi=None, pipfile=True, chdir=False): + def __init__(self, pypi=None, pipfile=True, chdir=False, path=None, home_dir=None): self.pypi = pypi self.original_umask = os.umask(0o007) self.original_dir = os.path.abspath(os.curdir) - self._path = TemporaryDirectory(suffix='-project', prefix='pipenv-') - path = Path(self._path.name) - try: - self.path = str(path.resolve()) - except OSError: - self.path = str(path.absolute()) + os.environ["PIPENV_NOSPIN"] = fs_str("1") + os.environ["CI"] = fs_str("1") + warnings.simplefilter("ignore", category=ResourceWarning) + warnings.filterwarnings("ignore", category=ResourceWarning, message="unclosed.*") + path = os.environ.get("PIPENV_PROJECT_DIR", None) + if not path: + self._path = TemporaryDirectory(suffix='-project', prefix='pipenv-') + path = Path(self._path.name) + try: + self.path = str(path.resolve()) + except OSError: + self.path = str(path.absolute()) + else: + self._path = None + self.path = path # set file creation perms self.pipfile_path = None self.chdir = chdir if self.pypi: - os.environ['PIPENV_TEST_INDEX'] = '{0}/simple'.format(self.pypi.url) + os.environ['PIPENV_TEST_INDEX'] = fs_str('{0}/simple'.format(self.pypi.url)) if pipfile: p_path = os.sep.join([self.path, 'Pipfile']) @@ -98,9 +156,10 @@ class _PipenvInstance(object): self.pipfile_path = p_path def __enter__(self): - os.environ['PIPENV_DONT_USE_PYENV'] = '1' - os.environ['PIPENV_IGNORE_VIRTUALENVS'] = '1' - os.environ['PIPENV_VENV_IN_PROJECT'] = '1' + os.environ['PIPENV_DONT_USE_PYENV'] = fs_str('1') + os.environ['PIPENV_IGNORE_VIRTUALENVS'] = fs_str('1') + os.environ['PIPENV_VENV_IN_PROJECT'] = fs_str('1') + os.environ['PIPENV_NOSPIN'] = fs_str('1') if self.chdir: os.chdir(self.path) return self @@ -110,21 +169,21 @@ class _PipenvInstance(object): if self.chdir: os.chdir(self.original_dir) self.path = None - try: - self._path.cleanup() - except OSError as e: - _warn_msg = warn_msg.format(e) - warnings.warn(_warn_msg, ResourceWarning) - finally: - os.umask(self.original_umask) + if self._path: + try: + self._path.cleanup() + except OSError as e: + _warn_msg = warn_msg.format(e) + warnings.warn(_warn_msg, ResourceWarning) + os.umask(self.original_umask) def pipenv(self, cmd, block=True): if self.pipfile_path: - os.environ['PIPENV_PIPFILE'] = self.pipfile_path + os.environ['PIPENV_PIPFILE'] = fs_str(self.pipfile_path) # a bit of a hack to make sure the virtualenv is created with TemporaryDirectory(prefix='pipenv-', suffix='-cache') as tempdir: - os.environ['PIPENV_CACHE_DIR'] = tempdir.name + os.environ['PIPENV_CACHE_DIR'] = fs_str(tempdir.name) c = delegator.run('pipenv {0}'.format(cmd), block=block) if 'PIPENV_CACHE_DIR' in os.environ: del os.environ['PIPENV_CACHE_DIR'] @@ -162,18 +221,16 @@ class _PipenvInstance(object): @pytest.fixture() def PipenvInstance(): - return _PipenvInstance + yield _PipenvInstance -@pytest.fixture(scope='module') -def pip_src_dir(request): +@pytest.fixture(autouse=True) +def pip_src_dir(request, pathlib_tmpdir): old_src_dir = os.environ.get('PIP_SRC', '') - new_src_dir = TemporaryDirectory(prefix='pipenv-', suffix='-testsrc') - os.environ['PIP_SRC'] = new_src_dir.name + os.environ['PIP_SRC'] = pathlib_tmpdir.as_posix() def finalize(): - new_src_dir.cleanup() - os.environ['PIP_SRC'] = old_src_dir + os.environ['PIP_SRC'] = fs_str(old_src_dir) request.addfinalizer(finalize) return request diff --git a/tests/integration/test_install_twists.py b/tests/integration/test_install_twists.py index 055c39b2..2ad12691 100644 --- a/tests/integration/test_install_twists.py +++ b/tests/integration/test_install_twists.py @@ -333,3 +333,54 @@ six = {{path = "./artifacts/{}"}} c = p.pipenv("install") assert c.return_code == 0 assert "six" in p.lockfile["default"] + + +@pytest.mark.files +@pytest.mark.install +@pytest.mark.run +def test_multiple_editable_packages_should_not_race(PipenvInstance, pypi, tmpdir, testsroot): + """Test for a race condition that can occur when installing multiple 'editable' packages at + once, and which causes some of them to not be importable. + + This issue had been fixed for VCS packages already, but not local 'editable' packages. + + So this test locally installs packages from tarballs that have already been committed in + the local `pypi` dir to avoid using VCS packages. + """ + pkgs = { + "requests-2.19.1": "requests/requests-2.19.1.tar.gz", + "Flask-0.12.2": "flask/Flask-0.12.2.tar.gz", + "six-1.11.0": "six/six-1.11.0.tar.gz", + "Jinja2-2.10": "jinja2/Jinja2-2.10.tar.gz", + } + + pipfile_string=""" +[packages] +""" + # Unzip tarballs to known location, and update Pipfile template. + for pkg_name, file_name in pkgs.items(): + source_path = str(Path(testsroot, "pypi", file_name)) + unzip_path = str(Path(tmpdir.strpath, pkg_name)) + + import tarfile + + with tarfile.open(source_path, "r:gz") as tgz: + tgz.extractall(path=tmpdir.strpath) + + pipfile_string += "'{0}' = {{path = '{1}', editable = true}}\n".format(pkg_name, unzip_path) + + with PipenvInstance(pypi=pypi, chdir=True) as p: + with open(p.pipfile_path, 'w') as f: + f.write(pipfile_string.strip()) + + c = p.pipenv('install') + assert c.return_code == 0 + + c = p.pipenv('run python -c "import requests"') + assert c.return_code == 0 + c = p.pipenv('run python -c "import flask"') + assert c.return_code == 0 + c = p.pipenv('run python -c "import six"') + assert c.return_code == 0 + c = p.pipenv('run python -c "import jinja2"') + assert c.return_code == 0 diff --git a/tests/integration/test_lock.py b/tests/integration/test_lock.py index 1f1719d0..f2f0ac76 100644 --- a/tests/integration/test_lock.py +++ b/tests/integration/test_lock.py @@ -37,9 +37,9 @@ flask = "==0.12.2" """.strip() f.write(contents) - req_list = ("requests==2.14.0") + req_list = ("requests==2.14.0",) - dev_req_list = ("flask==0.12.2") + dev_req_list = ("flask==0.12.2",) c = p.pipenv('lock -r') d = p.pipenv('lock -r -d') @@ -369,6 +369,42 @@ requests = {git = "https://github.com/requests/requests.git", ref = "master", ed assert c.return_code == 0 +@pytest.mark.lock +@pytest.mark.vcs +@pytest.mark.needs_internet +def test_lock_editable_vcs_with_ref_in_git(PipenvInstance, pypi): + with PipenvInstance(pypi=pypi, chdir=True) as p: + with open(p.pipfile_path, 'w') as f: + f.write(""" +[packages] +requests = {git = "https://github.com/requests/requests.git@883caaf", editable = true} + """.strip()) + c = p.pipenv('lock') + assert c.return_code == 0 + assert p.lockfile['default']['requests']['git'] == 'https://github.com/requests/requests.git' + assert p.lockfile['default']['requests']['ref'] == '883caaf145fbe93bd0d208a6b864de9146087312' + c = p.pipenv('install') + assert c.return_code == 0 + + +@pytest.mark.lock +@pytest.mark.vcs +@pytest.mark.needs_internet +def test_lock_editable_vcs_with_ref(PipenvInstance, pypi): + with PipenvInstance(pypi=pypi, chdir=True) as p: + with open(p.pipfile_path, 'w') as f: + f.write(""" +[packages] +requests = {git = "https://github.com/requests/requests.git", ref = "883caaf", editable = true} + """.strip()) + c = p.pipenv('lock') + assert c.return_code == 0 + assert p.lockfile['default']['requests']['git'] == 'https://github.com/requests/requests.git' + assert p.lockfile['default']['requests']['ref'] == '883caaf145fbe93bd0d208a6b864de9146087312' + c = p.pipenv('install') + assert c.return_code == 0 + + @pytest.mark.extras @pytest.mark.lock @pytest.mark.vcs @@ -450,3 +486,20 @@ def test_lockfile_with_empty_dict(PipenvInstance): assert c.return_code == 0 assert 'Pipfile.lock is corrupted' in c.err assert p.lockfile['_meta'] + + +@pytest.mark.lock +@pytest.mark.install +def test_lock_with_incomplete_source(PipenvInstance, pypi): + with PipenvInstance(pypi=pypi, chdir=True) as p: + with open(p.pipfile_path, 'w') as f: + f.write(""" +[[source]] +url = "https://test.pypi.org/simple" + +[packages] +requests = "*" + """) + c = p.pipenv('install') + assert c.return_code == 0 + assert p.lockfile['_meta']['sources'] diff --git a/tests/integration/test_project.py b/tests/integration/test_project.py index 5deccc84..5e1bafb9 100644 --- a/tests/integration/test_project.py +++ b/tests/integration/test_project.py @@ -143,3 +143,21 @@ six = {{version = "*", index = "pypi"}} f.write(contents) c = p.pipenv('install') assert c.return_code == 0 + + +@pytest.mark.install +@pytest.mark.project +def test_rewrite_outline_table(PipenvInstance, pypi): + with PipenvInstance(pypi=pypi, chdir=True) as p: + with open(p.pipfile_path, 'w') as f: + contents = """ +[packages.requests] +version = "*" + """.strip() + f.write(contents) + c = p.pipenv('install click') + assert c.return_code == 0 + with open(p.pipfile_path) as f: + contents = f.read() + assert "[packages.requests]" not in contents + assert 'requests = {version = "*"}' in contents diff --git a/tests/integration/test_sync.py b/tests/integration/test_sync.py index c50e259b..2ef06ddc 100644 --- a/tests/integration/test_sync.py +++ b/tests/integration/test_sync.py @@ -15,7 +15,7 @@ def test_sync_error_without_lockfile(PipenvInstance, pypi): c = p.pipenv('sync') assert c.return_code != 0 - assert 'Pipfile.lock is missing!' in c.err + assert 'Pipfile.lock not found!' in c.err @pytest.mark.sync diff --git a/tests/integration/test_uninstall.py b/tests/integration/test_uninstall.py index e19a1400..5f493cac 100644 --- a/tests/integration/test_uninstall.py +++ b/tests/integration/test_uninstall.py @@ -84,7 +84,7 @@ def test_uninstall_all_local_files(PipenvInstance, testsroot): # Not sure where travis/appveyor run tests from source_path = os.path.abspath(os.path.join(testsroot, "test_artifacts", file_name)) - with PipenvInstance() as p: + with PipenvInstance(chdir=True) as p: shutil.copy(source_path, os.path.join(p.path, file_name)) os.mkdir(os.path.join(p.path, "requests")) c = p.pipenv("install {}".format(file_name)) @@ -92,7 +92,9 @@ def test_uninstall_all_local_files(PipenvInstance, testsroot): c = p.pipenv("uninstall --all") assert c.return_code == 0 assert "requests" in c.out - assert "requests" not in p.pipfile["packages"] + # Uninstall --all is not supposed to remove things from the pipfile + # Note that it didn't before, but that instead local filenames showed as hashes + assert "requests" in p.pipfile["packages"] @pytest.mark.run diff --git a/tests/integration/test_windows.py b/tests/integration/test_windows.py index 128a6b5d..da138f7a 100644 --- a/tests/integration/test_windows.py +++ b/tests/integration/test_windows.py @@ -65,3 +65,16 @@ def test_local_path_windows_forward_slash(PipenvInstance, pypi): with PipenvInstance(pypi=pypi, chdir=True) as p: c = p.pipenv('install "{0}"'.format(whl.as_posix())) assert c.return_code == 0 + + +@pytest.mark.cli +def test_pipenv_clean_windows(PipenvInstance, pypi): + with PipenvInstance(pypi=pypi, chdir=True) as p: + c = p.pipenv('install requests') + assert c.return_code == 0 + c = p.pipenv('run pip install click') + assert c.return_code == 0 + + c = p.pipenv('clean --dry-run') + assert c.return_code == 0 + assert 'click' in c.out.strip() diff --git a/tests/pytest-pypi/Pipfile b/tests/pytest-pypi/Pipfile deleted file mode 100644 index 6abe710c..00000000 --- a/tests/pytest-pypi/Pipfile +++ /dev/null @@ -1,23 +0,0 @@ -[[source]] - -url = "https://pypi.org/simple" -verify_ssl = true -name = "pypi" - - -[packages] - - - -[dev-packages] - -pytest = "*" -"e1839a8" = {path = ".", editable = true} -httpbin = "*" -requests = "*" -twine = "*" - - -[requires] - -python_version = "3.6" diff --git a/tests/pytest-pypi/Pipfile.lock b/tests/pytest-pypi/Pipfile.lock deleted file mode 100644 index 25978b9d..00000000 --- a/tests/pytest-pypi/Pipfile.lock +++ /dev/null @@ -1,480 +0,0 @@ -{ - "_meta": { - "hash": { - "sha256": "b871e70f9f0b7bd3852672c371c754d8db50e5015c8dbdad1d217a3e58a26c33" - }, - "pipfile-spec": 6, - "requires": { - "python_version": "3.6" - }, - "sources": [ - { - "name": "pypi", - "url": "https://pypi.org/simple", - "verify_ssl": true - } - ] - }, - "default": {}, - "develop": { - "argparse": { - "hashes": [ - "sha256:62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4", - "sha256:c31647edb69fd3d465a847ea3157d37bed1f95f19760b11a47aa91c04b666314" - ], - "markers": "python_version == '2.6'", - "version": "==1.4.0" - }, - "attrs": { - "hashes": [ - "sha256:1c7960ccfd6a005cd9f7ba884e6316b5e430a3f1a6c37c5f87d8b43f83b54ec9", - "sha256:a17a9573a6f475c99b551c0e0a812707ddda1ec9653bed04c13841404ed6f450" - ], - "version": "==17.4.0" - }, - "blinker": { - "hashes": [ - "sha256:471aee25f3992bd325afa3772f1063dbdbbca947a041b8b89466dc00d606f8b6" - ], - "version": "==1.4" - }, - "brotlipy": { - "hashes": [ - "sha256:07194f4768eb62a4f4ea76b6d0df6ade185e24ebd85877c351daa0a069f1111a", - "sha256:091b299bf36dd6ef7a06570dbc98c0f80a504a56c5b797f31934d2ad01ae7d17", - "sha256:09ec3e125d16749b31c74f021aba809541b3564e5359f8c265cbae442810b41a", - "sha256:0be698678a114addcf87a4b9496c552c68a2c99bf93cf8e08f5738b392e82057", - "sha256:0fa6088a9a87645d43d7e21e32b4a6bf8f7c3939015a50158c10972aa7f425b7", - "sha256:1ea4e578241504b58f2456a6c69952c88866c794648bdc74baee74839da61d44", - "sha256:2699945a0a992c04fc7dc7fa2f1d0575a2c8b4b769f2874a08e8eae46bef36ae", - "sha256:2a80319ae13ea8dd60ecdc4f5ccf6da3ae64787765923256b62c598c5bba4121", - "sha256:2e5c64522364a9ebcdf47c5744a5ddeb3f934742d31e61ebfbbc095460b47162", - "sha256:36def0b859beaf21910157b4c33eb3b06d8ce459c942102f16988cca6ea164df", - "sha256:3a3e56ced8b15fbbd363380344f70f3b438e0fd1fcf27b7526b6172ea950e867", - "sha256:3c1d5e2cf945a46975bdb11a19257fa057b67591eb232f393d260e7246d9e571", - "sha256:50ca336374131cfad20612f26cc43c637ac0bfd2be3361495e99270883b52962", - "sha256:5de6f7d010b7558f72f4b061a07395c5c3fd57f0285c5af7f126a677b976a868", - "sha256:637847560d671657f993313ecc6c6c6666a936b7a925779fd044065c7bc035b9", - "sha256:653faef61241bf8bf99d73ca7ec4baa63401ba7b2a2aa88958394869379d67c7", - "sha256:786afc8c9bd67de8d31f46e408a3386331e126829114e4db034f91eacb05396d", - "sha256:79aaf217072840f3e9a3b641cccc51f7fc23037496bd71e26211856b93f4b4cb", - "sha256:7e31f7adcc5851ca06134705fcf3478210da45d35ad75ec181e1ce9ce345bb38", - "sha256:8b39abc3256c978f575df5cd7893153277216474f303e26f0e43ba3d3969ef96", - "sha256:9448227b0df082e574c45c983fa5cd4bda7bfb11ea6b59def0940c1647be0c3c", - "sha256:96bc59ff9b5b5552843dc67999486a220e07a0522dddd3935da05dc194fa485c", - "sha256:a07647886e24e2fb2d68ca8bf3ada398eb56fd8eac46c733d4d95c64d17f743b", - "sha256:af65d2699cb9f13b26ec3ba09e75e80d31ff422c03675fcb36ee4dabe588fdc2", - "sha256:b4c98b0d2c9c7020a524ca5bbff42027db1004c6571f8bc7b747f2b843128e7a", - "sha256:c6cc0036b1304dd0073eec416cb2f6b9e37ac8296afd9e481cac3b1f07f9db25", - "sha256:d2c1c724c4ac375feb2110f1af98ecdc0e5a8ea79d068efb5891f621a5b235cb", - "sha256:dc6c5ee0df9732a44d08edab32f8a616b769cc5a4155a12d2d010d248eb3fb07", - "sha256:fd1d1c64214af5d90014d82cee5d8141b13d44c92ada7a0c0ec0679c6f15a471" - ], - "version": "==0.7.0" - }, - "certifi": { - "hashes": [ - "sha256:14131608ad2fd56836d33a71ee60fa1c82bc9d2c8d98b7bdbc631fe1b3cd1296", - "sha256:edbc3f203427eef571f79a7692bb160a2b0f7ccaa31953e99bd17e307cf63f7d" - ], - "version": "==2018.1.18" - }, - "cffi": { - "hashes": [ - "sha256:151b7eefd035c56b2b2e1eb9963c90c6302dc15fbd8c1c0a83a163ff2c7d7743", - "sha256:1553d1e99f035ace1c0544050622b7bc963374a00c467edafac50ad7bd276aef", - "sha256:1b0493c091a1898f1136e3f4f991a784437fac3673780ff9de3bcf46c80b6b50", - "sha256:2ba8a45822b7aee805ab49abfe7eec16b90587f7f26df20c71dd89e45a97076f", - "sha256:3c85641778460581c42924384f5e68076d724ceac0f267d66c757f7535069c93", - "sha256:3eb6434197633b7748cea30bf0ba9f66727cdce45117a712b29a443943733257", - "sha256:4c91af6e967c2015729d3e69c2e51d92f9898c330d6a851bf8f121236f3defd3", - "sha256:770f3782b31f50b68627e22f91cb182c48c47c02eb405fd689472aa7b7aa16dc", - "sha256:79f9b6f7c46ae1f8ded75f68cf8ad50e5729ed4d590c74840471fc2823457d04", - "sha256:7a33145e04d44ce95bcd71e522b478d282ad0eafaf34fe1ec5bbd73e662f22b6", - "sha256:857959354ae3a6fa3da6651b966d13b0a8bed6bbc87a0de7b38a549db1d2a359", - "sha256:87f37fe5130574ff76c17cab61e7d2538a16f843bb7bca8ebbc4b12de3078596", - "sha256:95d5251e4b5ca00061f9d9f3d6fe537247e145a8524ae9fd30a2f8fbce993b5b", - "sha256:9d1d3e63a4afdc29bd76ce6aa9d58c771cd1599fbba8cf5057e7860b203710dd", - "sha256:a36c5c154f9d42ec176e6e620cb0dd275744aa1d804786a71ac37dc3661a5e95", - "sha256:ae5e35a2c189d397b91034642cb0eab0e346f776ec2eb44a49a459e6615d6e2e", - "sha256:b0f7d4a3df8f06cf49f9f121bead236e328074de6449866515cea4907bbc63d6", - "sha256:b75110fb114fa366b29a027d0c9be3709579602ae111ff61674d28c93606acca", - "sha256:ba5e697569f84b13640c9e193170e89c13c6244c24400fc57e88724ef610cd31", - "sha256:be2a9b390f77fd7676d80bc3cdc4f8edb940d8c198ed2d8c0be1319018c778e1", - "sha256:d5d8555d9bfc3f02385c1c37e9f998e2011f0db4f90e250e5bc0c0a85a813085", - "sha256:e55e22ac0a30023426564b1059b035973ec82186ddddbac867078435801c7801", - "sha256:e90f17980e6ab0f3c2f3730e56d1fe9bcba1891eeea58966e89d352492cc74f4", - "sha256:ecbb7b01409e9b782df5ded849c178a0aa7c906cf8c5a67368047daab282b184", - "sha256:ed01918d545a38998bfa5902c7c00e0fee90e957ce036a4000a88e3fe2264917", - "sha256:edabd457cd23a02965166026fd9bfd196f4324fe6032e866d0f3bd0301cd486f", - "sha256:fdf1c1dc5bafc32bc5d08b054f94d659422b05aba244d6be4ddc1c72d9aa70fb" - ], - "version": "==1.11.5" - }, - "chardet": { - "hashes": [ - "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", - "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" - ], - "version": "==3.0.4" - }, - "click": { - "hashes": [ - "sha256:29f99fc6125fbc931b758dc053b3114e55c77a6e4c6c3a2674a2dc986016381d", - "sha256:f15516df478d5a56180fbf80e68f206010e6d160fc39fa508b65e035fd75130b" - ], - "version": "==6.7" - }, - "colorama": { - "hashes": [ - "sha256:463f8483208e921368c9f306094eb6f725c6ca42b0f97e313cb5d5512459feda", - "sha256:48eb22f4f8461b1df5734a074b57042430fb06e1d61bd1e11b078c0fe6d7a1f1" - ], - "markers": "sys_platform == 'win32'", - "version": "==0.3.9" - }, - "contextlib2": { - "hashes": [ - "sha256:509f9419ee91cdd00ba34443217d5ca51f5a364a404e1dce9e8979cea969ca48", - "sha256:f5260a6e679d2ff42ec91ec5252f4eeffdcf21053db9113bd0a8e4d953769c00" - ], - "markers": "python_version < '3.2'", - "version": "==0.5.5" - }, - "crayons": { - "hashes": [ - "sha256:5e17691605e564d63482067eb6327d01a584bbaf870beffd4456a3391bd8809d", - "sha256:6f51241d0c4faec1c04c1c0ac6a68f1d66a4655476ce1570b3f37e5166a599cc" - ], - "version": "==0.1.2" - }, - "dateparser": { - "hashes": [ - "sha256:940828183c937bcec530753211b70f673c0a9aab831e43273489b310538dff86", - "sha256:b452ef8b36cd78ae86a50721794bc674aa3994e19b570f7ba92810f4e0a2ae03" - ], - "version": "==0.7.0" - }, - "decorator": { - "hashes": [ - "sha256:7d46dd9f3ea1cf5f06ee0e4e1277ae618cf48dfb10ada7c8427cd46c42702a0e", - "sha256:94d1d8905f5010d74bbbd86c30471255661a14187c45f8d7f3e5aa8540fdb2e5" - ], - "version": "==4.2.1" - }, - "e1839a8": { - "editable": true, - "path": "." - }, - "flask": { - "hashes": [ - "sha256:0749df235e3ff61ac108f69ac178c9770caeaccad2509cb762ce1f65570a8856", - "sha256:49f44461237b69ecd901cc7ce66feea0319b9158743dd27a2899962ab214dac1" - ], - "version": "==0.12.2" - }, - "flask-cache": { - "hashes": [ - "sha256:33187b3ddceeee233fe3db68ffcc118b5498e8ad28edde711bcbdcbf4924ce35", - "sha256:90126ca9bc063854ef8ee276e95d38b2b4ec8e45fd77d5751d37971ee27c7ef4", - "sha256:ae9d1ac4549517dfbc1f178ccc5429f61f836be3cc109a0b2481c98b3711c329" - ], - "version": "==0.13.1" - }, - "flask-common": { - "hashes": [ - "sha256:44fbb57a12bc7478d56c223eb5de7b2fb98ce42a70314c74ffecf5dbe75ed1b8" - ], - "version": "==0.2.0" - }, - "flask-limiter": { - "hashes": [ - "sha256:473aa5bc97310406aa8c12ab3dc080697bcfa8cd21a6d0aba30916911bbc673c", - "sha256:8cce98dcf25bf2ddbb824c2b503b4fc8e1a139154240fd2c60d9306bad8a0db8" - ], - "version": "==1.0.1" - }, - "funcsigs": { - "hashes": [ - "sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca", - "sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50" - ], - "markers": "python_version < '3.0'", - "version": "==1.0.2" - }, - "greenlet": { - "hashes": [ - "sha256:09ef2636ea35782364c830f07127d6c7a70542b178268714a9a9ba16318e7e8b", - "sha256:0fef83d43bf87a5196c91e73cb9772f945a4caaff91242766c5916d1dd1381e4", - "sha256:1b7df09c6598f5cfb40f843ade14ed1eb40596e75cd79b6fa2efc750ba01bb01", - "sha256:1fff21a2da5f9e03ddc5bd99131a6b8edf3d7f9d6bc29ba21784323d17806ed7", - "sha256:42118bf608e0288e35304b449a2d87e2ba77d1e373e8aa221ccdea073de026fa", - "sha256:50643fd6d54fd919f9a0a577c5f7b71f5d21f0959ab48767bd4bb73ae0839500", - "sha256:58798b5d30054bb4f6cf0f712f08e6092df23a718b69000786634a265e8911a9", - "sha256:5b49b3049697aeae17ef7bf21267e69972d9e04917658b4e788986ea5cc518e8", - "sha256:75c413551a436b462d5929255b6dc9c0c3c2b25cbeaee5271a56c7fda8ca49c0", - "sha256:769b740aeebd584cd59232be84fdcaf6270b8adc356596cdea5b2152c82caaac", - "sha256:ad2383d39f13534f3ca5c48fe1fc0975676846dc39c2cece78c0f1f9891418e0", - "sha256:b417bb7ff680d43e7bd7a13e2e08956fa6acb11fd432f74c97b7664f8bdb6ec1", - "sha256:b6ef0cabaf5a6ecb5ac122e689d25ba12433a90c7b067b12e5f28bdb7fb78254", - "sha256:c2de19c88bdb0366c976cc125dca1002ec1b346989d59524178adfd395e62421", - "sha256:c7b04a6dc74087b1598de8d713198de4718fa30ec6cbb84959b26426c198e041", - "sha256:f8f2a0ae8de0b49c7b5b2daca4f150fdd9c1173e854df2cce3b04123244f9f45", - "sha256:fcfadaf4bf68a27e5dc2f42cbb2f4b4ceea9f05d1d0b8f7787e640bed2801634" - ], - "version": "==0.4.13" - }, - "gunicorn": { - "hashes": [ - "sha256:75af03c99389535f218cc596c7de74df4763803f7b63eb09d77e92b3956b36c6", - "sha256:eee1169f0ca667be05db3351a0960765620dad53f53434262ff8901b68a1b622" - ], - "version": "==19.7.1" - }, - "httpbin": { - "hashes": [ - "sha256:0afa0486a76305cac441b5cc80d5d4ccd82b20875da7c5119ecfe616cefef45f", - "sha256:7c3a4d69f9d495e6df1d51a50ba7b1e4f33a1cb61e0db95ee7f8953e27a2243f" - ], - "version": "==0.6.2" - }, - "humanize": { - "hashes": [ - "sha256:a43f57115831ac7c70de098e6ac46ac13be00d69abbf60bdcac251344785bb19" - ], - "version": "==0.5.1" - }, - "idna": { - "hashes": [ - "sha256:2c6a5de3089009e3da7c5dde64a141dbc8551d5b7f6cf4ed7c2568d0cc520a8f", - "sha256:8c7309c718f94b3a625cb648ace320157ad16ff131ae0af362c9f21b80ef6ec4" - ], - "version": "==2.6" - }, - "itsdangerous": { - "hashes": [ - "sha256:cbb3fcf8d3e33df861709ecaf89d9e6629cff0a217bc2848f1b41cd30d360519" - ], - "version": "==0.24" - }, - "jinja2": { - "hashes": [ - "sha256:74c935a1b8bb9a3947c50a54766a969d4846290e1e788ea44c1392163723c3bd", - "sha256:f84be1bb0040caca4cea721fcbbbbd61f9be9464ca236387158b0feea01914a4" - ], - "version": "==2.10" - }, - "limits": { - "hashes": [ - "sha256:9df578f4161017d79f5188609f1d65f6b639f8aad2914c3960c9252e56a0ff95", - "sha256:a017b8d9e9da6761f4574642149c337f8f540d4edfe573fb91ad2c4001a2bc76" - ], - "version": "==1.3" - }, - "markupsafe": { - "hashes": [ - "sha256:a6be69091dac236ea9c6bc7d012beab42010fa914c459791d627dad4910eb665" - ], - "version": "==1.0" - }, - "maya": { - "hashes": [ - "sha256:ad1969bae78afb148c45a2f63591a7575ec05b4a0ab7ec04987ab7d73649f9d6", - "sha256:d8a7ed8513b2990036fe456c9f595b54d19ec49cb4461cd95a2ef6c487fb55eb" - ], - "version": "==0.3.4" - }, - "meinheld": { - "hashes": [ - "sha256:293eff4983b7fcbd9134b47706b22189883fe354993bd10163c65869d141e565", - "sha256:40d9dbce0165b2d9142f364d26fd6d59d3682f89d0dfe2117717a8ddad1f4133" - ], - "version": "==0.6.1" - }, - "pendulum": { - "hashes": [ - "sha256:0c14388546db6605a860b8b7112cb69d0b11c9ce5e072210504544e0d4575799", - "sha256:39a255776528afe11ea0d57814f9bf3729c1e0b99063af2e5c6cfd750c3e1f7f", - "sha256:3c85e8cbc91f45e1cc916cc9180b34153cd6aaaaacfb51a48b3156318314fa82", - "sha256:8199206c479b13947dcac63c025575d035331bb3819d1783dc1d568a11962906", - "sha256:8798aeca58b3dd7ffdc5a4993c9eaafedc4048165429e8f499ddd62c73bf3964", - "sha256:881efe37328de0785c0731d462e1485a45712f2cd5cb55907d6c15458460ebeb", - "sha256:bcca072f82e84b419efec1320cd3ee5c230d263f3a601b146651ed4db77d89f0", - "sha256:ff0c5fa3af4a471a218408c448b804ac6bccb105127727474f4e83c0e4072e97" - ], - "version": "==1.4.2" - }, - "pkginfo": { - "hashes": [ - "sha256:31a49103180ae1518b65d3f4ce09c784e2bc54e338197668b4fb7dc539521024", - "sha256:bb1a6aeabfc898f5df124e7e00303a5b3ec9a489535f346bfbddb081af93f89e" - ], - "version": "==1.4.1" - }, - "pluggy": { - "hashes": [ - "sha256:7f8ae7f5bdf75671a718d2daf0a64b7885f74510bcd98b1a0bb420eb9a9d0cff" - ], - "version": "==0.6.0" - }, - "py": { - "hashes": [ - "sha256:8cca5c229d225f8c1e3085be4fcf306090b00850fefad892f9d96c7b6e2f310f", - "sha256:ca18943e28235417756316bfada6cd96b23ce60dd532642690dcfdaba988a76d" - ], - "version": "==1.5.2" - }, - "pycparser": { - "hashes": [ - "sha256:99a8ca03e29851d96616ad0404b4aad7d9ee16f25c9f9708a11faf2810f7b226" - ], - "version": "==2.18" - }, - "pytest": { - "hashes": [ - "sha256:8970e25181e15ab14ae895599a0a0e0ade7d1f1c4c8ca1072ce16f25526a184d", - "sha256:9ddcb879c8cc859d2540204b5399011f842e5e8823674bf429f70ada281b3cc6" - ], - "version": "==3.4.1" - }, - "python-dateutil": { - "hashes": [ - "sha256:891c38b2a02f5bb1be3e4793866c8df49c7d19baabf9c1bad62547e0b4866aca", - "sha256:95511bae634d69bc7329ba55e646499a842bc4ec342ad54a8cdb65645a0aad3c" - ], - "version": "==2.6.1" - }, - "pytz": { - "hashes": [ - "sha256:07edfc3d4d2705a20a6e99d97f0c4b61c800b8232dc1c04d87e8554f130148dd", - "sha256:3a47ff71597f821cd84a162e71593004286e5be07a340fd462f0d33a760782b5", - "sha256:410bcd1d6409026fbaa65d9ed33bf6dd8b1e94a499e32168acfc7b332e4095c0", - "sha256:5bd55c744e6feaa4d599a6cbd8228b4f8f9ba96de2c38d56f08e534b3c9edf0d", - "sha256:61242a9abc626379574a166dc0e96a66cd7c3b27fc10868003fa210be4bff1c9", - "sha256:887ab5e5b32e4d0c86efddd3d055c1f363cbaa583beb8da5e22d2fa2f64d51ef", - "sha256:ba18e6a243b3625513d85239b3e49055a2f0318466e0b8a92b8fb8ca7ccdf55f", - "sha256:ed6509d9af298b7995d69a440e2822288f2eca1681b8cce37673dbb10091e5fe", - "sha256:f93ddcdd6342f94cea379c73cddb5724e0d6d0a1c91c9bdef364dc0368ba4fda" - ], - "version": "==2018.3" - }, - "pytzdata": { - "hashes": [ - "sha256:4e2cceb54335cd6c28caea46b15cd592e2aec5e8b05b0241cbccfb1b23c02ae7", - "sha256:7cd949123e2c2060fd12793de3a4a449e36b5dea5e169b810a3ac3f0b9877cfa" - ], - "version": "==2018.3" - }, - "raven": { - "hashes": [ - "sha256:738a52019d01955d5b44b49d67c9f2f4cedb1b4f70d4fb0b493931174d00e044", - "sha256:92bf4c4819472ed20f1b9905eeeafe1bc6fe5f273d7c14506fdb8fb3a6ab2074" - ], - "version": "==6.6.0" - }, - "regex": { - "hashes": [ - "sha256:1b428a296531ea1642a7da48562746309c5c06471a97bd0c02dd6a82e9cecee8", - "sha256:27d72bb42dffb32516c28d218bb054ce128afd3e18464f30837166346758af67", - "sha256:32cf4743debee9ea12d3626ee21eae83052763740e04086304e7a74778bf58c9", - "sha256:32f6408dbca35040bc65f9f4ae1444d5546411fde989cb71443a182dd643305e", - "sha256:333687d9a44738c486735955993f83bd22061a416c48f5a5f9e765e90cf1b0c9", - "sha256:35eeccf17af3b017a54d754e160af597036435c58eceae60f1dd1364ae1250c7", - "sha256:361a1fd703a35580a4714ec28d85e29780081a4c399a99bbfb2aee695d72aedb", - "sha256:494bed6396a20d3aa6376bdf2d3fbb1005b8f4339558d8ac7b53256755f80303", - "sha256:5b9c0ddd5b4afa08c9074170a2ea9b34ea296e32aeea522faaaaeeeb2fe0af2e", - "sha256:a50532f61b23d4ab9d216a6214f359dd05c911c1a1ad20986b6738a782926c1a", - "sha256:a9243d7b359b72c681a2c32eaa7ace8d346b7e8ce09d172a683acf6853161d9c", - "sha256:b44624a38d07d3c954c84ad302c29f7930f4bf01443beef5589e9157b14e2a29", - "sha256:be42a601aaaeb7a317f818490a39d153952a97c40c6e9beeb2a1103616405348", - "sha256:eee4d94b1a626490fc8170ffd788883f8c641b576e11ba9b4a29c9f6623371e0", - "sha256:f69d1201a4750f763971ea8364ed95ee888fc128968b39d38883a72a4d005895" - ], - "version": "==2018.2.21" - }, - "requests": { - "hashes": [ - "sha256:6a1b267aa90cac58ac3a765d067950e7dbbf75b1da07e895d1f594193a40a38b", - "sha256:9c443e7324ba5b85070c4a818ade28bfabedf16ea10206da1132edaa6dda237e" - ], - "version": "==2.18.4" - }, - "requests-toolbelt": { - "hashes": [ - "sha256:42c9c170abc2cacb78b8ab23ac957945c7716249206f90874651971a4acff237", - "sha256:f6a531936c6fa4c6cfce1b9c10d5c4f498d16528d2a54a22ca00011205a187b5" - ], - "version": "==0.8.0" - }, - "ruamel.yaml": { - "hashes": [ - "sha256:01e30ecb1b1c0ebf9fce814dc20dace402571517277799291202b61b22096c24", - "sha256:02babffd019911841ba01b76e23dfec7c9e9b2725503fb2698c4982fa1a6e835", - "sha256:072f6364a89972e8dc0afdce3335a709d5464dfeaa4f736d092a54574338b874", - "sha256:14d161558e3bf89e87d77c218098be22fa9a0d6d0bea40250fce525b1d0cbee2", - "sha256:5504398fc755a2b14c9983b2101161a8591a4b30812590cc1c365e7fcc117dfa", - "sha256:68c8f2986bcb91b6db1aea8698941769840c7257e951a9377048f7eff35be773", - "sha256:6d05c5a5baf829c70916c226ef3200650846a7227de226bca8a59efaf88bb973", - "sha256:6d7929b24e329d662fa43b657fddfee5260e2d35d0a543065cd755d4e17a9b2f", - "sha256:8dc74821e4bb6b21fb1ab35964e159391d99ee44981d07d57bf96e2395f3ef75", - "sha256:9225c83952d28f302cfc23c3d9a6f8231bfd581476d7aff1e3c7de49eecb4ee9", - "sha256:b6c5d5f03ba78e3f27c7188a00c4e09b6a4507fe3154ba40a294e09cb30ee016", - "sha256:c0908896e34b617ead40552cab03c1769bdc43d1da02419160dc900c5dfddde2", - "sha256:c41e04b526d0153c9246cfab87d7ddefdc9f165cb8886a8ec48ba7a2b73069f6", - "sha256:e2d2715bf92156bec5fb42e92e95dac1c4d9904f8a3d4e2d0c438758fe9092d7", - "sha256:e3bbfe0d294e08fdbb0cb05485435a2ceb4e168e98b5dc611f051c1864986b4b", - "sha256:f2d02a4af5a13b09d0b823cdd0317b54f3e0115e50b5ac4d9840c3a1b566817f", - "sha256:fcfc24a21594c071cc4588e84b7657a1f47ebcf6037c6c43fa15c4bbd3989ec2" - ], - "version": "==0.15.35" - }, - "six": { - "hashes": [ - "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", - "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb" - ], - "version": "==1.11.0" - }, - "tqdm": { - "hashes": [ - "sha256:5ec0d4442358e55cdb4a0471d04c6c831518fd8837f259db5537d90feab380df", - "sha256:f66468c14ccd011a627734c9b3fd72f20ce16f8faecc47384eb2507af5924fb9" - ], - "version": "==4.19.6" - }, - "twine": { - "hashes": [ - "sha256:caa45b7987fc96321258cd7668e3be2ff34064f5c66d2d975b641adca659c1ab", - "sha256:d3ce5c480c22ccfb761cd358526e862b32546d2fe4bc93d46b5cf04ea3cc46ca" - ], - "version": "==1.9.1" - }, - "tzlocal": { - "hashes": [ - "sha256:4ebeb848845ac898da6519b9b31879cf13b6626f7184c496037b818e238f2c4e" - ], - "version": "==1.5.1" - }, - "urllib3": { - "hashes": [ - "sha256:06330f386d6e4b195fbfc736b297f58c5a892e4440e54d294d7004e3a9bbea1b", - "sha256:cc44da8e1145637334317feebd728bd869a35285b93cbb4cca2577da7e62db4f" - ], - "version": "==1.22" - }, - "werkzeug": { - "hashes": [ - "sha256:c3fd7a7d41976d9f44db327260e263132466836cef6f91512889ed60ad26557c", - "sha256:d5da73735293558eb1651ee2fddc4d0dedcfa06538b8813a2e20011583c9e49b" - ], - "version": "==0.14.1" - }, - "whitenoise": { - "hashes": [ - "sha256:15f43b2e701821b95c9016cf469d29e2a546cb1c7dead584ba82c36f843995cf", - "sha256:9d81515f2b5b27051910996e1e860b1332e354d9e7bcf30c98f21dcb6713e0dd" - ], - "version": "==3.3.1" - } - } -} diff --git a/tests/unit/test_cmdparse.py b/tests/unit/test_cmdparse.py index 06012d07..1b329a53 100644 --- a/tests/unit/test_cmdparse.py +++ b/tests/unit/test_cmdparse.py @@ -47,3 +47,20 @@ def test_cmdify_complex(): '-c', """ "print(\'Double quote: \\\"\')" """.strip(), ]), script + + +@pytest.mark.run +@pytest.mark.script +def test_cmdify_quote_if_paren_in_command(): + """Ensure ONLY the command is quoted if it contains parentheses. + """ + script = Script.parse(' '.join([ + '"C:\\Python36(x86)\\python.exe"', + '-c', + "print(123)", + ])) + assert script.cmdify() == ' '.join([ + '"C:\\Python36(x86)\\python.exe"', + '-c', + "print(123)", + ]), script diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 98c85602..422c1002 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -4,6 +4,7 @@ import pytest from mock import patch, Mock from first import first import pipenv.utils +import pythonfinder.utils # Pipfile format <-> requirements.txt format. @@ -168,32 +169,6 @@ class TestUtils: from pipenv.vendor.requirementslib.utils import is_vcs assert is_vcs(entry) is expected - @pytest.mark.utils - def test_split_file(self): - pipfile_dict = { - "packages": { - "requests": {"git": "https://github.com/kennethreitz/requests.git"}, - "Flask": "*", - "tablib": {"path": ".", "editable": True}, - }, - "dev-packages": { - "Django": "==1.10", - "click": {"svn": "https://svn.notareal.com/click"}, - "crayons": {"hg": "https://hg.alsonotreal.com/crayons"}, - }, - } - split_dict = pipenv.utils.split_file(pipfile_dict) - assert list(split_dict["packages"].keys()) == ["Flask"] - assert split_dict["packages-vcs"] == { - "requests": {"git": "https://github.com/kennethreitz/requests.git"} - } - assert split_dict["packages-editable"] == { - "tablib": {"path": ".", "editable": True} - } - assert list(split_dict["dev-packages"].keys()) == ["Django"] - assert "click" in split_dict["dev-packages-vcs"] - assert "crayons" in split_dict["dev-packages-vcs"] - @pytest.mark.utils def test_python_version_from_bad_path(self): assert pipenv.utils.python_version("/fake/path") is None @@ -215,13 +190,13 @@ class TestUtils: ), ], ) - @patch("delegator.run") + # @patch(".vendor.pythonfinder.utils.get_python_version") def test_python_version_output_variants( - self, mocked_delegator, version_output, version + self, monkeypatch, version_output, version ): - run_ret = Mock() - run_ret.out = version_output - mocked_delegator.return_value = run_ret + def mock_version(path): + return version_output.split()[1] + monkeypatch.setattr("pipenv.vendor.pythonfinder.utils.get_python_version", mock_version) assert pipenv.utils.python_version("some/path") == version @pytest.mark.utils diff --git a/tests/unit/test_vendor.py b/tests/unit/test_vendor.py index 915af863..aea93112 100644 --- a/tests/unit/test_vendor.py +++ b/tests/unit/test_vendor.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # We need to import the patched packages directly from sys.path, so the # identity checks can pass. import pipenv # noqa @@ -7,35 +8,9 @@ import os import pytest import pytz +import tomlkit from pipfile.api import PipfileParser -from prettytoml import lexer, tokens -from prettytoml.elements.atomic import AtomicElement -from prettytoml.elements.metadata import ( - WhitespaceElement, PunctuationElement, CommentElement -) -from prettytoml.elements.table import TableElement -from prettytoml.tokens.py2toml import create_primitive_token - - -def test_table(): - initial_toml = """id=42 # My id\nage=14""" - tokens = tuple(lexer.tokenize(initial_toml)) - table = TableElement( - [ - AtomicElement(tokens[0:1]), - PunctuationElement(tokens[1:2]), - AtomicElement(tokens[2:3]), - WhitespaceElement(tokens[3:4]), - CommentElement(tokens[4:6]), - AtomicElement(tokens[6:7]), - PunctuationElement(tokens[7:8]), - AtomicElement(tokens[8:9]), - ] - ) - assert set(table.items()) == {('id', 42), ('age', 14)} - del table['id'] - assert set(table.items()) == {('age', 14)} class TestPipfileParser: @@ -82,7 +57,7 @@ class TestPipfileParser: ), ( # Aware time in UTC. datetime.time(15, 10, tzinfo=pytz.UTC), - '15:10:00Z', + '15:10:00+00:00', ), ( # Aware local time. datetime.time(15, 10, tzinfo=pytz.FixedOffset(8 * 60)), @@ -102,5 +77,11 @@ class TestPipfileParser: ), ]) def test_token_date(dt, content): - token = create_primitive_token(dt) - assert token == tokens.Token(tokens.TYPE_DATE, content) + item = tomlkit.item(dt) + assert item.as_string() == content + + +def test_dump_nonascii_string(): + content = u'name = "Stažené"\n' + toml_content = tomlkit.dumps(tomlkit.loads(content)) + assert toml_content == content