diff --git a/.azure-pipelines/docs.yml b/.azure-pipelines/docs.yml deleted file mode 100644 index b65fdfb4..00000000 --- a/.azure-pipelines/docs.yml +++ /dev/null @@ -1,21 +0,0 @@ -jobs: -- job: - displayName: Docs - pool: - vmImage: ubuntu-16.04 - - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.7' - - - template: steps/install-dependencies.yml - - - bash: tox -e docs - displayName: Build docs - - - task: PublishBuildArtifacts@1 - displayName: 'Publish Artifact: docs' - inputs: - pathToPublish: docs/build - artifactName: docs diff --git a/.azure-pipelines/jobs/run-manifest-check.yml b/.azure-pipelines/jobs/run-manifest-check.yml deleted file mode 100644 index 6d0b97eb..00000000 --- a/.azure-pipelines/jobs/run-manifest-check.yml +++ /dev/null @@ -1,14 +0,0 @@ -steps: -- task: UsePythonVersion@0 - displayName: Use Python $(python.version) - inputs: - versionSpec: '$(python.version)' - architecture: '$(python.architecture)' - -- template: ../steps/install-dependencies.yml - -- bash: | - export GIT_SSL_CAINFO=$(python -m certifi) - export LANG=C.UTF-8 - python -m pip install check-manifest - check-manifest diff --git a/.azure-pipelines/jobs/run-tests-windows.yml b/.azure-pipelines/jobs/run-tests-windows.yml deleted file mode 100644 index 6b6f86fa..00000000 --- a/.azure-pipelines/jobs/run-tests-windows.yml +++ /dev/null @@ -1,12 +0,0 @@ -steps: -- task: UsePythonVersion@0 - displayName: Use Python $(python.version) - inputs: - versionSpec: '$(python.version)' - architecture: '$(python.architecture)' - -- template: ../steps/install-dependencies.yml - -- template: ../steps/create-virtualenv.yml - -- template: ../steps/run-tests.yml diff --git a/.azure-pipelines/jobs/run-tests.yml b/.azure-pipelines/jobs/run-tests.yml deleted file mode 100644 index 3544af41..00000000 --- a/.azure-pipelines/jobs/run-tests.yml +++ /dev/null @@ -1,38 +0,0 @@ -steps: -- task: UsePythonVersion@0 - displayName: Use Python $(python.version) - inputs: - versionSpec: '$(python.version)' - architecture: '$(python.architecture)' - -- template: ../steps/install-dependencies.yml - -- bash: | - mkdir -p "$AGENT_HOMEDIRECTORY/.virtualenvs" - mkdir -p "$WORKON_HOME" - pip install certifi - export GIT_SSL_CAINFO="$(python -m certifi)" - export LANG="C.UTF-8" - export PIP_PROCESS_DEPENDENCY_LINKS="1" - echo "Path $PATH" - echo "Installing Pipenv…" - pip install -e "$(pwd)" --upgrade - pipenv install --deploy --dev - echo pipenv --venv && echo pipenv --py && echo pipenv run python --version - displayName: Make Virtualenv - -- script: | - # Fix Git SSL errors - export GIT_SSL_CAINFO="$(python -m certifi)" - export LANG="C.UTF-8" - export PIP_PROCESS_DEPENDENCY_LINKS="1" - git submodule sync && git submodule update --init --recursive - pipenv run pytest --junitxml=test-results.xml - displayName: Run integration tests - -- task: PublishTestResults@2 - displayName: Publish Test Results - inputs: - testResultsFiles: '**/test-results.xml' - testRunTitle: 'Python $(python.version)' - condition: succeededOrFailed() diff --git a/.azure-pipelines/jobs/run-vendor-scripts.yml b/.azure-pipelines/jobs/run-vendor-scripts.yml deleted file mode 100644 index 7fe75731..00000000 --- a/.azure-pipelines/jobs/run-vendor-scripts.yml +++ /dev/null @@ -1,39 +0,0 @@ -parameters: - vmImage: - -jobs: -- job: Vendor_Scripts - displayName: Test Vendor Scripts - pool: - vmImage: ${{ parameters.vmImage }} - strategy: - maxParallel: 4 - matrix: - ${{ if eq(parameters.vmImage, 'vs2017-win2016') }}: - # TODO remove once vs2017-win2016 has Python 3.7 - Python37: - python.version: '>= 3.7.0-b2' - python.architecture: x64 - ${{ if ne(parameters.vmImage, 'vs2017-win2016' )}}: - Python37: - python.version: '>= 3.7' - python.architecture: x64 - steps: - - task: UsePythonVersion@0 - displayName: Use Python $(python.version) - inputs: - versionSpec: '$(python.version)' - architecture: '$(python.architecture)' - - - template: ../steps/install-dependencies.yml - - - bash: | - mkdir -p "$AGENT_HOMEDIRECTORY/.virtualenvs" - mkdir -p "$WORKON_HOME" - pip install certifi - export GIT_SSL_CAINFO=$(python -m certifi) - export LANG=C.UTF-8 - python -m pip install --upgrade invoke requests parver - python -m invoke vendoring.update - - - template: ./run-manifest-check.yml diff --git a/.azure-pipelines/jobs/test.yml b/.azure-pipelines/jobs/test.yml deleted file mode 100644 index 150937ef..00000000 --- a/.azure-pipelines/jobs/test.yml +++ /dev/null @@ -1,48 +0,0 @@ -parameters: - vmImage: - -jobs: -- job: Test_Primary - displayName: Test Primary - pool: - vmImage: ${{ parameters.vmImage }} - strategy: - maxParallel: 4 - matrix: - Python27: - python.version: '2.7' - python.architecture: x64 - ${{ if eq(parameters.vmImage, 'vs2017-win2016') }}: - # TODO remove once vs2017-win2016 has Python 3.7 - Python37: - python.version: '>= 3.7.0-b2' - python.architecture: x64 - ${{ if ne(parameters.vmImage, 'vs2017-win2016' )}}: - Python37: - python.version: '>= 3.7' - python.architecture: x64 - steps: - - ${{ if eq(parameters.vmImage, 'vs2017-win2016') }}: - - template: ./run-tests-windows.yml - - - ${{ if ne(parameters.vmImage, 'vs2017-win2016') }}: - - template: ./run-tests.yml - -- job: Test_Secondary - displayName: Test python3.6 - # Run after Test_Primary so we don't devour time and jobs if tests are going to fail - # dependsOn: Test_Primary - pool: - vmImage: ${{ parameters.vmImage }} - strategy: - maxParallel: 4 - matrix: - Python36: - python.version: '3.6' - python.architecture: x64 - steps: - - ${{ if eq(parameters.vmImage, 'vs2017-win2016') }}: - - template: ./run-tests-windows.yml - - - ${{ if ne(parameters.vmImage, 'vs2017-win2016') }}: - - template: ./run-tests.yml diff --git a/.azure-pipelines/linux.yml b/.azure-pipelines/linux.yml deleted file mode 100644 index a2f34d4c..00000000 --- a/.azure-pipelines/linux.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: Pipenv Build Rules -trigger: - batch: true - branches: - include: - - master - paths: - exclude: - - docs/* - - news/* - - README.md - - pipenv/*.txt - - CHANGELOG.rst - - CONTRIBUTING.md - - CODE_OF_CONDUCT.md - - .gitignore - - .gitattributes - - .editorconfig - -jobs: -- template: jobs/test.yml - parameters: - vmImage: ubuntu-16.04 - -- template: jobs/run-vendor-scripts.yml - parameters: - vmImage: ubuntu-16.04 diff --git a/.azure-pipelines/steps/build-package.yml b/.azure-pipelines/steps/build-package.yml new file mode 100644 index 00000000..81a6160b --- /dev/null +++ b/.azure-pipelines/steps/build-package.yml @@ -0,0 +1,31 @@ +steps: +- task: UsePythonVersion@0 + inputs: + versionSpec: $(python.version) + architecture: '$(python.architecture)' + addToPath: true + displayName: Use Python $(python.version) + +- template: install-dependencies.yml + +- script: | + echo '##vso[task.setvariable variable=PIPENV_DEFAULT_PYTHON_VERSION]'$(python.version) + env: + PYTHON_VERSION: $(python.version) + +- template: create-virtualenv.yml + parameters: + python_version: $(python.version) + +- script: | + python -m pip install --upgrade wheel pip setuptools twine readme_renderer[md] + python setup.py sdist bdist_wheel + twine check dist/* + displayName: Build and check package + env: + PY_EXE: $(PY_EXE) + GIT_SSL_CAINFO: $(GIT_SSL_CAINFO) + LANG: $(LANG) + PIPENV_DEFAULT_PYTHON_VERSION: $(PIPENV_DEFAULT_PYTHON_VERSION) + PYTHONWARNINGS: ignore:DEPRECATION + PIPENV_NOSPIN: '1' diff --git a/.azure-pipelines/steps/create-virtualenv.yml b/.azure-pipelines/steps/create-virtualenv.yml index 9d1a6903..5f6160c4 100644 --- a/.azure-pipelines/steps/create-virtualenv.yml +++ b/.azure-pipelines/steps/create-virtualenv.yml @@ -1,6 +1,44 @@ +parameters: + python_version: '' + steps: + - script: | - virtualenv D:\.venv - D:\.venv\Scripts\pip.exe install -e . && D:\.venv\Scripts\pipenv install --dev - echo D:\.venv\Scripts\pipenv --venv && echo D:\.venv\Scripts\pipenv --py && echo D:\.venv\Scripts\pipenv run python --version + echo "##vso[task.setvariable variable=LANG]C.UTF-8" + echo "##vso[task.setvariable variable=PIP_PROCESS_DEPENDENCY_LINKS]1" + displayName: Set Environment Variables + +- ${{ if eq(parameters.vmImage, 'windows-2019') }}: + - powershell: | + pip install certifi + $env:PYTHON_PATH=$(python -c "import sys; print(sys.executable)") + $env:CERTIFI_CONTENT=$(python -m certifi) + echo "##vso[task.setvariable variable=GIT_SSL_CAINFO]$env:CERTIFI_CONTENT" + echo "##vso[task.setvariable variable=PY_EXE]$env:PYTHON_PATH" + displayName: Set Python Path + env: + PYTHONWARNINGS: 'ignore:DEPRECATION' +- ${{ if ne(parameters.vmImage, 'windows-2019') }}: + - bash: | + pip install certifi + PYTHON_PATH=$(python -c 'import sys; print(sys.executable)') + CERTIFI_CONTENT=$(python -m certifi) + echo "##vso[task.setvariable variable=GIT_SSL_CAINFO]$CERTIFI_CONTENT" + echo "##vso[task.setvariable variable=PY_EXE]$PYTHON_PATH" + displayName: Set Python Path + env: + PYTHONWARNINGS: 'ignore:DEPRECATION' + +- script: | + echo "Python path: $(PY_EXE)" + echo "GIT_SSL_CAINFO: $(GIT_SSL_CAINFO)" + echo "PIPENV PYTHON VERSION: $(python.version)" + echo "python_version: ${{ parameters.python_version }}" + git submodule sync + git submodule update --init --recursive + $(PY_EXE) -m pipenv install --deploy --dev --python="$(PY_EXE)" + env: + PIPENV_DEFAULT_PYTHON_VERSION: ${{ parameters.python_version }} + PYTHONWARNINGS: 'ignore:DEPRECATION' + PIPENV_NOSPIN: '1' displayName: Make Virtualenv diff --git a/.azure-pipelines/steps/install-dependencies.yml b/.azure-pipelines/steps/install-dependencies.yml index dfe733b6..79684d4a 100644 --- a/.azure-pipelines/steps/install-dependencies.yml +++ b/.azure-pipelines/steps/install-dependencies.yml @@ -1,3 +1,5 @@ steps: -- script: 'python -m pip install --upgrade pip && python -m pip install -e .' +- script: 'python -m pip install --upgrade pip setuptools wheel -e .[dev,tests] --upgrade' displayName: Upgrade Pip & Install Pipenv + env: + PYTHONWARNINGS: 'ignore:DEPRECATION' diff --git a/.azure-pipelines/steps/reinstall-pythons.yml b/.azure-pipelines/steps/reinstall-pythons.yml new file mode 100644 index 00000000..79647925 --- /dev/null +++ b/.azure-pipelines/steps/reinstall-pythons.yml @@ -0,0 +1,34 @@ +steps: + - script: | + # When you paste this, please make sure the indentation is preserved + # Fail out if any setups fail + set -e + + # Delete old Pythons + rm -rf $AGENT_TOOLSDIRECTORY/Python/2.7.16 + rm -rf $AGENT_TOOLSDIRECTORY/Python/3.5.7 + rm -rf $AGENT_TOOLSDIRECTORY/Python/3.7.3 + [ -e $AGENT_TOOLSDIRECTORY/Python/3.7.2 ] && [ -e $AGENT_TOOLSDIRECTORY/Python/3.5.5 ] && [ -e $AGENT_TOOLSDIRECTORY/Python/2.7.15 ] && exit 0 + # Download new Pythons + azcopy --recursive \ + --source https://vstsagenttools.blob.core.windows.net/tools/hostedtoolcache/linux/Python/2.7.15 \ + --destination $AGENT_TOOLSDIRECTORY/Python/2.7.15 + + azcopy --recursive \ + --source https://vstsagenttools.blob.core.windows.net/tools/hostedtoolcache/linux/Python/3.5.5 \ + --destination $AGENT_TOOLSDIRECTORY/Python/3.5.5 + + azcopy --recursive \ + --source https://vstsagenttools.blob.core.windows.net/tools/hostedtoolcache/linux/Python/3.7.2 \ + --destination $AGENT_TOOLSDIRECTORY/Python/3.7.2 + + # Install new Pythons + original_directory=$PWD + setups=$(find $AGENT_TOOLSDIRECTORY/Python -name setup.sh) + for setup in $setups; do + chmod +x $setup; + cd $(dirname $setup); + ./$(basename $setup); + cd $original_directory; + done; + displayName: 'Workaround: roll back Python versions' diff --git a/.azure-pipelines/steps/run-tests-linux.yml b/.azure-pipelines/steps/run-tests-linux.yml new file mode 100644 index 00000000..185a83b2 --- /dev/null +++ b/.azure-pipelines/steps/run-tests-linux.yml @@ -0,0 +1,14 @@ +parameters: + python_version: '' + +steps: +- script: | + # Fix Git SSL errors + echo "Using pipenv python version: $(PIPENV_DEFAULT_PYTHON_VERSION)" + git submodule sync && git submodule update --init --recursive + pipenv run pytest --junitxml=test-results.xml + displayName: Run integration tests + env: + PYTHONWARNINGS: ignore:DEPRECATION + PIPENV_NOSPIN: '1' + PIPENV_DEFAULT_PYTHON_VERSION: ${{ parameters.python_version }} diff --git a/.azure-pipelines/steps/run-tests-windows.yml b/.azure-pipelines/steps/run-tests-windows.yml new file mode 100644 index 00000000..1730fa14 --- /dev/null +++ b/.azure-pipelines/steps/run-tests-windows.yml @@ -0,0 +1,21 @@ +parameters: + python_version: '' + +steps: +- powershell: | + subst T: "$env:TEMP" + Write-Host "##vso[task.setvariable variable=TEMP]T:\" + Write-Host "##vso[task.setvariable variable=TMP]T:\" + Write-Host "##vso[task.setvariable variable=PIPENV_DEFAULT_PYTHON_VERSION]$env:PYTHON_VERSION" + Write-Host "##vso[task.setvariable variable=PIPENV_NOSPIN]1" + displayName: Fix Temp Variable + env: + PYTHON_VERSION: ${{ parameters.python_version }} + +- script: | + git submodule sync && git submodule update --init --recursive + pipenv run pytest -ra --ignore=pipenv\patched --ignore=pipenv\vendor --junitxml=test-results.xml tests + displayName: Run integration tests + env: + PYTHONWARNINGS: 'ignore:DEPRECATION' + PIPENV_NOSPIN: '1' diff --git a/.azure-pipelines/steps/run-tests.yml b/.azure-pipelines/steps/run-tests.yml index f33523f1..011cecf2 100644 --- a/.azure-pipelines/steps/run-tests.yml +++ b/.azure-pipelines/steps/run-tests.yml @@ -1,21 +1,30 @@ steps: -- powershell: | - # Fix Git SSL errors - pip install certifi - python -m certifi > cacert.txt - Write-Host "##vso[task.setvariable variable=GIT_SSL_CAINFO]$(Get-Content cacert.txt)" - $env:GIT_SSL_CAINFO="$(Get-Content cacert.txt)" - # Shorten paths to get under MAX_PATH or else integration tests will fail - # https://bugs.python.org/issue18199 - subst T: "$env:TEMP" - Write-Host "##vso[task.setvariable variable=TEMP]T:\" - $env:TEMP='T:\' - Write-Host "##vso[task.setvariable variable=TMP]T:\" - $env:TMP='T:\' - git submodule sync - git submodule update --init --recursive - D:\.venv\Scripts\pipenv run pytest -ra --ignore=pipenv\patched --ignore=pipenv\vendor --junitxml=test-results.xml tests - displayName: Run integration tests +- task: UsePythonVersion@0 + inputs: + versionSpec: $(python.version) + architecture: '$(python.architecture)' + addToPath: true + displayName: Use Python $(python.version) + +- template: install-dependencies.yml + +- script: | + echo '##vso[task.setvariable variable=PIPENV_DEFAULT_PYTHON_VERSION]'$(python.version) + env: + PYTHON_VERSION: $(python.version) + +- template: create-virtualenv.yml + parameters: + python_version: $(python.version) + +- ${{ if eq(parameters.vmImage, 'windows-2019') }}: + - template: run-tests-windows.yml + parameters: + python_version: $(python.version) +- ${{ if ne(parameters.vmImage, 'windows-2019') }}: + - template: run-tests-linux.yml + parameters: + python_version: $(python.version) - task: PublishTestResults@2 displayName: Publish Test Results diff --git a/.azure-pipelines/steps/run-vendor-scripts.yml b/.azure-pipelines/steps/run-vendor-scripts.yml new file mode 100644 index 00000000..2aca1fe0 --- /dev/null +++ b/.azure-pipelines/steps/run-vendor-scripts.yml @@ -0,0 +1,33 @@ +parameters: + python_version: '' + +steps: +- task: UsePythonVersion@0 + inputs: + versionSpec: $(python.version) + architecture: '$(python.architecture)' + addToPath: true + displayName: Use Python $(python.version) + +- template: install-dependencies.yml + +- script: | + echo '##vso[task.setvariable variable=PIPENV_DEFAULT_PYTHON_VERSION]'$(python.version) + env: + PYTHON_VERSION: $(python.version) + +- template: create-virtualenv.yml + parameters: + python_version: $(python.version) + +- script: | + python -m pip install --upgrade invoke requests parver bs4 vistir towncrier pip setuptools wheel --upgrade-strategy=eager + python -m invoke vendoring.update + displayName: Run Vendor Scripts + env: + PY_EXE: $(PY_EXE) + GIT_SSL_CAINFO: $(GIT_SSL_CAINFO) + LANG: $(LANG) + PIPENV_DEFAULT_PYTHON_VERSION: '${{ parameters.python_version }}' + PYTHONWARNINGS: ignore:DEPRECATION + PIPENV_NOSPIN: '1' diff --git a/.azure-pipelines/windows.yml b/.azure-pipelines/windows.yml deleted file mode 100644 index 35d5c16e..00000000 --- a/.azure-pipelines/windows.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: Pipenv Build Rules -trigger: - batch: true - branches: - include: - - master - paths: - exclude: - - docs/* - - news/* - - README.md - - pipenv/*.txt - - CHANGELOG.rst - - CONTRIBUTING.md - - CODE_OF_CONDUCT.md - - .gitignore - - .gitattributes - - .editorconfig - -jobs: -- template: jobs/test.yml - parameters: - vmImage: vs2017-win2016 diff --git a/.github/ISSUE_TEMPLATE/Bug_report.md b/.github/ISSUE_TEMPLATE/Bug_report.md index c470a867..3dc4613e 100644 --- a/.github/ISSUE_TEMPLATE/Bug_report.md +++ b/.github/ISSUE_TEMPLATE/Bug_report.md @@ -5,7 +5,7 @@ about: Create a report to help us improve Be sure to check the existing issues (both open and closed!), and make sure you are running the latest version of Pipenv. -Check the [diagnose documentation](https://docs.pipenv.org/diagnose/) for common issues before posting! We may close your issue if it is very similar to one of them. Please be considerate, or be on your way. +Check the [diagnose documentation](https://docs.pipenv.org/en/latest/diagnose/) for common issues before posting! We may close your issue if it is very similar to one of them. Please be considerate, or be on your way. Make sure to mention your debugging experience if the documented solution failed. diff --git a/.gitignore b/.gitignore index 766ffe3a..bea15631 100644 --- a/.gitignore +++ b/.gitignore @@ -145,6 +145,10 @@ venv.bak/ # mypy .mypy_cache/ +# Temporarily generating these with pytype locally for type safety +typeshed/ +pytype.cfg + ### Python Patch ### .venv/ @@ -153,3 +157,13 @@ venv.bak/ # Custom rules (everything added below won't be overriden by 'Generate .gitignore File' if you use 'Update' option) .vs/slnx.sqlite + +# mypy/typing section +typeshed/ +.dmypy.json +mypyhtml/ +XDG_CACHE_HOME/ +snap/ +prime/ +stage/ +pip-wheel-metadata/ diff --git a/.gitmodules b/.gitmodules index f40ee10c..e2f779af 100644 --- a/.gitmodules +++ b/.gitmodules @@ -6,10 +6,25 @@ url = https://github.com/pinax/pinax.git [submodule "tests/test_artifacts/git/requests"] path = tests/test_artifacts/git/requests - url = https://github.com/requests/requests.git + url = https://github.com/kennethreitz/requests.git [submodule "tests/test_artifacts/git/six"] path = tests/test_artifacts/git/six url = https://github.com/benjaminp/six.git [submodule "tests/test_artifacts/git/dateutil"] path = tests/test_artifacts/git/dateutil url = https://github.com/dateutil/dateutil +[submodule "tests/test_artifacts/git/pyinstaller"] + path = tests/test_artifacts/git/pyinstaller + url = https://github.com/pyinstaller/pyinstaller.git +[submodule "tests/test_artifacts/git/jinja2"] + path = tests/test_artifacts/git/jinja2 + url = https://github.com/pallets/jinja.git +[submodule "tests/test_artifacts/git/flask"] + path = tests/test_artifacts/git/flask + url = https://github.com/pallets/flask.git +[submodule "tests/test_artifacts/git/requests-2.18.4"] + path = tests/test_artifacts/git/requests-2.18.4 + url = https://github.com/kennethreitz/requests +[submodule "tests/pypi"] + path = tests/pypi + url = https://github.com/sarugaku/pipenv-test-artifacts.git diff --git a/CHANGELOG.draft.rst b/CHANGELOG.draft.rst deleted file mode 100644 index f861b218..00000000 --- a/CHANGELOG.draft.rst +++ /dev/null @@ -1,50 +0,0 @@ -2018.7.1.dev0 (2018-07-15) -========================== - - -Features & Improvements ------------------------ - -- Updated test-pypi addon to better support json-api access (forward compatibility). - Improved testing process for new contributors. `#2568 `_ - - -Behavior Changes ----------------- - -- Virtual environment activation for ``run`` is revised to improve interpolation - with other Python discovery tools. `#2503 `_ - -- Improve terminal coloring to display better in Powershell. `#2511 `_ - -- Invoke ``virtualenv`` directly for virtual environment creation, instead of depending on ``pew``. `#2518 `_ - -- ``pipenv --help`` will now include short help descriptions. `#2542 `_ - - -Bug Fixes ---------- - -- Fix subshell invocation on Windows for Python 2. `#2515 `_ - -- Fixed a bug which sometimes caused pipenv to throw a ``TypeError`` or to run into encoding issues when writing lockfiles on python 2. `#2561 `_ - -- Improve quoting logic for ``pipenv run`` so it works better with Windows - built-in commands. `#2563 `_ - -- Fixed a bug related to parsing vcs requirements with both extras and subdirectory fragments. - Corrected an issue in the ``requirementslib`` parser which led to some markers being discarded rather than evaluated. `#2564 `_ - - -Vendored Libraries ------------------- - -- Pew is no longer vendored. Entry point ``pewtwo``, packages ``pipenv.pew`` and - ``pipenv.patched.pew`` are removed. `#2521 `_ - - -Improved Documentation ----------------------- - -- Simplified the test configuration process. `#2568 `_ - diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 45494be7..c00ba932 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,3 +1,39 @@ +2018.11.26 (2018-11-26) +======================= + +Bug Fixes +--------- + +- Environment variables are expanded correctly before running scripts on POSIX. `#3178 `_ +- Pipenv will no longer disable user-mode installation when the ``--system`` flag is passed in. `#3222 `_ +- Fixed an issue with attempting to render unicode output in non-unicode locales. `#3223 `_ +- Fixed a bug which could cause failures to occur when parsing python entries from global pyenv version files. `#3224 `_ +- Fixed an issue which prevented the parsing of named extras sections from certain ``setup.py`` files. `#3230 `_ +- Correctly detect the virtualenv location inside an activated virtualenv. `#3231 `_ +- Fixed a bug which caused spinner frames to be written to stdout during locking operations which could cause redirection pipes to fail. `#3239 `_ +- Fixed a bug that editable pacakges can't be uninstalled correctly. `#3240 `_ +- Corrected an issue with installation timeouts which caused dependency resolution to fail for longer duration resolution steps. `#3244 `_ +- Adding normal pep 508 compatible markers is now fully functional when using VCS dependencies. `#3249 `_ +- Updated ``requirementslib`` and ``pythonfinder`` for multiple bugfixes. `#3254 `_ +- Pipenv will now ignore hashes when installing with ``--skip-lock``. `#3255 `_ +- Fixed an issue where pipenv could crash when multiple pipenv processes attempted to create the same directory. `#3257 `_ +- Fixed an issue which sometimes prevented successful creation of project pipfiles. `#3260 `_ +- ``pipenv install`` will now unset the ``PYTHONHOME`` environment variable when not combined with ``--system``. `#3261 `_ +- Pipenv will ensure that warnings do not interfere with the resolution process by suppressing warnings' usage of standard output and writing to standard error instead. `#3273 `_ +- Fixed an issue which prevented variables from the environment, such as ``PIPENV_DEV`` or ``PIPENV_SYSTEM``, from being parsed and implemented correctly. `#3278 `_ +- Clear pythonfinder cache after Python install. `#3287 `_ +- Fixed a race condition in hash resolution for dependencies for certain dependencies with missing cache entries or fresh Pipenv installs. `#3289 `_ +- Pipenv will now respect top-level pins over VCS dependency locks. `#3296 `_ + +Vendored Libraries +------------------ + +- Update vendored dependencies to resolve resolution output parsing and python finding: + - ``pythonfinder 1.1.9 -> 1.1.10`` + - ``requirementslib 1.3.1 -> 1.3.3`` + - ``vistir 0.2.3 -> 0.2.5`` `#3280 `_ + + 2018.11.14 (2018-11-14) ======================= @@ -91,33 +127,33 @@ Bug Fixes --------- - Fixed a bug in ``pipenv clean`` which caused global packages to sometimes be inadvertently targeted for cleanup. `#2849 `_ - + - Fix broken backport imports for vendored vistir. `#2950 `_, `#2955 `_, `#2961 `_ - + - Fixed a bug with importing local vendored dependencies when running ``pipenv graph``. `#2952 `_ - + - Fixed a bug which caused executable discovery to fail when running inside a virtualenv. `#2957 `_ - + - Fix parsing of outline tables. `#2971 `_ - + - Fixed a bug which caused ``verify_ssl`` to fail to drop through to ``pip install`` correctly as ``trusted-host``. `#2979 `_ - + - Fixed a bug which caused canonicalized package names to fail to resolve against PyPI. `#2989 `_ - + - Enhanced CI detection to detect Azure Devops builds. `#2993 `_ - + - Fixed a bug which prevented installing pinned versions which used redirection symbols from the command line. `#2998 `_ - + - Fixed a bug which prevented installing the local directory in non-editable mode. `#3005 `_ - + Vendored Libraries ------------------ - Updated ``requirementslib`` to version ``1.1.9``. `#2989 `_ - + - Upgraded ``pythonfinder => 1.1.1`` and ``vistir => 0.1.7``. `#3007 `_ @@ -129,55 +165,55 @@ Features & Improvements - Added environment variables `PIPENV_VERBOSE` and `PIPENV_QUIET` to control output verbosity without needing to pass options. `#2527 `_ - + - Updated test-pypi addon to better support json-api access (forward compatibility). Improved testing process for new contributors. `#2568 `_ - + - Greatly enhanced python discovery functionality: - Added pep514 (windows launcher/finder) support for python discovery. - Introduced architecture discovery for python installations which support different architectures. `#2582 `_ - + - Added support for ``pipenv shell`` on msys and cygwin/mingw/git bash for Windows. `#2641 `_ - + - Enhanced resolution of editable and VCS dependencies. `#2643 `_ - + - Deduplicate and refactor CLI to use stateful arguments and object passing. See `this issue `_ for reference. `#2814 `_ - + Behavior Changes ---------------- - Virtual environment activation for ``run`` is revised to improve interpolation with other Python discovery tools. `#2503 `_ - + - Improve terminal coloring to display better in Powershell. `#2511 `_ - + - Invoke ``virtualenv`` directly for virtual environment creation, instead of depending on ``pew``. `#2518 `_ - + - ``pipenv --help`` will now include short help descriptions. `#2542 `_ - + - Add ``COMSPEC`` to fallback option (along with ``SHELL`` and ``PYENV_SHELL``) if shell detection fails, improving robustness on Windows. `#2651 `_ - + - Fallback to shell mode if `run` fails with Windows error 193 to handle non-executable commands. This should improve usability on Windows, where some users run non-executable files without specifying a command, relying on Windows file association to choose the current command. `#2718 `_ - + Bug Fixes --------- - Fixed a bug which prevented installation of editable requirements using ``ssh://`` style urls `#1393 `_ - + - VCS Refs for locked local editable dependencies will now update appropriately to the latest hash when running ``pipenv update``. `#1690 `_ - + - ``.tar.gz`` and ``.zip`` artifacts will now have dependencies installed even when they are missing from the lockfile. `#2173 `_ - + - The command line parser will now handle multiple ``-e/--editable`` dependencies properly via click's option parser to help mitigate future parsing issues. `#2279 `_ - + - Fixed the ability of pipenv to parse ``dependency_links`` from ``setup.py`` when ``PIP_PROCESS_DEPENDENCY_LINKS`` is enabled. `#2434 `_ - + - Fixed a bug which could cause ``-i/--index`` arguments to sometimes be incorrectly picked up in packages. This is now handled in the command line parser. `#2494 `_ - + - Fixed non-deterministic resolution issues related to changes to the internal package finder in ``pip 10``. `#2499 `_, `#2529 `_, `#2589 `_, @@ -191,51 +227,51 @@ Bug Fixes `#2879 `_, `#2894 `_, `#2933 `_ - + - Fix subshell invocation on Windows for Python 2. `#2515 `_ - + - Fixed a bug which sometimes caused pipenv to throw a ``TypeError`` or to run into encoding issues when writing lockfiles on python 2. `#2561 `_ - + - Improve quoting logic for ``pipenv run`` so it works better with Windows built-in commands. `#2563 `_ - + - Fixed a bug related to parsing vcs requirements with both extras and subdirectory fragments. Corrected an issue in the ``requirementslib`` parser which led to some markers being discarded rather than evaluated. `#2564 `_ - + - Fixed multiple issues with finding the correct system python locations. `#2582 `_ - + - Catch JSON decoding error to prevent exception when the lock file is of invalid format. `#2607 `_ - + - Fixed a rare bug which could sometimes cause errors when installing packages with custom sources. `#2610 `_ - + - Update requirementslib to fix a bug which could raise an ``UnboundLocalError`` when parsing malformed VCS URIs. `#2617 `_ - + - Fixed an issue which prevented passing multiple ``--ignore`` parameters to ``pipenv check``. `#2632 `_ - + - Fixed a bug which caused attempted hashing of ``ssh://`` style URIs which could cause failures during installation of private ssh repositories. - Corrected path conversion issues which caused certain editable VCS paths to be converted to ``ssh://`` URIs improperly. `#2639 `_ - + - Fixed a bug which caused paths to be formatted incorrectly when using ``pipenv shell`` in bash for windows. `#2641 `_ - + - Dependency links to private repositories defined via ``ssh://`` schemes will now install correctly and skip hashing as long as ``PIP_PROCESS_DEPENDENCY_LINKS=1``. `#2643 `_ - + - Fixed a bug which sometimes caused pipenv to parse the ``trusted_host`` argument to pip incorrectly when parsing source URLs which specify ``verify_ssl = false``. `#2656 `_ - + - Prevent crashing when a virtual environment in ``WORKON_HOME`` is faulty. `#2676 `_ - + - Fixed virtualenv creation failure when a .venv file is present in the project root. `#2680 `_ - + - Fixed a bug which could cause the ``-e/--editable`` argument on a dependency to be accidentally parsed as a dependency itself. `#2714 `_ - + - Correctly pass `verbose` and `debug` flags to the resolver subprocess so it generates appropriate output. This also resolves a bug introduced by the fix to #2527. `#2732 `_ - + - All markers are now included in ``pipenv lock --requirements`` output. `#2748 `_ - + - Fixed a bug in marker resolution which could cause duplicate and non-deterministic markers. `#2760 `_ - + - Fixed a bug in the dependency resolver which caused regular issues when handling ``setup.py`` based dependency resolution. `#2766 `_ - + - Updated vendored dependencies: - ``pip-tools`` (updated and patched to latest w/ ``pip 18.0`` compatibilty) - ``pip 10.0.1 => 18.0`` @@ -257,24 +293,24 @@ Bug Fixes - ``vistir 0.1.4 => 0.1.6`` `#2802 `_, `#2867 `_, `#2880 `_ - + - Fixed a bug where `pipenv` crashes when the `WORKON_HOME` directory does not exist. `#2877 `_ - + - Fixed pip is not loaded from pipenv's patched one but the system one `#2912 `_ - + - Fixed various bugs related to ``pip 18.1`` release which prevented locking, installation, and syncing, and dumping to a ``requirements.txt`` file. `#2924 `_ - + Vendored Libraries ------------------ - Pew is no longer vendored. Entry point ``pewtwo``, packages ``pipenv.pew`` and ``pipenv.patched.pew`` are removed. `#2521 `_ - + - Update ``pythonfinder`` to major release ``1.0.0`` for integration. `#2582 `_ - + - Update requirementslib to fix a bug which could raise an ``UnboundLocalError`` when parsing malformed VCS URIs. `#2617 `_ - + - - Vendored new libraries ``vistir`` and ``pip-shims``, ``tomlkit``, ``modutil``, and ``plette``. - Update vendored libraries: @@ -289,7 +325,7 @@ Vendored Libraries - ``pythonfinder`` to ``1.0.2`` - ``pipdeptree`` to ``0.13.0`` - ``python-dotenv`` to ``0.9.1`` `#2639 `_ - + - Updated vendored dependencies: - ``pip-tools`` (updated and patched to latest w/ ``pip 18.0`` compatibilty) - ``pip 10.0.1 => 18.0`` @@ -310,31 +346,31 @@ Vendored Libraries - ``tomlkit 0.4.2 => 0.4.4`` - ``vistir 0.1.4 => 0.1.6`` `#2902 `_, `#2935 `_ - + Improved Documentation ---------------------- - Simplified the test configuration process. `#2568 `_ - + - Updated documentation to use working fortune cookie addon. `#2644 `_ - + - Added additional information about troubleshooting ``pipenv shell`` by using the the ``$PIPENV_SHELL`` environment variable. `#2671 `_ - + - Added a link to ``PEP-440`` version specifiers in the documentation for additional detail. `#2674 `_ - + - Added simple example to README.md for installing from git. `#2685 `_ - + - Stopped recommending `--system` for Docker contexts. `#2762 `_ - + - Fixed the example url for doing "pipenv install -e some-repo-url#egg=something", it was missing the "egg=" in the fragment identifier. `#2792 `_ - + - Fixed link to the "be cordial" essay in the contribution documentation. `#2793 `_ - + - Clarify `pipenv install` documentation `#2844 `_ - + - Replace reference to uservoice with PEEP-000 `#2909 `_ @@ -345,73 +381,73 @@ Features & Improvements ----------------------- - All calls to ``pipenv shell`` are now implemented from the ground up using `shellingham `_, a custom library which was purpose built to handle edge cases and shell detection. `#2371 `_ - + - Added support for python 3.7 via a few small compatibility / bugfixes. `#2427 `_, `#2434 `_, `#2436 `_ - + - Added new flag ``pipenv --support`` to replace the diagnostic command ``python -m pipenv.help``. `#2477 `_, `#2478 `_ - + - Improved import times and CLI runtimes with minor tweaks. `#2485 `_ - + Bug Fixes --------- - Fixed an ongoing bug which sometimes resolved incompatible versions into lockfiles. `#1901 `_ - + - Fixed a bug which caused errors when creating virtualenvs which contained leading dash characters. `#2415 `_ - + - Fixed a logic error which caused ``--deploy --system`` to overwrite editable vcs packages in the pipfile before installing, which caused any installation to fail by default. `#2417 `_ - + - Updated requirementslib to fix an issue with properly quoting markers in VCS requirements. `#2419 `_ - + - Installed new vendored jinja2 templates for ``click-completion`` which were causing template errors for users with completion enabled. `#2422 `_ - + - Added support for python 3.7 via a few small compatibility / bugfixes. `#2427 `_ - + - Fixed an issue reading package names from ``setup.py`` files in projects which imported utilities such as ``versioneer``. `#2433 `_ - + - Pipenv will now ensure that its internal package names registry files are written with unicode strings. `#2450 `_ - + - Fixed a bug causing requirements input as relative paths to be output as absolute paths or URIs. Fixed a bug affecting normalization of ``git+git@host`` uris. `#2453 `_ - + - Pipenv will now always use ``pathlib2`` for ``Path`` based filesystem interactions by default on ``python<3.5``. `#2454 `_ - + - Fixed a bug which prevented passing proxy PyPI indexes set with ``--pypi-mirror`` from being passed to pip during virtualenv creation, which could cause the creation to freeze in some cases. `#2462 `_ - + - Using the ``python -m pipenv.help`` command will now use proper encoding for the host filesystem to avoid encoding issues. `#2466 `_ - + - The new ``jinja2`` templates for ``click_completion`` will now be included in pipenv source distributions. `#2479 `_ - + - Resolved a long-standing issue with re-using previously generated ``InstallRequirement`` objects for resolution which could cause ``PKG-INFO`` file information to be deleted, raising a ``TypeError``. `#2480 `_ - + - Resolved an issue parsing usernames from private PyPI URIs in ``Pipfiles`` by updating ``requirementslib``. `#2484 `_ - + Vendored Libraries ------------------ - All calls to ``pipenv shell`` are now implemented from the ground up using `shellingham `_, a custom library which was purpose built to handle edge cases and shell detection. `#2371 `_ - + - Updated requirementslib to fix an issue with properly quoting markers in VCS requirements. `#2419 `_ - + - Installed new vendored jinja2 templates for ``click-completion`` which were causing template errors for users with completion enabled. `#2422 `_ - + - Add patch to ``prettytoml`` to support Python 3.7. `#2426 `_ - + - Patched ``prettytoml.AbstractTable._enumerate_items`` to handle ``StopIteration`` errors in preparation of release of python 3.7. `#2427 `_ - + - Fixed an issue reading package names from ``setup.py`` files in projects which imported utilities such as ``versioneer``. `#2433 `_ - + - Updated ``requirementslib`` to version ``1.0.9`` `#2453 `_ - + - Unraveled a lot of old, unnecessary patches to ``pip-tools`` which were causing non-deterministic resolution errors. `#2480 `_ - + - Resolved an issue parsing usernames from private PyPI URIs in ``Pipfiles`` by updating ``requirementslib``. `#2484 `_ - + Improved Documentation ---------------------- @@ -576,11 +612,11 @@ Improved Documentation ---------------------- - Update documentation wording to clarify Pipenv's overall role in the packaging ecosystem. `#2194 `_ - + - Added contribution documentation and guidelines. `#2205 `_ - + - Added instructions for supervisord compatibility. `#2215 `_ - + - Fixed broken links to development philosophy and contribution documentation. `#2248 `_ @@ -618,7 +654,6 @@ Vendored Libraries * certifi from version ``2018.1.16`` to ``2018.4.16``. * packaging from version ``16.8`` to ``17.1``. * six from version ``1.10.0`` to ``1.11.0``. - * requirementslib from version ``0.2.0`` to ``1.0.1``. + * requirementslib from version ``0.2.0`` to ``1.0.1``. In addition, scandir was vendored and patched to avoid importing host system binaries when falling back to pathlib2. `#2368 `_ - diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 77ee1ba0..88a3f28f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,7 +2,5 @@ Please see the [Contributing Guide](https://pipenv.readthedocs.io/en/latest/dev/contributing/). - - - - +1. Read our [Contributor's Guide](https://docs.pipenv.org/en/latest/dev/contributing/). +2. Understand our [development philosophy](https://docs.pipenv.org/en/latest/dev/philosophy/). diff --git a/Dockerfile b/Dockerfile index c6b6fb7d..0aae379c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,9 +3,11 @@ FROM heroku/heroku:18-build ENV DEBIAN_FRONTEND noninteractive ENV LC_ALL C.UTF-8 ENV LANG C.UTF-8 +# Python, don't write bytecode! +ENV PYTHONDONTWRITEBYTECODE 1 # -- Install Pipenv: -RUN apt update && apt upgrade -y && apt install python3.7-dev -y +RUN apt update && apt upgrade -y && apt install python3.7-dev libffi-dev -y RUN curl --silent https://bootstrap.pypa.io/get-pip.py | python3.7 # Backwards compatility. diff --git a/Makefile b/Makefile index 2722bdb4..059945bb 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,50 @@ +get_venv_dir:=$(shell mktemp -d 2>/dev/null || mktemp -d -t 'tmpvenv') +venv_dir := $(get_venv_dir)/pipenv_venv +venv_file := $(CURDIR)/.test_venv +get_venv_path =$(file < $(venv_file)) + format: black pipenv/*.py test: docker-compose up + +.PHONY: ramdisk +ramdisk: + sudo mkdir -p /mnt/ramdisk + sudo mount -t tmpfs -o size=2g tmpfs /mnt/ramdisk + sudo chown -R ${USER}:${USER} /mnt/ramdisk + +.PHONY: ramdisk-virtualenv +ramdisk-virtualenv: ramdisk + [ ! -e "/mnt/ramdisk/.venv/bin/activate" ] && \ + python -m virtualenv /mnt/ramdisk/.venv + @echo "/mnt/ramdisk/.venv" >> $(venv_file) + +.PHONY: virtualenv +virtualenv: + [ ! -e $(venv_dir) ] && rm -rf $(venv_file) && python -m virtualenv $(venv_dir) + @echo $(venv_dir) >> $(venv_file) + +.PHONY: test-install +test-install: virtualenv + . $(get_venv_path)/bin/activate && \ + python -m pip install --upgrade pip virtualenv -e .[tests,dev] && \ + pipenv install --dev + +.PHONY: submodules +submodules: + git submodule sync + git submodule update --init --recursive + +.PHONY: tests +tests: virtualenv submodules test-install + . $(get_venv_path)/bin/activate && \ + pipenv run pytest -ra -vvv --full-trace --tb=long + +.PHONY: test-specific +test-specific: submodules virtualenv test-install + . $(get_venv_path)/bin/activate && pipenv run pytest -ra -k '$(tests)' + +.PHONY: retest +retest: virtualenv submodules test-install + . $(get_venv_path)/bin/activate && pipenv run pytest -ra -k 'test_check_unused or test_install_editable_git_tag or test_get_vcs_refs or test_skip_requirements_when_pipfile or test_editable_vcs_install or test_basic_vcs_install or test_git_vcs_install or test_ssh_vcs_install or test_vcs_can_use_markers' -vvv --full-trace --tb=long diff --git a/Pipfile b/Pipfile index a1db217c..826df207 100644 --- a/Pipfile +++ b/Pipfile @@ -1,26 +1,13 @@ [dev-packages] -pipenv = {path = ".", editable = true} -"flake8" = ">=3.3.0,<4" -pytest = "*" -mock = "*" -sphinx = "<=1.5.5" -twine = "*" +pipenv = {path = ".", editable = true, extras = ["tests", "dev"]} sphinx-click = "*" -pytest-xdist = "*" click = "*" -pytest-pypy = {path = "./tests/pytest-pypi", editable = true} -pytest-tap = "*" -flaky = "*" +pytest_pypi = {path = "./tests/pytest-pypi", editable = true} stdeb = {version="*", markers="sys_platform == 'linux'"} -black = {version="*", markers="python_version >= '3.6'"} -pytz = "*" -towncrier = {git = "https://github.com/hawkowl/towncrier.git", editable = true, ref = "master"} -parver = "*" -invoke = "*" jedi = "*" isort = "*" rope = "*" -passa = {editable = true, git = "https://github.com/sarugaku/passa.git"} +passa = {git = "https://github.com/sarugaku/passa.git"} [packages] diff --git a/Pipfile.lock b/Pipfile.lock index f5e02ec8..b2586e8d 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "6acc712d82698e574727d19b22d05bf46565ecaa414e288fd0d79e385f8fdd10" + "sha256": "f4d89c0aab5c4e865f8c96ba24613fb1e66bae803a3ceaeadb6abf0061898091" }, "pipfile-spec": 6, "requires": {}, @@ -27,6 +27,7 @@ "sha256:37228cda29411948b422fae072f57e31d3396d2ee1c9783775980ee9c9990af6", "sha256:58587dd4dc3daefad0487f6d9ae32b4542b185e1c36db6993290e7c41ca2b47c" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.5" }, "appdirs": { @@ -36,14 +37,6 @@ ], "version": "==1.4.3" }, - "argparse": { - "hashes": [ - "sha256:62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4", - "sha256:c31647edb69fd3d465a847ea3157d37bed1f95f19760b11a47aa91c04b666314" - ], - "markers": "python_version == '2.6'", - "version": "==1.4.0" - }, "arpeggio": { "hashes": [ "sha256:a5258b84f76661d558492fa87e42db634df143685a0e51802d59cae7daad8732", @@ -53,90 +46,71 @@ }, "atomicwrites": { "hashes": [ - "sha256:0312ad34fcad8fac3704d441f7b317e50af620823353ec657a53e981f92920c0", - "sha256:ec9ae8adaae229e4f8446952d204a3e4b5fdd2d099f9be3aaf556120135fb3ee" + "sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4", + "sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6" ], - "version": "==1.2.1" + "version": "==1.3.0" }, "attrs": { "hashes": [ - "sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69", - "sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb" + "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79", + "sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399" ], - "version": "==18.2.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==19.1.0" }, "babel": { "hashes": [ - "sha256:6778d85147d5d85345c14a26aada5e478ab04e39b078b0745ee6870c2b5cf669", - "sha256:8cba50f48c529ca3fa18cf81fa9403be176d374ac4d60738b839122dfaaa3d23" + "sha256:af92e6106cb7c55286b25b38ad7695f8b4efb36a90ba483d7f7a6628c46158ab", + "sha256:e86135ae101e31e2c8ec20a4e0c5220f4eed12487d5cf3f78be7e98d3a57fc28" ], - "version": "==2.6.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.7.0" + }, + "backports.functools-lru-cache": { + "hashes": [ + "sha256:9d98697f088eb1b0fa451391f91afb5e3ebde16bbdb272819fd091151fda4f1a", + "sha256:f0b0e4eba956de51238e17573b7087e852dfe9854afd2e9c873f73fc0ca0a6dd" + ], + "markers": "python_version < '3'", + "version": "==1.5" + }, + "beautifulsoup4": { + "hashes": [ + "sha256:034740f6cb549b4e932ae1ab975581e6103ac8f942200a0e9759065984391858", + "sha256:945065979fb8529dd2f37dbb58f00b661bdbcbebf954f93b32fdf5263ef35348", + "sha256:ba6d5c59906a85ac23dadfe5c88deaf3e179ef565f4898671253e50a78680718" + ], + "version": "==4.7.1" }, "black": { "hashes": [ - "sha256:817243426042db1d36617910df579a54f1afd659adb96fc5032fcf4b36209739", - "sha256:e030a9a28f542debc08acceb273f228ac422798e5215ba2a791a6ddeaaca22a5" + "sha256:09a9dcb7c46ed496a9850b76e4e825d6049ecd38b611f1224857a79bd985a8cf", + "sha256:68950ffd4d9169716bcb8719a56c07a2f4485354fec061cdd5910aa07369731c" ], - "index": "pypi", "markers": "python_version >= '3.6'", - "version": "==18.9b0" + "version": "==19.3b0" }, "bleach": { "hashes": [ - "sha256:48d39675b80a75f6d1c3bdbffec791cf0bbbab665cf01e20da701c77de278718", - "sha256:73d26f018af5d5adcdabf5c1c974add4361a9c76af215fe32fdec8a6fc5fb9b9" + "sha256:213336e49e102af26d9cde77dd2d0397afabc5a6bf2fed985dc35b5d1e285a16", + "sha256:3fdf7f77adcf649c9911387df51254b813185e32b2c6619f690b593a617e19fa" ], - "version": "==3.0.2" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==3.1.0" }, - "cerberus": { + "bs4": { "hashes": [ - "sha256:f5c2e048fb15ecb3c088d192164316093fcfa602a74b3386eefb2983aa7e800a" + "sha256:36ecea1fd7cc5c0c6e4a1ff075df26d50da647b75376626cc186e2212886dd3a" ], - "version": "==1.2" + "version": "==0.0.1" }, "certifi": { "hashes": [ - "sha256:339dc09518b07e2fa7eda5450740925974815557727d6bd35d319c1524a04a4c", - "sha256:6d58c986d22b038c8c0df30d639f23a3e6d172a05c3583e766f4c0b785c0986a" + "sha256:046832c04d4e752f37383b628bc601a7ea7211496b4638f6514d0e5b9acc4939", + "sha256:945e3ba63a0b9f577b1395204e13c3a231f9bc0223888be653286534e5873695" ], - "version": "==2018.10.15" - }, - "cffi": { - "hashes": [ - "sha256:151b7eefd035c56b2b2e1eb9963c90c6302dc15fbd8c1c0a83a163ff2c7d7743", - "sha256:1553d1e99f035ace1c0544050622b7bc963374a00c467edafac50ad7bd276aef", - "sha256:1b0493c091a1898f1136e3f4f991a784437fac3673780ff9de3bcf46c80b6b50", - "sha256:2ba8a45822b7aee805ab49abfe7eec16b90587f7f26df20c71dd89e45a97076f", - "sha256:3bb6bd7266598f318063e584378b8e27c67de998a43362e8fce664c54ee52d30", - "sha256:3c85641778460581c42924384f5e68076d724ceac0f267d66c757f7535069c93", - "sha256:3eb6434197633b7748cea30bf0ba9f66727cdce45117a712b29a443943733257", - "sha256:495c5c2d43bf6cebe0178eb3e88f9c4aa48d8934aa6e3cddb865c058da76756b", - "sha256:4c91af6e967c2015729d3e69c2e51d92f9898c330d6a851bf8f121236f3defd3", - "sha256:57b2533356cb2d8fac1555815929f7f5f14d68ac77b085d2326b571310f34f6e", - "sha256:770f3782b31f50b68627e22f91cb182c48c47c02eb405fd689472aa7b7aa16dc", - "sha256:79f9b6f7c46ae1f8ded75f68cf8ad50e5729ed4d590c74840471fc2823457d04", - "sha256:7a33145e04d44ce95bcd71e522b478d282ad0eafaf34fe1ec5bbd73e662f22b6", - "sha256:857959354ae3a6fa3da6651b966d13b0a8bed6bbc87a0de7b38a549db1d2a359", - "sha256:87f37fe5130574ff76c17cab61e7d2538a16f843bb7bca8ebbc4b12de3078596", - "sha256:95d5251e4b5ca00061f9d9f3d6fe537247e145a8524ae9fd30a2f8fbce993b5b", - "sha256:9d1d3e63a4afdc29bd76ce6aa9d58c771cd1599fbba8cf5057e7860b203710dd", - "sha256:a36c5c154f9d42ec176e6e620cb0dd275744aa1d804786a71ac37dc3661a5e95", - "sha256:a6a5cb8809091ec9ac03edde9304b3ad82ad4466333432b16d78ef40e0cce0d5", - "sha256:ae5e35a2c189d397b91034642cb0eab0e346f776ec2eb44a49a459e6615d6e2e", - "sha256:b0f7d4a3df8f06cf49f9f121bead236e328074de6449866515cea4907bbc63d6", - "sha256:b75110fb114fa366b29a027d0c9be3709579602ae111ff61674d28c93606acca", - "sha256:ba5e697569f84b13640c9e193170e89c13c6244c24400fc57e88724ef610cd31", - "sha256:be2a9b390f77fd7676d80bc3cdc4f8edb940d8c198ed2d8c0be1319018c778e1", - "sha256:ca1bd81f40adc59011f58159e4aa6445fc585a32bb8ac9badf7a2c1aa23822f2", - "sha256:d5d8555d9bfc3f02385c1c37e9f998e2011f0db4f90e250e5bc0c0a85a813085", - "sha256:e55e22ac0a30023426564b1059b035973ec82186ddddbac867078435801c7801", - "sha256:e90f17980e6ab0f3c2f3730e56d1fe9bcba1891eeea58966e89d352492cc74f4", - "sha256:ecbb7b01409e9b782df5ded849c178a0aa7c906cf8c5a67368047daab282b184", - "sha256:ed01918d545a38998bfa5902c7c00e0fee90e957ce036a4000a88e3fe2264917", - "sha256:edabd457cd23a02965166026fd9bfd196f4324fe6032e866d0f3bd0301cd486f", - "sha256:fdf1c1dc5bafc32bc5d08b054f94d659422b05aba244d6be4ddc1c72d9aa70fb" - ], - "version": "==1.11.5" + "version": "==2019.6.16" }, "chardet": { "hashes": [ @@ -145,6 +119,14 @@ ], "version": "==3.0.4" }, + "check-manifest": { + "hashes": [ + "sha256:8754cc8efd7c062a3705b442d1c23ff702d4477b41a269c2e354b25e1f5535a4", + "sha256:a4c555f658a7c135b8a22bd26c2e55cfaf5876e4d5962d8c25652f2addd556bc" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.39" + }, "click": { "hashes": [ "sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13", @@ -153,64 +135,35 @@ "index": "pypi", "version": "==7.0" }, - "cmarkgfm": { - "hashes": [ - "sha256:0186dccca79483e3405217993b83b914ba4559fe9a8396efc4eea56561b74061", - "sha256:1a625afc6f62da428df96ec325dc30866cc5781520cbd904ff4ec44cf018171c", - "sha256:207b7673ff4e177374c572feeae0e4ef33be620ec9171c08fd22e2b796e03e3d", - "sha256:275905bb371a99285c74931700db3f0c078e7603bed383e8cf1a09f3ee05a3de", - "sha256:50098f1c4950722521f0671e54139e0edc1837d63c990cf0f3d2c49607bb51a2", - "sha256:50ed116d0b60a07df0dc7b180c28569064b9d37d1578d4c9021cff04d725cb63", - "sha256:61a72def110eed903cd1848245897bcb80d295cd9d13944d4f9f30cba5b76655", - "sha256:64186fb75d973a06df0e6ea12879533b71f6e7ba1ab01ffee7fc3e7534758889", - "sha256:665303d34d7f14f10d7b0651082f25ebf7107f29ef3d699490cac16cdc0fc8ce", - "sha256:70b18f843aec58e4e64aadce48a897fe7c50426718b7753aaee399e72df64190", - "sha256:761ee7b04d1caee2931344ac6bfebf37102ffb203b136b676b0a71a3f0ea3c87", - "sha256:811527e9b7280b136734ed6cb6845e5fbccaeaa132ddf45f0246cbe544016957", - "sha256:987b0e157f70c72a84f3c2f9ef2d7ab0f26c08f2bf326c12c087ff9eebcb3ff5", - "sha256:9fc6a2183d0a9b0974ec7cdcdad42bd78a3be674cc3e65f87dd694419b3b0ab7", - "sha256:a3d17ee4ae739fe16f7501a52255c2e287ac817cfd88565b9859f70520afffea", - "sha256:ba5b5488719c0f2ced0aa1986376f7baff1a1653a8eb5fdfcf3f84c7ce46ef8d", - "sha256:c573ea89dd95d41b6d8cf36799c34b6d5b1eac4aed0212dee0f0a11fb7b01e8f", - "sha256:c5f1b9e8592d2c448c44e6bc0d91224b16ea5f8293908b1561de1f6d2d0658b1", - "sha256:cbe581456357d8f0674d6a590b1aaf46c11d01dd0a23af147a51a798c3818034", - "sha256:cf219bec69e601fe27e3974b7307d2f06082ab385d42752738ad2eb630a47d65", - "sha256:cf5014eb214d814a83a7a47407272d5db10b719dbeaf4d3cfe5969309d0fcf4b", - "sha256:d08bad67fa18f7e8ff738c090628ee0cbf0505d74a991c848d6d04abfe67b697", - "sha256:d6f716d7b1182bf35862b5065112f933f43dd1aa4f8097c9bcfb246f71528a34", - "sha256:e08e479102627641c7cb4ece421c6ed4124820b1758765db32201136762282d9", - "sha256:e20ac21418af0298437d29599f7851915497ce9f2866bc8e86b084d8911ee061", - "sha256:e25f53c37e319241b9a412382140dffac98ca756ba8f360ac7ab5e30cad9670a", - "sha256:e8932bddf159064f04e946fbb64693753488de21586f20e840b3be51745c8c09", - "sha256:f20900f16377f2109783ae9348d34bc80530808439591c3d3df73d5c7ef1a00c" - ], - "version": "==0.4.2" - }, "colorama": { "hashes": [ - "sha256:a3d89af5db9e9806a779a50296b5fdb466e281147c2c235e8225ecc6dbf7bbf3", - "sha256:c9b54bebe91a6a803e0772c8561d53f2926bfeb17cd141fbabcb08424086595c" + "sha256:05eed71e2e327246ad6b38c540c4a3117230b19679b875190486ddd2d721422d", + "sha256:f8ac84de7840f5b9c4e3347b3c1eaa50f7e49c2b07596221daec5edaabbd7c48" ], - "version": "==0.4.0" + "version": "==0.4.1" }, "configparser": { "hashes": [ - "sha256:5308b47021bc2340965c371f0f058cc6971a04502638d4244225c49d80db273a" + "sha256:8be81d89d6e7b4c0d4e44bcc525845f6da25821de80cb5e06e7e0238a2899e32", + "sha256:da60d0014fd8c55eb48c1c5354352e363e2d30bbf7057e5e171a468390184c75" ], - "markers": "python_version < '3.2'", - "version": "==3.5.0" + "markers": "python_version < '3'", + "version": "==3.7.4" }, - "cursor": { + "contextlib2": { "hashes": [ - "sha256:8ee9fe5b925e1001f6ae6c017e93682583d2b4d1ef7130a26cfcdf1651c0032c" + "sha256:509f9419ee91cdd00ba34443217d5ca51f5a364a404e1dce9e8979cea969ca48", + "sha256:f5260a6e679d2ff42ec91ec5252f4eeffdcf21053db9113bd0a8e4d953769c00" ], - "version": "==1.2.0" + "markers": "python_version < '3'", + "version": "==0.5.5" }, - "distlib": { + "decorator": { "hashes": [ - "sha256:57977cd7d9ea27986ec62f425630e4ddb42efe651ff80bc58ed8dbc3c7c21f19" + "sha256:86156361c50488b84a3f148056ea716ca587df2f0de1d34750d35c21312725de", + "sha256:f069f3a01830ca754ba5258fde2278454a0b5b79e0d7f5c13b3b97e57d4acff6" ], - "version": "==0.2.8" + "version": "==4.4.0" }, "docutils": { "hashes": [ @@ -220,6 +173,14 @@ ], "version": "==0.14" }, + "entrypoints": { + "hashes": [ + "sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19", + "sha256:c70dd71abe5a8c85e55e12c19bd91ccfeec11a6e99044204511f9ed547d48451" + ], + "markers": "python_version >= '2.7'", + "version": "==0.3" + }, "enum34": { "hashes": [ "sha256:2d81cbbe0e73112bdfe6ef8576f2238f2ba27dd0d55752a776c41d38b7da2850", @@ -227,82 +188,94 @@ "sha256:6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79", "sha256:8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1" ], - "markers": "python_version < '3.4'", + "markers": "python_version < '3'", "version": "==1.1.6" }, "execnet": { "hashes": [ - "sha256:a7a84d5fa07a089186a329528f127c9d73b9de57f1a1131b82bb5320ee651f6a", - "sha256:fc155a6b553c66c838d1a22dba1dc9f5f505c43285a878c6f74a79c024750b83" + "sha256:027ee5d961afa01e97b90d6ccc34b4ed976702bc58e7f092b3c513ea288cb6d2", + "sha256:752a3786f17416d491f833a29217dda3ea4a471fc5269c492eebcee8cc4772d3" ], - "version": "==1.5.0" - }, - "first": { - "hashes": [ - "sha256:3bb3de3582cb27071cfb514f00ed784dc444b7f96dc21e140de65fe00585c95e", - "sha256:41d5b64e70507d0c3ca742d68010a76060eea8a3d863e9b5130ab11a4a91aa0e" - ], - "version": "==2.0.1" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.6.0" }, "flake8": { "hashes": [ - "sha256:7253265f7abd8b313e3892944044a365e3f4ac3fcdcfb4298f55ee9ddf188ba0", - "sha256:c7841163e2b576d435799169b78703ad6ac1bbb0f199994fc05f700b2a90ea37" + "sha256:859996073f341f2670741b51ec1e67a01da142831aa1fdc6242dbf88dffbe661", + "sha256:a796a115208f5c03b18f332f7c11729812c8c3ded6c46319c59b53efd3819da8" ], - "index": "pypi", - "version": "==3.5.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==3.7.7" }, "flaky": { "hashes": [ - "sha256:4ad7880aef8c35a34ddb394d4fa33047765bca1e3d67d182bf6eba9c8eabf3a2", - "sha256:d0533f473a46b916e6db6e84e20b06d8a70656600a0c14e819b0760b63f70226" + "sha256:12bd5e41f372b2190e8d754b6e5829c2f11dbc764e10b30f57e59f829c9ca1da", + "sha256:a94931c46a33469ec26f09b652bc88f55a8f5cc77807b90ca7bbafef1108fd7d" ], - "index": "pypi", - "version": "==3.4.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==3.5.3" }, "flask": { "hashes": [ - "sha256:2271c0070dbcb5275fad4a82e29f23ab92682dc45f9dfbc22c02ba9b9322ce48", - "sha256:a080b744b7e345ccfcbc77954861cb05b3c63786e93f2b3875e0913d44b43f05" + "sha256:ad7c6d841e64296b962296c2c2dabc6543752985727af86a975072dea984b6f3", + "sha256:e7d32475d1de5facaa55e3958bc4ec66d3762076b074296aa50ef8fdc5b9df61" ], - "version": "==1.0.2" + "version": "==1.0.3" }, "funcsigs": { "hashes": [ "sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca", "sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50" ], - "markers": "python_version < '3.3'", + "markers": "python_version < '3.0'", "version": "==1.0.2" }, + "functools32": { + "hashes": [ + "sha256:89d824aa6c358c421a234d7f9ee0bd75933a67c29588ce50aaa3acdf4d403fa0", + "sha256:f6253dfbe0538ad2e387bd8fdfd9293c925d63553f5813c4e587745416501e6d" + ], + "markers": "python_version < '3.2'", + "version": "==3.2.3.post2" + }, "future": { "hashes": [ - "sha256:e39ced1ab767b5936646cedba8bcce582398233d6a627067d4c6a454c90cfedb" + "sha256:67045236dcfd6816dc439556d009594abf643e5eb48992e36beac09c2ca659b8" ], - "version": "==0.16.0" + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.17.1" }, "futures": { "hashes": [ "sha256:9ec02aa7d674acb8618afb127e27fde7fc68994c0437ad759fa094a574adb265", "sha256:ec0a6cb848cc212002b9828c3e34c675e0c9ff6741dc445cab6fdd4e1085d1f1" ], - "markers": "python_version < '3' and python_version >= '2.6'", + "markers": "python_version < '3.2'", "version": "==3.2.0" }, "idna": { "hashes": [ - "sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e", - "sha256:684a38a6f903c1d71d6d5fac066b58d7768af4de2b832e426ec79c30daa94a16" + "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", + "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c" ], - "version": "==2.7" + "version": "==2.8" }, "imagesize": { "hashes": [ "sha256:3f349de3eb99145973fefb7dbe38554414e5c30abd0c8e4b970a7c9d09f3a1d8", "sha256:f3832918bc3c66617f92e35f5d70729187676313caa60c187eb0f28b8fe5e3b5" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.1.0" }, + "importlib-metadata": { + "hashes": [ + "sha256:6dfd58dfe281e8d240937776065dd3624ad5469c835248219bd16cf2e12dbeb7", + "sha256:cb6ee23b46173539939964df59d3d72c3e0c1b5d54b84f1d8a7e912fe43612db" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.18" + }, "incremental": { "hashes": [ "sha256:717e12246dddf231a349175f48d74d93e2897244939173b01974ab6661406b9f", @@ -316,72 +289,72 @@ "sha256:dc492f8f17a0746e92081aec3f86ae0b4750bf41607ea2ad87e5a7b5705121b7", "sha256:eb6f9262d4d25b40330fb21d1e99bf0f85011ccc3526980f8a3eaedd4b43892e" ], - "index": "pypi", "version": "==1.2.0" }, "isort": { "hashes": [ - "sha256:1153601da39a25b14ddc54955dbbacbb6b2d19135386699e2ad58517953b34af", - "sha256:b9c40e9750f3d77e6e4d441d8b0266cf555e7cdabdcff33c4fd06366ca761ef8", - "sha256:ec9ef8f4a9bc6f71eec99e1806bfa2de401650d996c59330782b89a5555c1497" + "sha256:c40744b6bc5162bbb39c1257fe298b7a393861d50978b565f3ccd9cb9de0182a", + "sha256:f57abacd059dc3bd666258d1efb0377510a89777fda3e3274e3c01f7c03ae22d" ], "index": "pypi", - "version": "==4.3.4" + "version": "==4.3.20" }, "itsdangerous": { "hashes": [ "sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19", "sha256:b12271b2047cb23eeb98c8b5622e2e5c5e9abd9784a153e9d8ef9cb4dd09d749" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.1.0" }, "jedi": { "hashes": [ - "sha256:0191c447165f798e6a730285f2eee783fff81b0d3df261945ecb80983b5c3ca7", - "sha256:b7493f73a2febe0dc33d51c99b474547f7f6c0b2c8fb2b21f453eef204c12148" + "sha256:2bb0603e3506f708e792c7f4ad8fc2a7a9d9c2d292a358fbbd58da531695595b", + "sha256:2c6bcd9545c7d6440951b12b44d373479bf18123a401a52025cf98563fbd826c" ], "index": "pypi", - "version": "==0.13.1" + "version": "==0.13.3" }, "jinja2": { "hashes": [ - "sha256:74c935a1b8bb9a3947c50a54766a969d4846290e1e788ea44c1392163723c3bd", - "sha256:f84be1bb0040caca4cea721fcbbbbd61f9be9464ca236387158b0feea01914a4" + "sha256:065c4f02ebe7f7cf559e49ee5a95fb800a9e4528727aec6f24402a5374c65013", + "sha256:14dd6caf1527abb21f08f86c784eac40853ba93edb79552aa1e4b8aef1b61c7b" ], - "version": "==2.10" + "version": "==2.10.1" }, "markupsafe": { "hashes": [ - "sha256:048ef924c1623740e70204aa7143ec592504045ae4429b59c30054cb31e3c432", - "sha256:130f844e7f5bdd8e9f3f42e7102ef1d49b2e6fdf0d7526df3f87281a532d8c8b", - "sha256:19f637c2ac5ae9da8bfd98cef74d64b7e1bb8a63038a3505cd182c3fac5eb4d9", - "sha256:1b8a7a87ad1b92bd887568ce54b23565f3fd7018c4180136e1cf412b405a47af", - "sha256:1c25694ca680b6919de53a4bb3bdd0602beafc63ff001fea2f2fc16ec3a11834", - "sha256:1f19ef5d3908110e1e891deefb5586aae1b49a7440db952454b4e281b41620cd", - "sha256:1fa6058938190ebe8290e5cae6c351e14e7bb44505c4a7624555ce57fbbeba0d", - "sha256:31cbb1359e8c25f9f48e156e59e2eaad51cd5242c05ed18a8de6dbe85184e4b7", - "sha256:3e835d8841ae7863f64e40e19477f7eb398674da6a47f09871673742531e6f4b", - "sha256:4e97332c9ce444b0c2c38dd22ddc61c743eb208d916e4265a2a3b575bdccb1d3", - "sha256:525396ee324ee2da82919f2ee9c9e73b012f23e7640131dd1b53a90206a0f09c", - "sha256:52b07fbc32032c21ad4ab060fec137b76eb804c4b9a1c7c7dc562549306afad2", - "sha256:52ccb45e77a1085ec5461cde794e1aa037df79f473cbc69b974e73940655c8d7", - "sha256:5c3fbebd7de20ce93103cb3183b47671f2885307df4a17a0ad56a1dd51273d36", - "sha256:5e5851969aea17660e55f6a3be00037a25b96a9b44d2083651812c99d53b14d1", - "sha256:5edfa27b2d3eefa2210fb2f5d539fbed81722b49f083b2c6566455eb7422fd7e", - "sha256:7d263e5770efddf465a9e31b78362d84d015cc894ca2c131901a4445eaa61ee1", - "sha256:83381342bfc22b3c8c06f2dd93a505413888694302de25add756254beee8449c", - "sha256:857eebb2c1dc60e4219ec8e98dfa19553dae33608237e107db9c6078b1167856", - "sha256:98e439297f78fca3a6169fd330fbe88d78b3bb72f967ad9961bcac0d7fdd1550", - "sha256:bf54103892a83c64db58125b3f2a43df6d2cb2d28889f14c78519394feb41492", - "sha256:d9ac82be533394d341b41d78aca7ed0e0f4ba5a2231602e2f05aa87f25c51672", - "sha256:e982fe07ede9fada6ff6705af70514a52beb1b2c3d25d4e873e82114cf3c5401", - "sha256:edce2ea7f3dfc981c4ddc97add8a61381d9642dc3273737e756517cc03e84dd6", - "sha256:efdc45ef1afc238db84cb4963aa689c0408912a0239b0721cb172b4016eb31d6", - "sha256:f137c02498f8b935892d5c0172560d7ab54bc45039de8805075e19079c639a9c", - "sha256:f82e347a72f955b7017a39708a3667f106e6ad4d10b25f237396a7115d8ed5fd", - "sha256:fb7c206e01ad85ce57feeaaa0bf784b97fa3cad0d4a5737bc5295785f5c613a1" + "sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473", + "sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161", + "sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235", + "sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5", + "sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff", + "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b", + "sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1", + "sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e", + "sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183", + "sha256:535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66", + "sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1", + "sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1", + "sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e", + "sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b", + "sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905", + "sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735", + "sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d", + "sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e", + "sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d", + "sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c", + "sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21", + "sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2", + "sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5", + "sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b", + "sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6", + "sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f", + "sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f", + "sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7" ], - "version": "==1.1.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.1.1" }, "mccabe": { "hashes": [ @@ -392,153 +365,149 @@ }, "mock": { "hashes": [ - "sha256:5ce3c71c5545b472da17b72268978914d0252980348636840bd34a00b5cc96c1", - "sha256:b158b6df76edd239b8208d481dc46b6afd45a846b7812ff0ce58971cf5bc8bba" + "sha256:83657d894c90d5681d62155c82bda9c1187827525880eda8ff5df4ec813437c3", + "sha256:d157e52d4e5b938c550f39eb2fd15610db062441a9c2747d3dbfa9298211d0f8" ], - "index": "pypi", - "version": "==2.0.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==3.0.5" }, "more-itertools": { "hashes": [ - "sha256:c187a73da93e7a8acc0001572aebc7e3c69daf7bf6881a2cea10650bd4420092", - "sha256:c476b5d3a34e12d40130bc2f935028b5f636df8f372dc2c1c01dc19681b2039e", - "sha256:fcbfeaea0be121980e15bc97b3817b5202ca73d0eae185b4550cbfce2a3ebb3d" + "sha256:38a936c0a6d98a38bcc2d03fdaaedaba9f412879461dd2ceff8d37564d6522e4", + "sha256:c0a5785b1109a6bd7fac76d6837fd1feca158e54e521ccd2ae8bfe393cc9d4fc", + "sha256:fe7a7cae1ccb57d33952113ff4fa1bc5f879963600ed74918f1236e212ee50b9" ], - "version": "==4.3.0" + "version": "==5.0.0" + }, + "orderedmultidict": { + "hashes": [ + "sha256:24e3b730cf84e4a6a68be5cc760864905cf66abc89851e724bd5b4e849eaa96b", + "sha256:b89895ba6438038d0bdf88020ceff876cf3eae0d5c66a69b526fab31125db2c5" + ], + "version": "==1.0" }, "packaging": { "hashes": [ - "sha256:0886227f54515e592aaa2e5a553332c73962917f2831f1b0f9b9f4380a4b9807", - "sha256:f95a1e147590f204328170981833854229bb2912ac3d5f89e2a8ccd2834800c9" + "sha256:0c98a5d0be38ed775798ece1b9727178c4469d9c3b4ada66e8e6b7849f8732af", + "sha256:9e1cbf8c12b1f1ce0bb5344b8d7ecf66a6f8a6e91bcb0c84593ed6d3ab5c4ab3" ], - "version": "==18.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==19.0" }, "parso": { "hashes": [ - "sha256:35704a43a3c113cce4de228ddb39aab374b8004f4f2407d070b6a2ca784ce8a2", - "sha256:895c63e93b94ac1e1690f5fdd40b65f07c8171e3e53cbd7793b5b96c0e0a7f24" + "sha256:5052bb33be034cba784193e74b1cde6ebf29ae8b8c1e4ad94df0c4209bfc4826", + "sha256:db5881df1643bf3e66c097bfd8935cf03eae73f4cb61ae4433c9ea4fb6613446" ], - "version": "==0.3.1" + "version": "==0.5.0" }, "parver": { "hashes": [ - "sha256:ac4afff688d19d5e1876bb68d4bccc1a1b6a5cc8bd6a646939a14d366695ba15", - "sha256:f025fba8f88a9c776971df6d62b6cf7f37d1108f84c163bda91e157d7d527075" + "sha256:1b37a691af145a3a193eff269d53ba5b2ab16dfbb65d47d85360755919f5fe4b", + "sha256:72d056b8f8883ac90eef5554a9c8a47fac39d3b66479f3d2c8d5bc21b849cdba" ], - "index": "pypi", - "version": "==0.1.1" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.2.1" }, "passa": { - "editable": true, "git": "https://github.com/sarugaku/passa.git", - "ref": "4f3b8102f122cf0b75e5d7c513a2e61b0b093dcd" + "ref": "a2ba0b30c86339cae5ef3a03046fc9c583452c40", + "version": "==0.3.1.dev0" + }, + "pathlib2": { + "hashes": [ + "sha256:25199318e8cc3c25dcb45cbe084cc061051336d5a9ea2a12448d3d8cb748f742", + "sha256:5887121d7f7df3603bca2f710e7219f3eca0eb69e0b7cc6e0a022e155ac931a7" + ], + "markers": "python_version < '3.6'", + "version": "==2.3.3" }, "pbr": { "hashes": [ - "sha256:f59d71442f9ece3dffc17bc36575768e1ee9967756e6b6535f0ee1f0054c3d68", - "sha256:f6d5b23f226a2ba58e14e49aa3b1bfaf814d0199144b95d78458212444de1387" + "sha256:9181e2a34d80f07a359ff1d0504fad3a47e00e1cf2c475b0aa7dcb030af54c40", + "sha256:94bdc84da376b3dd5061aa0c3b6faffe943ee2e56fa4ff9bd63e1643932f34fc" ], - "version": "==5.1.1" - }, - "pep517": { - "hashes": [ - "sha256:cc663a438fdfe2e88d8d3c5ef2203ac858de34e31b6609b1fc505d611490a926", - "sha256:f79bb08fb064dfc5b141204bfeb56a4141a6d504677fab4723036a464fc25cc1" - ], - "version": "==0.3" - }, - "pip-shims": { - "hashes": [ - "sha256:3bc24ec050a6b9eea35419467237e4f47eaf806dadc9999bf887355c377edea7", - "sha256:edb4cf3c509eab2f36b55c1ac1a59a4c485ccd537cc87934d74950880f641256" - ], - "version": "==0.3.2" + "version": "==5.3.1" }, "pipenv": { "editable": true, + "extras": [ + "dev", + "tests" + ], "path": "." }, "pkginfo": { "hashes": [ - "sha256:5878d542a4b3f237e359926384f1dde4e099c9f5525d236b1840cf704fa8d474", - "sha256:a39076cb3eb34c333a0dd390b568e9e1e881c7bf2cc0aee12120636816f55aee" + "sha256:7424f2c8511c186cd5424bbf31045b77435b37a8d604990b79d4e70d741148bb", + "sha256:a6d9e40ca61ad3ebd0b72fbadd4fba16e4c0e4df0428c041e01e06eb6ee71f32" ], - "version": "==1.4.2" - }, - "plette": { - "extras": [ - "validation" - ], - "hashes": [ - "sha256:c0e3553c1e581d8423daccbd825789c6e7f29b7d9e00e5331b12e1642a1a26d3", - "sha256:dde5d525cf5f0cbad4d938c83b93db17887918daf63c13eafed257c4f61b07b4" - ], - "version": "==0.2.2" + "version": "==1.5.0.1" }, "pluggy": { "hashes": [ - "sha256:447ba94990e8014ee25ec853339faf7b0fc8050cdc3289d4d71f7f410fb90095", - "sha256:bde19360a8ec4dfd8a20dcb811780a30998101f078fc7ded6162f0076f50508f" + "sha256:0825a152ac059776623854c1543d65a4ad408eb3d33ee114dff91e57ec6ae6fc", + "sha256:b9817417e95936bf75d85d3f8767f7df6cdde751fc40aed3bb3074cbcb77757c" ], - "version": "==0.8.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.12.0" }, "py": { "hashes": [ - "sha256:bf92637198836372b520efcba9e020c330123be8ce527e535d185ed4b6f45694", - "sha256:e76826342cefe3c3d5f7e8ee4316b80d1dd8a300781612ddbc765c17ba25a6c6" + "sha256:64f65755aee5b381cea27766a3a147c3f15b9b6b9ac88676de66ba2ae36793fa", + "sha256:dc639b046a6e2cff5bbe40194ad65936d6ba360b52b3c3fe1d08a82dd50b5e53" ], - "version": "==1.7.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.8.0" }, "pycodestyle": { "hashes": [ - "sha256:cbc619d09254895b0d12c2c691e237b2e91e9b2ecf5e84c26b35400f93dcfb83", - "sha256:cbfca99bd594a10f674d0cd97a3d802a1fdef635d4361e1a2658de47ed261e3a" + "sha256:95a2219d12372f05704562a14ec30bc76b05a5b297b21a5dfe3f6fac3491ae56", + "sha256:e40a936c9a450ad81df37f549d676d127b1b66000a6c500caa2b085bc0ca976c" ], - "version": "==2.4.0" - }, - "pycparser": { - "hashes": [ - "sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3" - ], - "version": "==2.19" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.5.0" }, "pyflakes": { "hashes": [ - "sha256:9a7662ec724d0120012f6e29d6248ae3727d821bba522a0e6b356eff19126a49", - "sha256:f661252913bc1dbe7fcfcbf0af0db3f42ab65aabd1a6ca68fe5d466bace94dae" + "sha256:17dbeb2e3f4d772725c777fabc446d5634d1038f234e77343108ce445ea69ce0", + "sha256:d976835886f8c5b31d47970ed689944a0262b5f3afa00a5a7b4dc81e5449f8a2" ], - "version": "==2.0.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.1.1" }, "pygments": { "hashes": [ - "sha256:78f3f434bcc5d6ee09020f92ba487f95ba50f1e3ef83ae96b9d5ffa1bab25c5d", - "sha256:dbae1046def0efb574852fab9e90209b23f556367b5a320c0bcb871c77c3e8cc" + "sha256:71e430bc85c88a430f000ac1d9b331d2407f681d6f6aec95e8bcfbc3df5b0127", + "sha256:881c4c157e45f30af185c1ffe8d549d48ac9127433f2c380c24b84572ad66297" ], - "version": "==2.2.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.4.2" }, "pyparsing": { "hashes": [ - "sha256:40856e74d4987de5d01761a22d1621ae1c7f8774585acae358aa5c5936c6c90b", - "sha256:f353aab21fd474459d97b709e527b5571314ee5f067441dc9f88e33eecd96592" + "sha256:1873c03321fc118f4e9746baf201ff990ceb915f433f23b395f5580d1840cb2a", + "sha256:9b6323ef4ab914af344ba97510e966d64ba91055d6b9afa6b30799340e89cc03" ], - "version": "==2.3.0" + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.4.0" }, "pytest": { "hashes": [ - "sha256:7e258ee50338f4e46957f9e09a0f10fb1c2d05493fa901d113a8dafd0790de4e", - "sha256:9332147e9af2dcf46cd7ceb14d5acadb6564744ddff1fe8c17f0ce60ece7d9a2" + "sha256:4a784f1d4f2ef198fe9b7aef793e9fa1a3b2f84e822d9b3a64a181293a572d45", + "sha256:926855726d8ae8371803f7b2e6ec0a69953d9c6311fa7c3b6c1b929ff92d27da" ], - "index": "pypi", - "version": "==3.8.2" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==4.6.3" }, "pytest-forked": { "hashes": [ - "sha256:e4500cd0509ec4a26535f7d4112a8cc0f17d3a41c29ffd4eab479d2a55b30805", - "sha256:f275cb48a73fc61a6710726348e1da6d68a978f0ec0c54ece5a5fae5977e5a08" + "sha256:5fe33fbd07d7b1302c95310803a5e5726a4ff7f19d5a542b7ce57c76fed8135f", + "sha256:d352aaced2ebd54d42a65825722cb433004b4446ab5d2044851d9cc7a00c9e38" ], - "version": "==0.2" + "version": "==1.0.2" }, - "pytest-pypy": { + "pytest-pypi": { "editable": true, "path": "./tests/pytest-pypi" }, @@ -547,30 +516,22 @@ "sha256:3b05ec931424bbe44e944726b68f7ef185bb6d25ce9ce21ac52c9af7ffa9b506", "sha256:ca063de56298034302f3cbce55c87a27d7bfa7af7de591cdb9ec6ce45fea5467" ], - "index": "pypi", "version": "==2.3" }, "pytest-xdist": { "hashes": [ - "sha256:06aa39361694c9365baaa03bec71159b59ad06c9826c6279ebba368cb3571561", - "sha256:1ef0d05c905cfa0c5442c90e9e350e65c6ada120e33a00a066ca51c89f5f869a" + "sha256:3489d91516d7847db5eaecff7a2e623dba68984835dbe6cedb05ae126c4fb17f", + "sha256:501795cb99e567746f30fe78850533d4cd500c93794128e6ab9988e92a17b1f8" ], - "index": "pypi", - "version": "==1.23.2" - }, - "pytoml": { - "hashes": [ - "sha256:ca2d0cb127c938b8b76a9a0d0f855cf930c1d50cc3a0af6d3595b566519a1013" - ], - "version": "==0.1.20" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.29.0" }, "pytz": { "hashes": [ - "sha256:a061aa0a9e06881eb8b3b2b43f05b9439d6583c206d0a6c340ff72a7b6669053", - "sha256:ffb9ef1de172603304d9d2819af6f5ece76f2e85ec10692a524dd876e72bf277" + "sha256:303879e36b721603cc54604edcac9d20401bdbe31e1e4fdee5b9f98d5d31dfda", + "sha256:d747dd3d23d77ef44c6a3526e274af6efeb0a6f1afd5a69ba4d5be4098c8e141" ], - "index": "pypi", - "version": "==2018.5" + "version": "==2019.1" }, "readme-renderer": { "hashes": [ @@ -581,45 +542,59 @@ }, "requests": { "hashes": [ - "sha256:65b3a120e4329e33c9889db89c80976c5272f56ea92d3e74da8a463992e3ff54", - "sha256:ea881206e59f41dbd0bd445437d792e43906703fff75ca8ff43ccdb11f33f263" + "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4", + "sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31" ], - "version": "==2.20.1" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.22.0" }, "requests-toolbelt": { "hashes": [ - "sha256:42c9c170abc2cacb78b8ab23ac957945c7716249206f90874651971a4acff237", - "sha256:f6a531936c6fa4c6cfce1b9c10d5c4f498d16528d2a54a22ca00011205a187b5" + "sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f", + "sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0" ], - "version": "==0.8.0" + "version": "==0.9.1" }, - "requirementslib": { + "retry": { "hashes": [ - "sha256:441a5bfa487d3f3f5fd5d81c27071d9fd36bb385f538b3a87d20556a80b76f76", - "sha256:89e1e02ff0b52ce9c610124eb990ae706e0aee08beef8c718e7b87e470cdceeb" + "sha256:ccddf89761fa2c726ab29391837d4327f819ea14d244c232a1d24c67a2f98606", + "sha256:f8bfa8b99b69c4506d6f5bd3b0aabf77f98cdb17f3c9fc3f5ca820033336fba4" ], - "version": "==1.3.1.post1" - }, - "resolvelib": { - "hashes": [ - "sha256:6c4c6690b0bdd78bcc002e1a5d1b6abbde58c694a6ea1838f165b20d2c943db7", - "sha256:8734e53271ef98f38a2c99324d5e7905bc00c97dc3fc5bb7d83c82a979e71c04" - ], - "version": "==0.2.2" + "version": "==0.9.2" }, "rope": { "hashes": [ - "sha256:a108c445e1cd897fe19272ab7877d172e7faf3d4148c80e7d20faba42ea8f7b2" + "sha256:6b728fdc3e98a83446c27a91fc5d56808a004f8beab7a31ab1d7224cecc7d969", + "sha256:c5c5a6a87f7b1a2095fb311135e2a3d1f194f5ecb96900fdd0a9100881f48aaf", + "sha256:f0dcf719b63200d492b85535ebe5ea9b29e0d0b8aebeb87fe03fc1a65924fdaf" ], "index": "pypi", - "version": "==0.11.0" + "version": "==0.14.0" + }, + "scandir": { + "hashes": [ + "sha256:2586c94e907d99617887daed6c1d102b5ca28f1085f90446554abf1faf73123e", + "sha256:2ae41f43797ca0c11591c0c35f2f5875fa99f8797cb1a1fd440497ec0ae4b022", + "sha256:2b8e3888b11abb2217a32af0766bc06b65cc4a928d8727828ee68af5a967fa6f", + "sha256:2c712840c2e2ee8dfaf36034080108d30060d759c7b73a01a52251cc8989f11f", + "sha256:4d4631f6062e658e9007ab3149a9b914f3548cb38bfb021c64f39a025ce578ae", + "sha256:67f15b6f83e6507fdc6fca22fedf6ef8b334b399ca27c6b568cbfaa82a364173", + "sha256:7d2d7a06a252764061a020407b997dd036f7bd6a175a5ba2b345f0a357f0b3f4", + "sha256:8c5922863e44ffc00c5c693190648daa6d15e7c1207ed02d6f46a8dcc2869d32", + "sha256:92c85ac42f41ffdc35b6da57ed991575bdbe69db895507af88b9f499b701c188", + "sha256:b24086f2375c4a094a6b51e78b4cf7ca16c721dcee2eddd7aa6494b42d6d519d", + "sha256:cb925555f43060a1745d0a321cca94bcea927c50114b623d73179189a4e100ac" + ], + "markers": "python_version < '3.5'", + "version": "==1.10.0" }, "six": { "hashes": [ - "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", - "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb" + "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", + "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" ], - "version": "==1.11.0" + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.12.0" }, "snowballstemmer": { "hashes": [ @@ -628,34 +603,41 @@ ], "version": "==1.2.1" }, + "soupsieve": { + "hashes": [ + "sha256:72b5f1aea9101cf720a36bb2327ede866fd6f1a07b1e87c92a1cc18113cbc946", + "sha256:e4e9c053d59795e440163733a7fec6c5972210e1790c507e4c7b051d6c5259de" + ], + "version": "==1.9.2" + }, "sphinx": { "hashes": [ - "sha256:11f271e7a9398385ed730e90f0bb41dc3815294bdcd395b46ed2d033bc2e7d87", - "sha256:4064ea6c56feeb268838cb8fbbee507d0c3d5d92fa63a7df935a916b52c9e2f5" + "sha256:9f3e17c64b34afc653d7c5ec95766e03043cc6d80b0de224f59b6b6e19d37c3c", + "sha256:c7658aab75c920288a8cf6f09f244c6cfdae30d82d803ac1634d9f223a80ca08" ], - "index": "pypi", - "version": "==1.5.5" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.8.5" }, "sphinx-click": { "hashes": [ - "sha256:0550d3e5dcd6244847bd0861ebe64101a2ef302913866e0ccd9095b2aa230051", - "sha256:404784f724504e3da2cb056767ba64955c4bfb9bfca8cfedd7142a962bafd70f" + "sha256:2c7847607d07bc0ddf28acff3aa639b2660d06c5d95d1efe89eca6494fc750de", + "sha256:814b2463b576dfafaf4a6f8ed9585f6d9696073ed5e4cca5b59d2dc9d29d3bc0" ], "index": "pypi", - "version": "==1.3.0" + "version": "==2.2.0" }, "sphinxcontrib-websupport": { "hashes": [ - "sha256:68ca7ff70785cbe1e7bccc71a48b5b6d965d79ca50629606c7861a21b206d9dd", - "sha256:9de47f375baf1ea07cdb3436ff39d7a9c76042c10a769c52353ec46e4e8fc3b9" + "sha256:1501befb0fdf1d1c29a800fdbf4ef5dc5369377300ddbdd16d2cd40e54c6eefc", + "sha256:e02f717baf02d0b6c3dd62cf81232ffca4c9d5c331e03766982e3ff9f1d2bc3f" ], - "version": "==1.1.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.1.2" }, "stdeb": { "hashes": [ "sha256:0ed2c2cc6b8ba21da7d646c6f37ca60b22e9e4950e3cec6bcd9c2e7e57e3747e" ], - "index": "pypi", "markers": "sys_platform == 'linux'", "version": "==0.8.5" }, @@ -666,105 +648,105 @@ ], "version": "==2.5" }, + "termcolor": { + "hashes": [ + "sha256:1d6d69ce66211143803fbc56652b41d73b4a400a2891d7bf7a1cdf4c02de613b" + ], + "version": "==1.1.0" + }, "toml": { "hashes": [ "sha256:229f81c57791a41d65e399fc06bf0848bab550a9dfd5ed66df18ce5f05e73d5c", - "sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e" + "sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e", + "sha256:f1db651f9657708513243e61e6cc67d101a39bad662eaa9b5546f789338e07a3" ], "version": "==0.10.0" }, - "tomlkit": { - "hashes": [ - "sha256:82a8fbb8d8c6af72e96ba00b9db3e20ef61be6c79082552c9363f4559702258b", - "sha256:a43e0195edc9b3c198cd4b5f0f3d427a395d47c4a76ceba7cc875ed030756c39" - ], - "version": "==0.5.2" - }, "towncrier": { - "editable": true, - "git": "https://github.com/hawkowl/towncrier.git", - "ref": "47754a607a9b03f06affaf167d65b990786aae25" + "hashes": [ + "sha256:48251a1ae66d2cf7e6fa5552016386831b3e12bb3b2d08eb70374508c17a8196", + "sha256:de19da8b8cb44f18ea7ed3a3823087d2af8fcf497151bb9fd1e1b092ff56ed8d" + ], + "version": "==19.2.0" }, "tqdm": { "hashes": [ - "sha256:3c4d4a5a41ef162dd61f1edb86b0e1c7859054ab656b2e7c7b77e7fbf6d9f392", - "sha256:5b4d5549984503050883bc126280b386f5f4ca87e6c023c5d015655ad75bdebb" + "sha256:14a285392c32b6f8222ecfbcd217838f88e11630affe9006cd0e94c7eff3cb61", + "sha256:25d4c0ea02a305a688e7e9c2cdc8f862f989ef2a4701ab28ee963295f5b109ab" ], - "version": "==4.28.1" + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==4.32.2" }, "twine": { "hashes": [ - "sha256:7d89bc6acafb31d124e6e5b295ef26ac77030bf098960c2a4c4e058335827c5c", - "sha256:fad6f1251195f7ddd1460cb76d6ea106c93adb4e56c41e0da79658e56e547d2c" + "sha256:0fb0bfa3df4f62076cab5def36b1a71a2e4acb4d1fa5c97475b048117b1a6446", + "sha256:d6c29c933ecfc74e9b1d9fa13aa1f87c5d5770e119f5a4ce032092f0ff5b14dc" ], - "index": "pypi", - "version": "==1.12.1" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.13.0" }, "typing": { "hashes": [ - "sha256:3a887b021a77b292e151afb75323dea88a7bc1b3dfa92176cff8e44c8b68bddf", - "sha256:b2c689d54e1144bbcfd191b0832980a21c2dbcf7b5ff7a66248a60c90e951eb8", - "sha256:d400a9344254803a2368533e4533a4200d21eb7b6b729c173bc38201a74db3f2" + "sha256:4027c5f6127a6267a435201981ba156de91ad0d1d98e9ddc2aa173453453492d", + "sha256:57dcf675a99b74d64dacf6fba08fb17cf7e3d5fdff53d4a30ea2a5e7e52543d4", + "sha256:a4c8473ce11a65999c8f59cb093e70686b6c84c98df58c1dae9b3b196089858a" ], "markers": "python_version < '3.5'", - "version": "==3.6.4" + "version": "==3.6.6" }, "urllib3": { "hashes": [ - "sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39", - "sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22" + "sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1", + "sha256:dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232" ], - "version": "==1.24.1" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4'", + "version": "==1.25.3" }, "virtualenv": { "hashes": [ - "sha256:686176c23a538ecc56d27ed9d5217abd34644823d6391cbeb232f42bf722baad", - "sha256:f899fafcd92e1150f40c8215328be38ff24b519cd95357fa6e78e006c7638208" + "sha256:b7335cddd9260a3dd214b73a2521ffc09647bde3e9457fcca31dc3be3999d04a", + "sha256:d28ca64c0f3f125f59cabf13e0a150e1c68e5eea60983cc4395d88c584495783" ], - "version": "==16.1.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==16.6.1" }, "virtualenv-clone": { "hashes": [ - "sha256:afce268508aa5596c90dda234abe345deebc401a57d287bcbd76baa140a1aa58" + "sha256:532f789a5c88adf339506e3ca03326f20ee82fd08ee5586b44dc859b5b4468c5", + "sha256:c88ae171a11b087ea2513f260cdac9232461d8e9369bcd1dc143fc399d220557" ], - "version": "==0.4.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.5.3" }, - "vistir": { - "extras": [ - "spinner" - ], + "wcwidth": { "hashes": [ - "sha256:851bd783f2b85a372e563db741dc689cb9263ce2e067e387facdca0c36b6a6ea", - "sha256:b38ffc8ef83f85d81b4efa4cd31ea3bcd37bdb2bc9e8da9f20a40859bc44b57e" + "sha256:3df37372226d6e63e1b1e1eda15c594bca98a22d33a23832a90998faa96bc65e", + "sha256:f4ebe71925af7b40a864553f761ed559b43544f8f71746c2d756c7fe788ade7c" ], - "version": "==0.2.4" + "version": "==0.1.7" }, "webencodings": { "hashes": [ - "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78" + "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78", + "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923" ], "version": "==0.5.1" }, "werkzeug": { "hashes": [ - "sha256:c3fd7a7d41976d9f44db327260e263132466836cef6f91512889ed60ad26557c", - "sha256:d5da73735293558eb1651ee2fddc4d0dedcfa06538b8813a2e20011583c9e49b" + "sha256:865856ebb55c4dcd0630cdd8f3331a1847a819dda7e8c750d3db6f2aa6c0209c", + "sha256:a0b915f0815982fb2a09161cb8f31708052d0951c3ba433ccc5e1aa276507ca6" ], - "version": "==0.14.1" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.15.4" }, - "wheel": { + "zipp": { "hashes": [ - "sha256:196c9842d79262bb66fcf59faa4bd0deb27da911dbc7c6cdca931080eb1f0783", - "sha256:c93e2d711f5f9841e17f53b0e6c0ff85593f3b416b6eec7a9452041a59a42688" + "sha256:8c1019c6aad13642199fbe458275ad6a84907634cc9f0989877ccc4a2840139d", + "sha256:ca943a7e809cc12257001ccfb99e3563da9af99d52f261725e96dfe0f9275bc3" ], - "version": "==0.32.2" - }, - "yaspin": { - "hashes": [ - "sha256:36fdccc5e0637b5baa8892fe2c3d927782df7d504e9020f40eb2c1502518aa5a", - "sha256:8e52bf8079a48e2a53f3dfeec9e04addb900c101d1591c85df69cf677d3237e7" - ], - "version": "==0.14.0" + "markers": "python_version >= '2.7'", + "version": "==0.5.1" } } } diff --git a/README.md b/README.md index 3b5b1efb..ea2bf074 100644 --- a/README.md +++ b/README.md @@ -3,9 +3,7 @@ Pipenv: Python Development Workflow for Humans [![image](https://img.shields.io/pypi/v/pipenv.svg)](https://python.org/pypi/pipenv) [![image](https://img.shields.io/pypi/l/pipenv.svg)](https://python.org/pypi/pipenv) -[![image](https://badge.buildkite.com/79c7eccf056b17c3151f3c4d0e4c4b8b724539d84f1e037b9b.svg?branch=master)](https://code.kennethreitz.org/source/pipenv/) -[![VSTS build status (Windows)](https://dev.azure.com/pypa/pipenv/_apis/build/status/pipenv%20CI%20(Windows)?branchName=master&label=Windows)](https://dev.azure.com/pypa/pipenv/_build/latest?definitionId=9&branchName=master) -[![VSTS build status (Linux)](https://dev.azure.com/pypa/pipenv/_apis/build/status/pipenv%20CI%20(Linux)?branchName=master&label=Linux)](https://dev.azure.com/pypa/pipenv/_build/latest?definitionId=10&branchName=master) +[![Azure Pipelines Build Status](https://dev.azure.com/pypa/pipenv/_apis/build/status/Pipenv%20CI?branchName=master)](https://dev.azure.com/pypa/pipenv/_build/latest?definitionId=16&branchName=master) [![image](https://img.shields.io/pypi/pyversions/pipenv.svg)](https://python.org/pypi/pipenv) [![image](https://img.shields.io/badge/Say%20Thanks-!-1EAEDB.svg)](https://saythanks.io/to/kennethreitz) @@ -13,14 +11,14 @@ Pipenv: Python Development Workflow for Humans **Pipenv** is a tool that aims to bring the best of all packaging worlds (bundler, composer, npm, cargo, yarn, etc.) to the Python world. -*Windows is a first--class citizen, in our world.* +*Windows is a first-class citizen, in our world.* It automatically creates and manages a virtualenv for your projects, as well as adds/removes packages from your `Pipfile` as you -install/uninstall packages. It also generates the ever--important +install/uninstall packages. It also generates the ever-important `Pipfile.lock`, which is used to produce deterministic builds. -![image](http://media.kennethreitz.com.s3.amazonaws.com/pipenv.gif) +![image](https://s3.amazonaws.com/media.kennethreitz.com/pipenv.gif) The problems that Pipenv seeks to solve are multi-faceted: @@ -46,11 +44,19 @@ If you\'re on MacOS, you can install Pipenv easily with Homebrew: $ brew install pipenv +Or, if you\'re using Debian Buster+: + + $ sudo apt install pipenv + Or, if you\'re using Fedora 28: $ sudo dnf install pipenv + +Or, if you\'re using FreeBSD: -Otherwise, refer to the [documentation](https://docs.pipenv.org/install/) for instructions. + # pkg install py36-pipenv + +Otherwise, refer to the [documentation](https://docs.pipenv.org/en/latest/install/#installing-pipenv) for instructions. ✨🍰✨ diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 00000000..4b4061b0 --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,110 @@ +name: Pipenv Build Rules +trigger: + batch: true + branches: + include: + - master + paths: + exclude: + - docs/* + - news/* + - peeps/* + - examples/* + - pytest.ini + - README.md + - pipenv/*.txt + - CHANGELOG.rst + - CONTRIBUTING.md + - CODE_OF_CONDUCT.md + - .gitignore + - .gitattributes + - .editorconfig + +variables: +- group: CI + +jobs: +- job: TestLinux + pool: + vmImage: 'Ubuntu-16.04' + strategy: + matrix: + Python27: + python.version: '2.7' + python.architecture: x64 + Python36: + python.version: '3.6' + python.architecture: x64 + Python37: + python.version: '3.7' + python.architecture: x64 + maxParallel: 4 + steps: + - template: .azure-pipelines/steps/reinstall-pythons.yml + - template: .azure-pipelines/steps/run-tests.yml + parameters: + vmImage: 'Ubuntu-16.04' + +- job: TestVendoring + pool: + vmImage: 'Ubuntu-16.04' + variables: + python.version: '3.7' + python.architecture: x64 + steps: + - template: .azure-pipelines/steps/reinstall-pythons.yml + - template: .azure-pipelines/steps/run-vendor-scripts.yml + parameters: + vmImage: 'Ubuntu-16.04' + +- job: TestPackaging + pool: + vmImage: 'Ubuntu-16.04' + variables: + python.version: '3.7' + python.architecture: x64 + steps: + - template: .azure-pipelines/steps/reinstall-pythons.yml + - template: .azure-pipelines/steps/build-package.yml + parameters: + vmImage: 'Ubuntu-16.04' + +- job: TestWindows + pool: + vmImage: windows-2019 + strategy: + matrix: + Python27: + python.version: '2.7' + python.architecture: x64 + Python36: + python.version: '3.6' + python.architecture: x64 + Python37: + python.version: '3.7' + python.architecture: x64 + maxParallel: 4 + steps: + - template: .azure-pipelines/steps/run-tests.yml + parameters: + vmImage: windows-2019 + +- job: TestMacOS + pool: + vmImage: macOS-10.13 + strategy: + matrix: + Python27: + python.version: '2.7' + python.architecture: x64 + Python36: + python.version: '3.6' + python.architecture: x64 + Python37: + python.version: '3.7' + python.architecture: x64 + maxParallel: 4 + steps: + - template: .azure-pipelines/steps/run-tests.yml + parameters: + vmImage: macOS-10.13 diff --git a/docs/_templates/hacks.html b/docs/_templates/hacks.html index 0ec542fa..9736d409 100644 --- a/docs/_templates/hacks.html +++ b/docs/_templates/hacks.html @@ -18,7 +18,6 @@ /* Remain Responsive! */ @media screen and (max-width: 1008px) { - div.sphinxsidebar {display: none;} div.document {width: 100%!important;} /* Have code blocks escape the document right-margin. */ diff --git a/docs/_templates/sidebarintro.html b/docs/_templates/sidebarintro.html index 23b91e7e..8390940e 100644 --- a/docs/_templates/sidebarintro.html +++ b/docs/_templates/sidebarintro.html @@ -8,12 +8,32 @@

+ + + +

Pipenv is a production-ready tool that aims to bring the best of all packaging worlds to the Python world. It harnesses Pipfile, pip, and virtualenv into one single command.

It features very pretty terminal colors.

-

Stay Informed

Receive updates on new releases and upcoming projects.

diff --git a/docs/_templates/sidebarlogo.html b/docs/_templates/sidebarlogo.html index 5107e4de..00fb20c4 100644 --- a/docs/_templates/sidebarlogo.html +++ b/docs/_templates/sidebarlogo.html @@ -8,6 +8,27 @@

+ + + +

Pipenv is a production-ready tool that aims to bring the best of all packaging worlds to the Python world. It harnesses Pipfile, pip, and virtualenv into one single command. diff --git a/docs/advanced.rst b/docs/advanced.rst index 236f8c12..4fab3b1f 100644 --- a/docs/advanced.rst +++ b/docs/advanced.rst @@ -71,6 +71,11 @@ Luckily - pipenv will hash your Pipfile *before* expanding environment variables (and, helpfully, will substitute the environment variables again when you install from the lock file - so no need to commit any secrets! Woo!) +If your credentials contain a special character, surround the references to the environment variables with quotation marks. For example, if your password contain a double quotation mark, surround the password variable with single quotation marks. Otherwise, you may get a ``ValueError, "No closing quotation"`` error while installing dependencies. :: + + [[source]] + url = "https://$USERNAME:'${PASSWORD}'@mypypi.example.com/simple" + ☤ Specifying Basically Anything ------------------------------- @@ -358,7 +363,9 @@ Pipenv supports creating custom shortcuts in the (optional) ``[scripts]`` sectio You can then run ``pipenv run `` in your terminal to run the command in the context of your pipenv virtual environment even if you have not activated the pipenv shell first. -For example, in your Pipfile:: +For example, in your Pipfile: + +.. code-block:: toml [scripts] printspam = "python -c \"print('I am a silly example, no one would need to do this')\"" @@ -369,18 +376,25 @@ And then in your terminal:: I am a silly example, no one would need to do this Commands that expect arguments will also work. -For example:: +For example: +.. code-block:: toml [scripts] echospam = "echo I am really a very silly example" +:: + $ pipenv run echospam "indeed" I am really a very silly example indeed ☤ Support for Environment Variables ----------------------------------- -Pipenv supports the usage of environment variables in values. For example:: +Pipenv supports the usage of environment variables in place of authentication fragments +in your Pipfile. These will only be parsed if they are present in the ``[[source]]`` +section. For example: + +.. code-block:: toml [[source]] url = "https://${PYPI_USERNAME}:${PYPI_PASSWORD}@my_private_repo.example.com/simple" @@ -395,6 +409,7 @@ Pipenv supports the usage of environment variables in values. For example:: records = "*" Environment variables may be specified as ``${MY_ENVAR}`` or ``$MY_ENVAR``. + On Windows, ``%MY_ENVAR%`` is supported in addition to ``${MY_ENVAR}`` or ``$MY_ENVAR``. .. _configuration-with-environment-variables: @@ -468,7 +483,7 @@ and the corresponding Makefile:: pipenv install --dev test: - pipenv run py.test tests + pipenv run pytest tests Tox Automation Project @@ -484,7 +499,7 @@ and external testing:: deps = pipenv commands= pipenv install --dev - pipenv run py.test tests + pipenv run pytest tests [testenv:flake8-py3] basepython = python3.4 @@ -493,7 +508,7 @@ and external testing:: pipenv run flake8 --version pipenv run flake8 setup.py docs project test -Pipenv will automatically use the virtualenv provided by ``tox``. If ``pipenv install --dev`` installs e.g. ``pytest``, then installed command ``py.test`` will be present in given virtualenv and can be called directly by ``py.test tests`` instead of ``pipenv run py.test tests``. +Pipenv will automatically use the virtualenv provided by ``tox``. If ``pipenv install --dev`` installs e.g. ``pytest``, then installed command ``pytest`` will be present in given virtualenv and can be called directly by ``pytest tests`` instead of ``pipenv run pytest tests``. You might also want to add ``--ignore-pipfile`` to ``pipenv install``, as to not accidentally modify the lock-file on each test run. This causes Pipenv diff --git a/docs/basics.rst b/docs/basics.rst index 4ba6116d..a78df254 100644 --- a/docs/basics.rst +++ b/docs/basics.rst @@ -127,6 +127,7 @@ Example Pipfile.lock - Do not keep ``Pipfile.lock`` in version control if multiple versions of Python are being targeted. - Specify your target Python version in your `Pipfile`'s ``[requires]`` section. Ideally, you should only have one target Python version, as this is a deployment tool. - ``pipenv install`` is fully compatible with ``pip install`` syntax, for which the full documentation can be found `here `_. +- Note that the ``Pipfile`` uses the `TOML Spec `_. @@ -203,11 +204,11 @@ To make inclusive or exclusive version comparisons you can use: :: $ pipenv install "requests<=2.13" # will install a version equal or lower than 2.13.0 $ pipenv install "requests>2.19" # will install 2.19.1 but not 2.19.0 -.. note:: The use of ``" "`` around the package and version specification is highly recommended +.. note:: The use of double quotes around the package and version specification (i.e. ``"requests>2.19"``) is highly recommended to avoid issues with `Input and output redirection `_ in Unix-based operating systems. -The use of ``~=`` is preferred over the ``==`` identifier as the former prevents pipenv from updating the packages: :: +The use of ``~=`` is preferred over the ``==`` identifier as the latter prevents pipenv from updating the packages: :: $ pipenv install "requests~=2.2" # locks the major version of the package (this is equivalent to using ==2.*) diff --git a/docs/conf.py b/docs/conf.py index f3d0b6e2..c5d6fbe0 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -18,6 +18,7 @@ # import os + # Path hackery to get current version number. here = os.path.abspath(os.path.dirname(__file__)) diff --git a/docs/dev/contributing.rst b/docs/dev/contributing.rst index d6b7e4da..0fb0c36f 100644 --- a/docs/dev/contributing.rst +++ b/docs/dev/contributing.rst @@ -245,10 +245,15 @@ Three ways of running the tests are as follows: 1. ``make test`` (which uses ``docker``) 2. ``./run-tests.sh`` or ``run-tests.bat`` -3. Using pipenv:: +3. Using pipenv: - pipenv install --dev - pipenv run pytest +.. code-block:: console + + $ git clone https://github.com/pypa/pipenv.git + $ cd pipenv + $ git submodule sync && git submodule update --init --recursive + $ pipenv install --dev + $ pipenv run pytest For the last two, it is important that your environment is setup correctly, and this may take some work, for example, on a specific Mac installation, the following diff --git a/docs/index.rst b/docs/index.rst index b6822409..0f3ae522 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -26,10 +26,12 @@ It automatically creates and manages a virtualenv for your projects, as well as Pipenv is primarily meant to provide users and developers of applications with an easy method to setup a working environment. For the distinction between libraries and applications and the usage of ``setup.py`` vs ``Pipfile`` to define dependencies, see :ref:`pipfile-vs-setuppy`. -.. raw:: html - - - +.. image:: https://s3.amazonaws.com/media.kennethreitz.com/pipenv.gif + :height: 341px + :width: 654px + :scale: 100 % + :alt: a short animation of pipenv at work + The problems that Pipenv seeks to solve are multi-faceted: - You no longer need to use ``pip`` and ``virtualenv`` separately. They work together. @@ -49,7 +51,7 @@ You can quickly play with Pipenv right in your browser: Install Pipenv Today! --------------------- -If you're on MacOS, you can install Pipenv easily with Homebrew:: +If you're on MacOS, you can install Pipenv easily with Homebrew. You can also use Linuxbrew on Linux using the same command:: $ brew install pipenv @@ -70,9 +72,6 @@ Otherwise, refer to the :ref:`installing-pipenv` chapter for instructions. User Testimonials ----------------- -**Jannis Leidel**, former pip maintainer— - *Pipenv is the porcelain I always wanted to build for pip. It fits my brain and mostly replaces virtualenvwrapper and manual pip calls for me. Use it.* - **David Gang**— *This package manager is really awesome. For the first time I know exactly what my dependencies are which I installed and what the transitive dependencies are. Combined with the fact that installs are deterministic, makes this package manager first class, like cargo*. diff --git a/docs/install.rst b/docs/install.rst index f529e249..b5acd312 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -52,16 +52,16 @@ check this by running:: $ pip --version pip 9.0.1 -If you installed Python from source, with an installer from `python.org`_, or -via `Homebrew`_ you should already have pip. If you're on Linux and installed +If you installed Python from source, with an installer from `python.org`_, via `Homebrew`_ or via `Linuxbrew`_ you should already have pip. If you're on Linux and installed using your OS package manager, you may have to `install pip `_ separately. -If you plan to install Pipenv using Homebrew you can skip this step. The -Homebrew installer takes care of pip for you. +If you plan to install Pipenv using Homebrew or Linuxbrew you can skip this step. The +Homebrew/Linuxbrew installer takes care of pip for you. .. _getting started tutorial: https://opentechschool.github.io/python-beginners/en/getting_started.html#what-is-python-exactly .. _python.org: https://python.org .. _Homebrew: https://brew.sh +.. _Linuxbrew: https://linuxbrew.sh/ .. _Installing Python: http://docs.python-guide.org/en/latest/starting/installation/ @@ -83,13 +83,13 @@ cases. ☤ Homebrew Installation of Pipenv --------------------------------- -Homebrew is a popular open-source package management system for macOS. +`Homebrew`_ is a popular open-source package management system for macOS. For Linux users, `Linuxbrew`_ is a Linux port of that. -Installing pipenv via Homebrew will keep pipenv and all of its dependencies in +Installing pipenv via Homebrew or Linuxbrew will keep pipenv and all of its dependencies in an isolated virtual environment so it doesn't interfere with the rest of your Python installation. -Once you have installed `Homebrew`_ simply run:: +Once you have installed Homebrew or Linuxbrew simply run:: $ brew install pipenv @@ -223,7 +223,7 @@ have access to your installed packages with ``$ pipenv shell``. ☤ Virtualenv mapping caveat -============ +=========================== - Pipenv automatically maps projects to their specific virtualenvs. - The virtualenv is stored globally with the name of the project’s root directory plus the hash of the full path to the project's root (e.g., ``my_project-a3de50``). diff --git a/docs/requirements.txt b/docs/requirements.txt index d35a9f32..ee44a2ca 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,28 +1,26 @@ alabaster==0.7.10 -Babel==2.5.0 -certifi==2017.7.27.1 +Babel==2.6.0 +certifi==2018.10.15 chardet==3.0.4 -click==6.7 +click==7.0 docutils==0.14 first==2.0.1 -idna==2.6 -imagesize==0.7.1 +idna==2.7 +imagesize==1.1.0 Jinja2==2.10 -MarkupSafe==1.0 -pbr==3.1.1 -pip-tools==1.9.0 +MarkupSafe==1.1.0 +pbr==5.1.1 -e . -Pygments==2.2.0 +Pygments==2.3.0 pythonz-bd==1.11.4 -pytz==2017.2 -requests==2.20.0 +pytz==2018.5 +requests==2.20.1 resumable-urlretrieve==0.1.5 -semver==2.7.8 -six==1.10.0 +six==1.11.0 snowballstemmer==1.2.1 Sphinx==1.6.3 -sphinx-click==1.2.0 -sphinxcontrib-websupport==1.0.1 -urllib3==1.22 -virtualenv==15.1.0 -virtualenv-clone==0.2.6 +sphinx-click==1.3.0 +sphinxcontrib-websupport==1.1.0 +urllib3==1.24.1 +virtualenv==16.1.0 +virtualenv-clone==0.4.0 diff --git a/news/2317.doc.rst b/news/2317.doc.rst new file mode 100644 index 00000000..ff56fe4d --- /dev/null +++ b/news/2317.doc.rst @@ -0,0 +1 @@ +Added documenation about variable expansion in ``Pipfile`` entries. diff --git a/news/2373.bugfix.rst b/news/2373.bugfix.rst new file mode 100644 index 00000000..9b42add1 --- /dev/null +++ b/news/2373.bugfix.rst @@ -0,0 +1 @@ +Raise `PipenvUsageError` when [[source]] does not contain url field. diff --git a/news/2553.behavior.rst b/news/2553.behavior.rst new file mode 100644 index 00000000..d66edfa2 --- /dev/null +++ b/news/2553.behavior.rst @@ -0,0 +1 @@ +Make conservative checks of known exceptions when subprocess returns output, so user won't see the whole traceback - just the error. \ No newline at end of file diff --git a/news/2722.bugfix.rst b/news/2722.bugfix.rst new file mode 100644 index 00000000..8c26df8d --- /dev/null +++ b/news/2722.bugfix.rst @@ -0,0 +1 @@ +Fixed a bug which caused editable package resolution to sometimes fail with an unhelpful setuptools-related error message. diff --git a/news/2783.bugfix.rst b/news/2783.bugfix.rst new file mode 100644 index 00000000..7fa3cfd1 --- /dev/null +++ b/news/2783.bugfix.rst @@ -0,0 +1,2 @@ +Fixed an issue which caused errors due to reliance on the system utilities ``which`` and ``where`` which may not always exist on some systems. +- Fixed a bug which caused periodic failures in python discovery when executables named ``python`` were not present on the target ``$PATH``. diff --git a/news/3053.bugfix.rst b/news/3053.bugfix.rst new file mode 100644 index 00000000..21134f59 --- /dev/null +++ b/news/3053.bugfix.rst @@ -0,0 +1 @@ +Dependency resolution now writes hashes for local and remote files to the lockfile. diff --git a/news/3071.bugfix.rst b/news/3071.bugfix.rst new file mode 100644 index 00000000..dd4145ea --- /dev/null +++ b/news/3071.bugfix.rst @@ -0,0 +1 @@ +Fixed a bug which prevented ``pipenv graph`` from correctly showing all dependencies when running from within ``pipenv shell``. diff --git a/news/3148.bugfix.rst b/news/3148.bugfix.rst new file mode 100644 index 00000000..1f0f4a62 --- /dev/null +++ b/news/3148.bugfix.rst @@ -0,0 +1 @@ +Fixed resolution of direct-url dependencies in ``setup.py`` files to respect ``PEP-508`` style URL dependencies. diff --git a/news/3148.feature.rst b/news/3148.feature.rst new file mode 100644 index 00000000..e33434db --- /dev/null +++ b/news/3148.feature.rst @@ -0,0 +1 @@ +Added support for resolution of direct-url dependencies in ``setup.py`` files to respect ``PEP-508`` style URL dependencies. diff --git a/news/3178.bugfix.rst b/news/3178.bugfix.rst deleted file mode 100644 index d3a4fd6c..00000000 --- a/news/3178.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Environment variables are expanded correctly before running scripts on POSIX. diff --git a/news/3222.bugfix.rst b/news/3222.bugfix.rst deleted file mode 100644 index 2f71f552..00000000 --- a/news/3222.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Pipenv will no longer disable user-mode installation when the ``--system`` flag is passed in. diff --git a/news/3223.bugfix.rst b/news/3223.bugfix.rst deleted file mode 100644 index 44f4a83e..00000000 --- a/news/3223.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed an issue with attempting to render unicode output in non-unicode locales. diff --git a/news/3224.bugfix.rst b/news/3224.bugfix.rst deleted file mode 100644 index e6eed912..00000000 --- a/news/3224.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed a bug which could cause failures to occur when parsing python entries from global pyenv version files. diff --git a/news/3230.bugfix.rst b/news/3230.bugfix.rst deleted file mode 100644 index f8295b55..00000000 --- a/news/3230.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed an issue which prevented the parsing of named extras sections from certain ``setup.py`` files. diff --git a/news/3231.bugfix.rst b/news/3231.bugfix.rst deleted file mode 100644 index 036b2c12..00000000 --- a/news/3231.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Correctly detect the virtualenv location inside an activated virtualenv. diff --git a/news/3239.bugfix.rst b/news/3239.bugfix.rst deleted file mode 100644 index 526e80e1..00000000 --- a/news/3239.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed a bug which caused spinner frames to be written to stdout during locking operations which could cause redirection pipes to fail. diff --git a/news/3240.bugfix.rst b/news/3240.bugfix.rst deleted file mode 100644 index 88375bf9..00000000 --- a/news/3240.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed a bug that editable pacakges can't be uninstalled correctly. diff --git a/news/3249.bugfix.rst b/news/3249.bugfix.rst deleted file mode 100644 index 26d708cb..00000000 --- a/news/3249.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Adding normal pep 508 compatible markers is now fully functional when using VCS dependencies. diff --git a/news/3254.bugfix.rst b/news/3254.bugfix.rst deleted file mode 100644 index 11b18201..00000000 --- a/news/3254.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Updated ``requirementslib`` and ``pythonfinder`` for multiple bugfixes. diff --git a/news/3257.bugfix.rst b/news/3257.bugfix.rst deleted file mode 100644 index c816c93a..00000000 --- a/news/3257.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed an issue where pipenv could crash when multiple pipenv processes attempted to create the same directory. diff --git a/news/3260.bugfix.rst b/news/3260.bugfix.rst deleted file mode 100644 index 4a835e6a..00000000 --- a/news/3260.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed an issue which sometimes prevented successful creation of project pipfiles. diff --git a/news/3292.trivial.rst b/news/3292.trivial.rst new file mode 100644 index 00000000..9cab5de1 --- /dev/null +++ b/news/3292.trivial.rst @@ -0,0 +1 @@ +Update pytest-pypi documentation not to be pytest-httpbin documentation. diff --git a/news/3298.bugfix.rst b/news/3298.bugfix.rst new file mode 100644 index 00000000..aa378723 --- /dev/null +++ b/news/3298.bugfix.rst @@ -0,0 +1,3 @@ +Fixed a bug which caused failures in warning reporting when running pipenv inside a virtualenv under some circumstances. + +- Fixed a bug with package discovery when running ``pipenv clean``. diff --git a/news/3298.feature.rst b/news/3298.feature.rst new file mode 100644 index 00000000..65a49424 --- /dev/null +++ b/news/3298.feature.rst @@ -0,0 +1,5 @@ +Added full support for resolution of all dependency types including direct URLs, zip archives, tarballs, etc. + +- Improved error handling and formatting. + +- Introduced improved cross platform stream wrappers for better ``stdout`` and ``stderr`` consistency. diff --git a/news/3298.vendor.rst b/news/3298.vendor.rst new file mode 100644 index 00000000..cab9a50b --- /dev/null +++ b/news/3298.vendor.rst @@ -0,0 +1,34 @@ +Updated vendored dependencies: + + - **attrs**: ``18.2.0`` => ``19.1.0`` + - **certifi**: ``2018.10.15`` => ``2019.3.9`` + - **cached_property**: ``1.4.3`` => ``1.5.1`` + - **cerberus**: ``1.2.0`` => ``1.3.1`` + - **click-completion**: ``0.5.0`` => ``0.5.1`` + - **colorama**: ``0.3.9`` => ``0.4.1`` + - **distlib**: ``0.2.8`` => ``0.2.9`` + - **idna**: ``2.7`` => ``2.8`` + - **jinja2**: ``2.10.0`` => ``2.10.1`` + - **markupsafe**: ``1.0`` => ``1.1.1`` + - **orderedmultidict**: ``(new)`` => ``1.0`` + - **packaging**: ``18.0`` => ``19.0`` + - **parse**: ``1.9.0`` => ``1.12.0`` + - **pathlib2**: ``2.3.2`` => ``2.3.3`` + - **pep517**: ``(new)`` => ``0.5.0`` + - **pexpect**: ``4.6.0`` => ``4.7.0`` + - **pipdeptree**: ``0.13.0`` => ``0.13.2`` + - **pyparsing**: ``2.2.2`` => ``2.3.1`` + - **python-dotenv**: ``0.9.1`` => ``0.10.2`` + - **pythonfinder**: ``1.1.10`` => ``1.2.1`` + - **pytoml**: ``(new)`` => ``0.1.20`` + - **requests**: ``2.20.1`` => ``2.21.0`` + - **requirementslib**: ``1.3.3`` => ``1.5.0`` + - **scandir**: ``1.9.0`` => ``1.10.0`` + - **shellingham**: ``1.2.7`` => ``1.3.1`` + - **six**: ``1.11.0`` => ``1.12.0`` + - **tomlkit**: ``0.5.2`` => ``0.5.3`` + - **urllib3**: ``1.24`` => ``1.25.2`` + - **vistir**: ``0.3.0`` => ``0.4.1`` + - **yaspin**: ``0.14.0`` => ``0.14.3`` + +- Removed vendored dependency **cursor**. diff --git a/news/3307.bugfix.rst b/news/3307.bugfix.rst new file mode 100644 index 00000000..0f095c1a --- /dev/null +++ b/news/3307.bugfix.rst @@ -0,0 +1 @@ +Quote command arguments with carets (``^``) on Windows to work around unintended shell escapes. diff --git a/news/3313.bugfix.rst b/news/3313.bugfix.rst new file mode 100644 index 00000000..2f7a6ffc --- /dev/null +++ b/news/3313.bugfix.rst @@ -0,0 +1 @@ +Handle alternate names for UTF-8 encoding. diff --git a/news/3318.bugfix.rst b/news/3318.bugfix.rst new file mode 100644 index 00000000..b56f75dd --- /dev/null +++ b/news/3318.bugfix.rst @@ -0,0 +1 @@ +Abort pipenv before adding the non-exist package to Pipfile. diff --git a/news/3324.bugfix.rst b/news/3324.bugfix.rst new file mode 100644 index 00000000..d13a8d46 --- /dev/null +++ b/news/3324.bugfix.rst @@ -0,0 +1 @@ +Don't normalize the package name user passes in. diff --git a/news/3328.feature.rst b/news/3328.feature.rst new file mode 100644 index 00000000..7e92d39f --- /dev/null +++ b/news/3328.feature.rst @@ -0,0 +1 @@ +Pipenv will now successfully recursively lock VCS sub-dependencies. diff --git a/news/3339.bugfix b/news/3339.bugfix new file mode 100644 index 00000000..8e67e36f --- /dev/null +++ b/news/3339.bugfix @@ -0,0 +1 @@ +Fix a bug where custom virtualenv can not be activated with pipenv shell diff --git a/news/3348.feature.rst b/news/3348.feature.rst new file mode 100644 index 00000000..50547a3d --- /dev/null +++ b/news/3348.feature.rst @@ -0,0 +1 @@ +Added support for ``--verbose`` output to ``pipenv run``. \ No newline at end of file diff --git a/news/3351.bugfix.rst b/news/3351.bugfix.rst new file mode 100644 index 00000000..d2d9c675 --- /dev/null +++ b/news/3351.bugfix.rst @@ -0,0 +1 @@ +Fix a bug that ``--site-packages`` flag is not recognized. diff --git a/news/3353.bugfix b/news/3353.bugfix new file mode 100644 index 00000000..23e2b6af --- /dev/null +++ b/news/3353.bugfix @@ -0,0 +1 @@ +Fix a bug where pipenv --clear is not working diff --git a/news/3362.trivial.rst b/news/3362.trivial.rst new file mode 100644 index 00000000..2216b071 --- /dev/null +++ b/news/3362.trivial.rst @@ -0,0 +1 @@ +The inline tables won't be rewritten now. diff --git a/news/3368.feature.rst b/news/3368.feature.rst new file mode 100644 index 00000000..a998fce1 --- /dev/null +++ b/news/3368.feature.rst @@ -0,0 +1 @@ +Pipenv will now discover and resolve the intrinsic dependencies of **all** VCS dependencies, whether they are editable or not, to prevent resolution conflicts. diff --git a/news/3384.bugfix.rst b/news/3384.bugfix.rst new file mode 100644 index 00000000..f85cd168 --- /dev/null +++ b/news/3384.bugfix.rst @@ -0,0 +1 @@ +Fix unhashable type error during ``$ pipenv install --selective-upgrade`` diff --git a/news/3386.behavior.rst b/news/3386.behavior.rst new file mode 100644 index 00000000..8ddc27c6 --- /dev/null +++ b/news/3386.behavior.rst @@ -0,0 +1 @@ +Do not touch Pipfile early and rely on it so that one can do ``pipenv sync`` without a Pipfile. diff --git a/news/3404.bugfix.rst b/news/3404.bugfix.rst new file mode 100644 index 00000000..fa678d6a --- /dev/null +++ b/news/3404.bugfix.rst @@ -0,0 +1 @@ +Fixed a keyerror which could occur when locking VCS dependencies in some cases. diff --git a/news/3427.bugfix.rst b/news/3427.bugfix.rst new file mode 100644 index 00000000..76aeb489 --- /dev/null +++ b/news/3427.bugfix.rst @@ -0,0 +1 @@ +Fixed a bug that ``ValidationError`` is thrown when some fields are missing in source section. diff --git a/news/3434.trivial.rst b/news/3434.trivial.rst new file mode 100644 index 00000000..622b52db --- /dev/null +++ b/news/3434.trivial.rst @@ -0,0 +1 @@ +Improve the error message when one tries to initialize a Pipenv project under ``/``. diff --git a/news/3446.trivial.rst b/news/3446.trivial.rst new file mode 100644 index 00000000..c3f6a000 --- /dev/null +++ b/news/3446.trivial.rst @@ -0,0 +1 @@ +Fixed the wrong order of old and new hashes in message. diff --git a/news/3449.bugfix.rst b/news/3449.bugfix.rst new file mode 100644 index 00000000..4ed07046 --- /dev/null +++ b/news/3449.bugfix.rst @@ -0,0 +1 @@ +Updated the index names in lock file when source name in Pipfile is changed. diff --git a/news/3479.bugfix.rst b/news/3479.bugfix.rst new file mode 100644 index 00000000..15e8e0f6 --- /dev/null +++ b/news/3479.bugfix.rst @@ -0,0 +1 @@ +Fixed an issue which caused ``pipenv install --help`` to show duplicate entries for ``--pre``. diff --git a/news/3499.doc.rst b/news/3499.doc.rst new file mode 100644 index 00000000..d98b0f15 --- /dev/null +++ b/news/3499.doc.rst @@ -0,0 +1 @@ +Replace (non-existant) video on docs index.rst with equivalent gif. diff --git a/news/3502.bugfix.rst b/news/3502.bugfix.rst new file mode 100644 index 00000000..2700a740 --- /dev/null +++ b/news/3502.bugfix.rst @@ -0,0 +1 @@ +Fix bug causing ``[SSL: CERTIFICATE_VERIFY_FAILED]`` when Pipfile ``[[source]]`` has verify_ssl=false and url with custom port. diff --git a/news/3522.doc.rst b/news/3522.doc.rst new file mode 100644 index 00000000..3d71061f --- /dev/null +++ b/news/3522.doc.rst @@ -0,0 +1 @@ +Clarify wording in Basic Usage example on using double quotes to escape shell redirection diff --git a/news/3527.doc.rst b/news/3527.doc.rst new file mode 100644 index 00000000..b6043a08 --- /dev/null +++ b/news/3527.doc.rst @@ -0,0 +1 @@ +Ensure docs show navigation on small-screen devices diff --git a/news/3537.bugfix.rst b/news/3537.bugfix.rst new file mode 100644 index 00000000..779b9d7c --- /dev/null +++ b/news/3537.bugfix.rst @@ -0,0 +1 @@ +Fix ``sync --sequential`` ignoring ``pip install`` errors and logs. diff --git a/news/3577.feature.rst b/news/3577.feature.rst new file mode 100644 index 00000000..7944c098 --- /dev/null +++ b/news/3577.feature.rst @@ -0,0 +1 @@ +Added a new environment variable, ``PIPENV_RESOLVE_VCS``, to toggle dependency resolution off for non-editable VCS, file, and URL based dependencies. diff --git a/news/3584.bugfix.rst b/news/3584.bugfix.rst new file mode 100644 index 00000000..09684d1d --- /dev/null +++ b/news/3584.bugfix.rst @@ -0,0 +1 @@ +Fix the issue that lock file can't be created when ``PIPENV_PIPFILE`` is not under working directory. diff --git a/news/3595.feature.rst b/news/3595.feature.rst new file mode 100644 index 00000000..30b755b9 --- /dev/null +++ b/news/3595.feature.rst @@ -0,0 +1 @@ +Added the ability for Windows users to enable emojis by setting ``PIPENV_HIDE_EMOJIS=0``. diff --git a/news/3621.trivial.rst b/news/3621.trivial.rst new file mode 100644 index 00000000..4d38a31e --- /dev/null +++ b/news/3621.trivial.rst @@ -0,0 +1 @@ +Removed unused vendored package shutilwhich diff --git a/news/3629.doc.rst b/news/3629.doc.rst new file mode 100644 index 00000000..4d878c40 --- /dev/null +++ b/news/3629.doc.rst @@ -0,0 +1 @@ +Added a link to the TOML Spec under General Recommendations & Version Control to clarify how Pipfiles should be written. diff --git a/news/3640.trivial.rst b/news/3640.trivial.rst new file mode 100644 index 00000000..eb9b718d --- /dev/null +++ b/news/3640.trivial.rst @@ -0,0 +1 @@ +Removed unused vendored package blindspin diff --git a/news/3644.trivial.rst b/news/3644.trivial.rst new file mode 100644 index 00000000..5a7db2e1 --- /dev/null +++ b/news/3644.trivial.rst @@ -0,0 +1 @@ +Use tablib instead of requests in tests to avoid failures when vendored diff --git a/news/3647.bugfix.rst b/news/3647.bugfix.rst new file mode 100644 index 00000000..cb64edc1 --- /dev/null +++ b/news/3647.bugfix.rst @@ -0,0 +1 @@ +Pipenv will no longer inadvertently set ``editable=True`` on all vcs dependencies. diff --git a/news/3652.feature.rst b/news/3652.feature.rst new file mode 100644 index 00000000..7e5becb9 --- /dev/null +++ b/news/3652.feature.rst @@ -0,0 +1 @@ +Allow overriding PIPENV_INSTALL_TIMEOUT environment variable (in seconds). diff --git a/news/3656.bugfix.rst b/news/3656.bugfix.rst new file mode 100644 index 00000000..58df2020 --- /dev/null +++ b/news/3656.bugfix.rst @@ -0,0 +1,2 @@ +The ``--keep-outdated`` argument to ``pipenv install`` and ``pipenv lock`` will now drop specifier constraints when encountering editable dependencies. +- In addition, ``--keep-outdated`` will retain specifiers that would otherwise be dropped from any entries that have not been updated. diff --git a/news/3669.trivial.rst b/news/3669.trivial.rst new file mode 100644 index 00000000..86ff9280 --- /dev/null +++ b/news/3669.trivial.rst @@ -0,0 +1 @@ +Allow KeyboardInterrupt to cancel test suite checks for working internet and ssh diff --git a/news/3684.trivial.rst b/news/3684.trivial.rst new file mode 100644 index 00000000..64561ec7 --- /dev/null +++ b/news/3684.trivial.rst @@ -0,0 +1 @@ +Cleaned up some conditional logic that would always evaluate ``True``. diff --git a/news/3711.trivial.rst b/news/3711.trivial.rst new file mode 100644 index 00000000..48c531b2 --- /dev/null +++ b/news/3711.trivial.rst @@ -0,0 +1 @@ +Add installation instructions for Debian Buster+ in README diff --git a/news/3724.trivial.rst b/news/3724.trivial.rst new file mode 100644 index 00000000..63a55013 --- /dev/null +++ b/news/3724.trivial.rst @@ -0,0 +1 @@ +Update pytest configuration to support pytest 4. diff --git a/news/3738.feature.rst b/news/3738.feature.rst new file mode 100644 index 00000000..bb8237e7 --- /dev/null +++ b/news/3738.feature.rst @@ -0,0 +1,3 @@ +Allow overriding PIP_EXISTS_ACTION evironment variable (value is passed to pip install). +Possible values here: https://pip.pypa.io/en/stable/reference/pip/#exists-action-option +Useful when you need to `PIP_EXISTS_ACTION=i` (ignore existing packages) - great for CI environments, where you need really fast setup. diff --git a/news/3745.bugfix.rst b/news/3745.bugfix.rst new file mode 100644 index 00000000..229047a4 --- /dev/null +++ b/news/3745.bugfix.rst @@ -0,0 +1 @@ +Normalize the package names to lowercase when comparing used and in-Pipfile packages. diff --git a/news/3753.trivial.rst b/news/3753.trivial.rst new file mode 100644 index 00000000..2ab71d38 --- /dev/null +++ b/news/3753.trivial.rst @@ -0,0 +1 @@ +Improve the error message of ``pipenv --py`` when virtualenv can't be found. diff --git a/news/3759.doc.rst b/news/3759.doc.rst new file mode 100644 index 00000000..5aebd29e --- /dev/null +++ b/news/3759.doc.rst @@ -0,0 +1 @@ +Updated the documentation with the new ``pytest`` entrypoint. diff --git a/news/3763.feature.rst b/news/3763.feature.rst new file mode 100644 index 00000000..544a1ace --- /dev/null +++ b/news/3763.feature.rst @@ -0,0 +1 @@ +Pipenv will no longer forcibly override ``PIP_NO_DEPS`` on all vcs and file dependencies as resolution happens on these in a pre-lock step. diff --git a/news/3766.bugfix.rst b/news/3766.bugfix.rst new file mode 100644 index 00000000..f7f7d304 --- /dev/null +++ b/news/3766.bugfix.rst @@ -0,0 +1 @@ +``pipenv update --outdated`` will now correctly handle comparisons between pre/post-releases and normal releases. diff --git a/news/3766.vendor.rst b/news/3766.vendor.rst new file mode 100644 index 00000000..16ebbed9 --- /dev/null +++ b/news/3766.vendor.rst @@ -0,0 +1 @@ +Updated ``pip_shims`` to support ``--outdated`` with new pip versions. diff --git a/news/3768.bugfix.rst b/news/3768.bugfix.rst new file mode 100644 index 00000000..8efe0197 --- /dev/null +++ b/news/3768.bugfix.rst @@ -0,0 +1 @@ +Fixed a ``KeyError`` which could occur when pinning outdated VCS dependencies via ``pipenv lock --keep-outdated``. diff --git a/news/3786.bugfix.rst b/news/3786.bugfix.rst new file mode 100644 index 00000000..210f7973 --- /dev/null +++ b/news/3786.bugfix.rst @@ -0,0 +1 @@ +Resolved an issue which caused resolution to fail when encountering poorly formatted ``python_version`` markers in ``setup.py`` and ``setup.cfg`` files. diff --git a/news/3794.bugfix.rst b/news/3794.bugfix.rst new file mode 100644 index 00000000..a2999fdd --- /dev/null +++ b/news/3794.bugfix.rst @@ -0,0 +1 @@ +Fix a bug that installation errors are displayed as a list. diff --git a/news/3809.bugfix.rst b/news/3809.bugfix.rst new file mode 100644 index 00000000..bd603aaf --- /dev/null +++ b/news/3809.bugfix.rst @@ -0,0 +1 @@ +Fixed several bugs which could prevent editable VCS dependencies from being installed into target environments, even when reporting successful installation. diff --git a/news/3810.feature.rst b/news/3810.feature.rst new file mode 100644 index 00000000..33503779 --- /dev/null +++ b/news/3810.feature.rst @@ -0,0 +1 @@ +Improved verbose logging output during ``pipenv lock`` will now stream output to the console while maintaining a spinner. diff --git a/news/3819.bugfix.rst b/news/3819.bugfix.rst new file mode 100644 index 00000000..a6e05fb5 --- /dev/null +++ b/news/3819.bugfix.rst @@ -0,0 +1 @@ +``pipenv check --system`` should find the correct Python interpreter when ``python`` does not exist on the system. diff --git a/news/3842.bugfix.rst b/news/3842.bugfix.rst new file mode 100644 index 00000000..fb21be89 --- /dev/null +++ b/news/3842.bugfix.rst @@ -0,0 +1 @@ +Resolve the symlinks when the path is absolute. diff --git a/peeps/PEEP-0004.md b/peeps/PEEP-0004.md new file mode 100644 index 00000000..aea8d00a --- /dev/null +++ b/peeps/PEEP-0004.md @@ -0,0 +1,9 @@ +## PEEP-003: Subcommands + +NOT YET ACCEPTED + +Pipenv will automatically run commands like "pipenv deploy" if the "pipenv-deploy" binary is available on the path. + +These subcommands cannot overwrite built-in commands. + +These subcommands will receive environment variables with contextual information. diff --git a/peeps/PEEP-003.md b/peeps/PEEP-003.md new file mode 100644 index 00000000..d0493eaa --- /dev/null +++ b/peeps/PEEP-003.md @@ -0,0 +1,9 @@ +# PEEP-003: Revocation of Power of BDFL + +**ACCEPTED** + +Pipenv will be governed by a board of maintainers (trusted collaborators to the project on GitHub), not a BDFL. + +The BDFL retains his title, however, revokes himself of his powers. + +PEEP approval will be determined by available members of the board of maintainers, in private or public channels. diff --git a/peeps/PEEP-005.md b/peeps/PEEP-005.md new file mode 100644 index 00000000..2cc0279f --- /dev/null +++ b/peeps/PEEP-005.md @@ -0,0 +1,65 @@ +# PEEP-005: Do Not Remove Entries from the Lockfile When Using `--keep-outdated` + +**PROPOSED** + +This PEEP describes a change that would retain entries in the Lockfile even if they were not returned during resolution when the user passes the `--keep-outdated` flag. + +☤ + +The `--keep-outdated` flag is currently provided by Pipenv for the purpose of holding back outdated dependencies (i.e. dependencies that are not newly introduced). This proposal attempts to identify the reasoning behind the flag and identifies a need for a project-wide scoping. Finally, this proposal outlines the expected behavior of `--keep-outdated` under the specified circumstances, as well as the required changes to achieve full implementation. + +## Retaining Outdated Dependencies + +The purpose of retaining outdated dependencies is to allow the user to introduce a new package to their environment with a minimal impact on their existing environment. In an effort to achieve this, `keep_outdated` was proposed as both a flag and a Pipfile setting [in this issue](https://github.com/pypa/pipenv/issues/1255#issuecomment-354585775), originally described as follows: + +> pipenv lock --keep-outdated to request a minimal update that only adjusts the lock file to account for Pipfile changes (additions, removals, and changes to version constraints)... and pipenv install --keep-outdated needed to request only the minimal changes required to satisfy the installation request + +However, the current implementation always fully re-locks, rather than only locking the new dependencies. As a result, dependencies in the `Pipfile.lock` with markers for a python version different from that of the running interpreter will be removed, even if they have nothing to do with the current changeset. For instance, say you have the following dependency in your `Pipfile.lock`: + +```json +{ + "default": { + "backports.weakref": { + "hashes": [...], + "version": "==1.5", + "markers": "python_version<='3.4'" + } + } +} +``` + +If this lockfile were to be re-generated with Python 3, even with `--keep-outdated`, this entry would be removed. This makes it very difficult to maintain lockfiles which are compatible across major python versions, yet all that would be required to correct this would be a tweak to the implementation of `keep-outdated`. I believe this was the goal to begin with, but I feel this behavior should be documented and clarified before moving forward. + +## Desired Behavior + +1. The only changes that should occur in `Pipfile.lock` when `--keep-outdated` is passed should be changes resulting from new packages added or pin changes in the project `Pipfile`; +2. Existing packages in the project `Pipfile.lock` should remain in place, even if they are not returned during resolution; +3. New dependencies should be written to the lockfile; +4. Conflicts should be resolved as outlined below. + +## Conflict Resolution + +If a conflict should occur due to the presence in the `Pipfile.lock` of a dependency of a new package, the following steps should be undertaken before alerting the user: + +1. Determine whether the previously locked version of the dependency meets the constraints required of the new package; if so, pin that version; +2. If the previously locked version is not present in the `Pipfile` and is not a dependency of any other dependencies (i.e. has no presence in `pipenv graph`, etc), update the lockfile with the new version; +3. If there is a new or existing dependency which has a conflict with existing entries in the lockfile, perform an intermediate resolution step by checking: + a. If the new dependency can be satisfied by existing installs; + b. Whether conflicts can be upgraded without affecting locked dependencies; + c. If locked dependencies must be upgraded, whether those dependencies ultimately have any dependencies in the `Pipfile`; + d. If a traversal up the graph lands in the `Pipfile`, create _abstract dependencies_ from the `Pipfile` entries and determine whether they will still be satisfied by the new version; + e. If a new pin is required, ensure that any subdependencies of the newly pinned dependencies are therefore also re-pinned (simply prefer the updated lockfile instead of the cached version); + +4. Raise an Exception alerting the user that they either need to do a full lock or manually pin a version. + +## Necessary Changes + +In order to make these changes, we will need to modify the dependency resolution process. Overall, locking will require the following implementation changes: + +1. The ability to restore any entries that would otherwise be removed when the `--keep-outdated` flag is passed. The process already provides a caching mechanism, so we simply need to restore missing cache keys; +2. Conflict resolution steps: + a. Check an abstract dependency/candidate against a lockfile entry; + b. Requirements mapping for each dependency in the environment to determine if a lockfile entry is a descendent of any other entries; + + +Author: Dan Ryan diff --git a/pipenv/__init__.py b/pipenv/__init__.py index 4d137e7f..31d49fc1 100644 --- a/pipenv/__init__.py +++ b/pipenv/__init__.py @@ -10,6 +10,7 @@ import warnings from .__version__ import __version__ + PIPENV_ROOT = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) PIPENV_VENDOR = os.sep.join([PIPENV_ROOT, "vendor"]) PIPENV_PATCHED = os.sep.join([PIPENV_ROOT, "patched"]) @@ -20,20 +21,11 @@ sys.path.insert(0, PIPENV_PATCHED) from pipenv.vendor.urllib3.exceptions import DependencyWarning from pipenv.vendor.vistir.compat import ResourceWarning, fs_str + warnings.filterwarnings("ignore", category=DependencyWarning) warnings.filterwarnings("ignore", category=ResourceWarning) warnings.filterwarnings("ignore", category=UserWarning) -if sys.version_info >= (3, 1) and sys.version_info <= (3, 6): - if sys.stdout.isatty() and sys.stderr.isatty(): - import io - import atexit - stdout_wrapper = io.TextIOWrapper(sys.stdout.buffer, encoding='utf8') - atexit.register(stdout_wrapper.close) - stderr_wrapper = io.TextIOWrapper(sys.stderr.buffer, encoding='utf8') - atexit.register(stderr_wrapper.close) - sys.stdout = stdout_wrapper - sys.stderr = stderr_wrapper os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = fs_str("1") @@ -44,6 +36,22 @@ try: except Exception: pass +from pipenv.vendor.vistir.misc import get_text_stream + +stdout = get_text_stream("stdout") +stderr = get_text_stream("stderr") + +if os.name == "nt": + from pipenv.vendor.vistir.misc import _can_use_color, _wrap_for_color + + if _can_use_color(stdout): + stdout = _wrap_for_color(stdout) + if _can_use_color(stderr): + stderr = _wrap_for_color(stderr) + +sys.stdout = stdout +sys.stderr = stderr + from .cli import cli from . import resolver diff --git a/pipenv/__main__.py b/pipenv/__main__.py index 56494106..491a4d13 100644 --- a/pipenv/__main__.py +++ b/pipenv/__main__.py @@ -1,4 +1,5 @@ from .cli import cli + if __name__ == "__main__": - cli(auto_envvar_prefix="PIPENV") + cli() diff --git a/pipenv/__version__.py b/pipenv/__version__.py index 802c3b9c..a745170a 100644 --- a/pipenv/__version__.py +++ b/pipenv/__version__.py @@ -2,4 +2,4 @@ # // ) ) / / // ) ) //___) ) // ) ) || / / # //___/ / / / //___/ / // // / / || / / # // / / // ((____ // / / ||/ / -__version__ = "2018.11.15.dev0" +__version__ = "2018.11.27.dev0" diff --git a/pipenv/_compat.py b/pipenv/_compat.py index 09f65669..caba80fe 100644 --- a/pipenv/_compat.py +++ b/pipenv/_compat.py @@ -5,39 +5,17 @@ Exposes a standard API that enables compatibility across python versions, operating systems, etc. """ -import functools -import importlib -import io import os -import six import sys import warnings + +import six import vistir -from tempfile import _bin_openflags, gettempdir, _mkstemp_inner, mkdtemp -try: - from tempfile import _infer_return_type -except ImportError: +from .vendor.vistir.compat import ( + NamedTemporaryFile, Path, ResourceWarning, TemporaryDirectory +) - def _infer_return_type(*args): - _types = set() - for arg in args: - if isinstance(type(arg), six.string_types): - _types.add(str) - elif isinstance(type(arg), bytes): - _types.add(bytes) - elif arg: - _types.add(type(arg)) - return _types.pop() - - -if sys.version_info[:2] >= (3, 5): - try: - from pathlib import Path - except ImportError: - from .vendor.pathlib2 import Path -else: - from .vendor.pathlib2 import Path # Backport required for earlier versions of Python. if sys.version_info < (3, 3): @@ -45,257 +23,14 @@ if sys.version_info < (3, 3): else: from shutil import get_terminal_size -try: - from weakref import finalize -except ImportError: - try: - from .vendor.backports.weakref import finalize - except ImportError: - - class finalize(object): - def __init__(self, *args, **kwargs): - from .utils import logging - logging.warn("weakref.finalize unavailable, not cleaning...") - - def detach(self): - return False - - -from vistir.compat import ResourceWarning - - warnings.filterwarnings("ignore", category=ResourceWarning) -class TemporaryDirectory(object): - - """ - Create and return a temporary directory. This has the same - behavior as mkdtemp but can be used as a context manager. For - example: - - with TemporaryDirectory() as tmpdir: - ... - - Upon exiting the context, the directory and everything contained - in it are removed. - """ - - def __init__(self, suffix="", prefix="", dir=None): - if "RAM_DISK" in os.environ: - import uuid - - name = uuid.uuid4().hex - dir_name = os.path.join(os.environ["RAM_DISK"].strip(), name) - os.mkdir(dir_name) - self.name = dir_name - else: - self.name = mkdtemp(suffix, prefix, dir) - self._finalizer = finalize( - self, - self._cleanup, - self.name, - warn_message="Implicitly cleaning up {!r}".format(self), - ) - - @classmethod - def _cleanup(cls, name, warn_message): - vistir.path.rmtree(name) - warnings.warn(warn_message, ResourceWarning) - - def __repr__(self): - return "<{} {!r}>".format(self.__class__.__name__, self.name) - - def __enter__(self): - return self - - def __exit__(self, exc, value, tb): - self.cleanup() - - def cleanup(self): - if self._finalizer.detach(): - vistir.path.rmtree(self.name) - - -def _sanitize_params(prefix, suffix, dir): - """Common parameter processing for most APIs in this module.""" - output_type = _infer_return_type(prefix, suffix, dir) - if suffix is None: - suffix = output_type() - if prefix is None: - if output_type is str: - prefix = "tmp" - else: - prefix = os.fsencode("tmp") - if dir is None: - if output_type is str: - dir = gettempdir() - else: - dir = os.fsencode(gettempdir()) - return prefix, suffix, dir, output_type - - -class _TemporaryFileCloser: - """ - A separate object allowing proper closing of a temporary file's - underlying file object, without adding a __del__ method to the - temporary file. - """ - - file = None # Set here since __del__ checks it - close_called = False - - def __init__(self, file, name, delete=True): - self.file = file - self.name = name - self.delete = delete - - # NT provides delete-on-close as a primitive, so we don't need - # the wrapper to do anything special. We still use it so that - # file.name is useful (i.e. not "(fdopen)") with NamedTemporaryFile. - if os.name != "nt": - - # Cache the unlinker so we don't get spurious errors at - # shutdown when the module-level "os" is None'd out. Note - # that this must be referenced as self.unlink, because the - # name TemporaryFileWrapper may also get None'd out before - # __del__ is called. - - def close(self, unlink=os.unlink): - if not self.close_called and self.file is not None: - self.close_called = True - try: - self.file.close() - finally: - if self.delete: - unlink(self.name) - - # Need to ensure the file is deleted on __del__ - - def __del__(self): - self.close() - - else: - - def close(self): - if not self.close_called: - self.close_called = True - self.file.close() - - -class _TemporaryFileWrapper: - - """ - Temporary file wrapper - This class provides a wrapper around files opened for - temporary use. In particular, it seeks to automatically - remove the file when it is no longer needed. - """ - - def __init__(self, file, name, delete=True): - self.file = file - self.name = name - self.delete = delete - self._closer = _TemporaryFileCloser(file, name, delete) - - def __getattr__(self, name): - # Attribute lookups are delegated to the underlying file - # and cached for non-numeric results - # (i.e. methods are cached, closed and friends are not) - file = self.__dict__["file"] - a = getattr(file, name) - if hasattr(a, "__call__"): - func = a - - @functools.wraps(func) - def func_wrapper(*args, **kwargs): - return func(*args, **kwargs) - - # Avoid closing the file as long as the wrapper is alive, - # see issue #18879. - func_wrapper._closer = self._closer - a = func_wrapper - if not isinstance(a, int): - setattr(self, name, a) - return a - - # The underlying __enter__ method returns the wrong object - # (self.file) so override it to return the wrapper - - def __enter__(self): - self.file.__enter__() - return self - - # Need to trap __exit__ as well to ensure the file gets - # deleted when used in a with statement - - def __exit__(self, exc, value, tb): - result = self.file.__exit__(exc, value, tb) - self.close() - return result - - def close(self): - """ - Close the temporary file, possibly deleting it. - """ - self._closer.close() - - # iter() doesn't use __getattr__ to find the __iter__ method - - def __iter__(self): - # Don't return iter(self.file), but yield from it to avoid closing - # file as long as it's being used as iterator (see issue #23700). We - # can't use 'yield from' here because iter(file) returns the file - # object itself, which has a close method, and thus the file would get - # closed when the generator is finalized, due to PEP380 semantics. - for line in self.file: - yield line - - -def NamedTemporaryFile( - mode="w+b", - buffering=-1, - encoding=None, - newline=None, - suffix=None, - prefix=None, - dir=None, - delete=True, -): - """ - Create and return a temporary file. - Arguments: - 'prefix', 'suffix', 'dir' -- as for mkstemp. - 'mode' -- the mode argument to io.open (default "w+b"). - 'buffering' -- the buffer size argument to io.open (default -1). - 'encoding' -- the encoding argument to io.open (default None) - 'newline' -- the newline argument to io.open (default None) - 'delete' -- whether the file is deleted on close (default True). - The file is created as mkstemp() would do it. - Returns an object with a file-like interface; the name of the file - is accessible as its 'name' attribute. The file will be automatically - deleted when it is closed unless the 'delete' argument is set to False. - """ - prefix, suffix, dir, output_type = _sanitize_params(prefix, suffix, dir) - flags = _bin_openflags - # Setting O_TEMPORARY in the flags causes the OS to delete - # the file when it is closed. This is only supported by Windows. - if os.name == "nt" and delete: - flags |= os.O_TEMPORARY - if sys.version_info < (3, 5): - (fd, name) = _mkstemp_inner(dir, prefix, suffix, flags) - else: - (fd, name) = _mkstemp_inner(dir, prefix, suffix, flags, output_type) - try: - file = io.open( - fd, mode, buffering=buffering, newline=newline, encoding=encoding - ) - return _TemporaryFileWrapper(file, name, delete) - - except BaseException: - os.unlink(name) - os.close(fd) - raise +__all__ = [ + "NamedTemporaryFile", "Path", "ResourceWarning", "TemporaryDirectory", + "get_terminal_size", "getpreferredencoding", "DEFAULT_ENCODING", "canonical_encoding_name", + "force_encoding", "UNICODE_TO_ASCII_TRANSLATION_MAP", "decode_output", "fix_utf8" +] def getpreferredencoding(): @@ -313,6 +48,16 @@ def getpreferredencoding(): DEFAULT_ENCODING = getpreferredencoding() +def canonical_encoding_name(name): + import codecs + try: + codec = codecs.lookup(name) + except LookupError: + return name + else: + return codec.name + + # From https://github.com/CarlFK/veyepar/blob/5c5de47/dj/scripts/fixunicode.py # MIT LIcensed, thanks Carl! def force_encoding(): @@ -324,20 +69,23 @@ def force_encoding(): else: if not (stdout_isatty() and stderr_isatty()): return DEFAULT_ENCODING, DEFAULT_ENCODING - stdout_encoding = sys.stdout.encoding - stderr_encoding = sys.stderr.encoding + stdout_encoding = canonical_encoding_name(sys.stdout.encoding) + stderr_encoding = canonical_encoding_name(sys.stderr.encoding) if sys.platform == "win32" and sys.version_info >= (3, 1): return DEFAULT_ENCODING, DEFAULT_ENCODING - if stdout_encoding.lower() != "utf-8" or stderr_encoding.lower() != "utf-8": + if stdout_encoding != "utf-8" or stderr_encoding != "utf-8": - from ctypes import pythonapi, py_object, c_char_p + try: + from ctypes import pythonapi, py_object, c_char_p + except ImportError: + return DEFAULT_ENCODING, DEFAULT_ENCODING try: PyFile_SetEncoding = pythonapi.PyFile_SetEncoding except AttributeError: return DEFAULT_ENCODING, DEFAULT_ENCODING else: PyFile_SetEncoding.argtypes = (py_object, c_char_p) - if stdout_encoding.lower() != "utf-8": + if stdout_encoding != "utf-8": try: was_set = PyFile_SetEncoding(sys.stdout, "utf-8") except OSError: @@ -347,7 +95,7 @@ def force_encoding(): else: stdout_encoding = "utf-8" - if stderr_encoding.lower() != "utf-8": + if stderr_encoding != "utf-8": try: was_set = PyFile_SetEncoding(sys.stderr, "utf-8") except OSError: @@ -366,11 +114,17 @@ OUT_ENCODING, ERR_ENCODING = force_encoding() UNICODE_TO_ASCII_TRANSLATION_MAP = { 8230: u"...", 8211: u"-", - 10004: u"x", - 10008: u"Ok" + 10004: u"OK", + 10008: u"x", } +def decode_for_output(output, target=sys.stdout): + return vistir.misc.decode_for_output( + output, sys.stdout, translation_map=UNICODE_TO_ASCII_TRANSLATION_MAP + ) + + def decode_output(output): if not isinstance(output, six.string_types): return output @@ -384,13 +138,11 @@ def decode_output(output): output = output.translate(UNICODE_TO_ASCII_TRANSLATION_MAP) output = output.encode(DEFAULT_ENCODING, "replace") return vistir.misc.to_text(output, encoding=DEFAULT_ENCODING, errors="replace") - return output def fix_utf8(text): if not isinstance(text, six.string_types): return text - from ._compat import decode_output try: text = decode_output(text) except UnicodeDecodeError: diff --git a/pipenv/cli/__init__.py b/pipenv/cli/__init__.py index 605f4c10..d1819953 100644 --- a/pipenv/cli/__init__.py +++ b/pipenv/cli/__init__.py @@ -1,3 +1,4 @@ # -*- coding=utf-8 -*- from __future__ import absolute_import + from .command import cli diff --git a/pipenv/cli/command.py b/pipenv/cli/command.py index 720ce753..04748456 100644 --- a/pipenv/cli/command.py +++ b/pipenv/cli/command.py @@ -4,39 +4,41 @@ from __future__ import absolute_import import os import sys -import crayons -import delegator - from click import ( argument, echo, edit, group, option, pass_context, secho, version_option ) -import click_completion - -from click_didyoumean import DYMCommandCollection +from ..vendor import click_completion +from ..vendor import delegator +from ..patched import crayons from ..__version__ import __version__ from .options import ( CONTEXT_SETTINGS, PipenvGroup, code_option, common_options, deploy_option, - general_options, install_options, lock_options, pass_state, skip_lock_option, - pypi_mirror_option, python_option, requirementstxt_option, sync_options, - system_option, three_option, verbose_option, uninstall_options + general_options, install_options, lock_options, pass_state, + pypi_mirror_option, python_option, requirementstxt_option, + skip_lock_option, sync_options, system_option, three_option, + uninstall_options, verbose_option ) # Enable shell completion. click_completion.init() +subcommand_context = CONTEXT_SETTINGS.copy() +subcommand_context.update({ + "ignore_unknown_options": True, + "allow_extra_args": True +}) +subcommand_context_no_interspersion = subcommand_context.copy() +subcommand_context_no_interspersion["allow_interspersed_args"] = False + @group(cls=PipenvGroup, invoke_without_command=True, context_settings=CONTEXT_SETTINGS) @option("--where", is_flag=True, default=False, help="Output project home information.") @option("--venv", is_flag=True, default=False, help="Output virtualenv information.") -@option( - "--py", is_flag=True, default=False, help="Output Python interpreter information." -) -@option( - "--envs", is_flag=True, default=False, help="Output Environment Variable options." -) +@option("--py", is_flag=True, default=False, help="Output Python interpreter information.") +@option("--envs", is_flag=True, default=False, help="Output Environment Variable options.") @option("--rm", is_flag=True, default=False, help="Remove the virtualenv.") @option("--bare", is_flag=True, default=False, help="Minimal output.") @option( @@ -60,19 +62,15 @@ def cli( state, where=False, venv=False, + py=False, + envs=False, rm=False, bare=False, - three=False, - python=False, - help=False, - py=False, - site_packages=False, - envs=False, - man=False, completion=False, - pypi_mirror=None, + man=False, support=None, - clear=False, + help=False, + site_packages=False, **kwargs ): # Handle this ASAP to make shell startup fast. @@ -142,16 +140,19 @@ def cli( get_pipenv_diagnostics() return 0 # --clear was passed… - elif clear: + elif state.clear: do_clear() return 0 - # --venv was passed… elif venv: # There is no virtualenv yet. if not project.virtualenv_exists: echo( - crayons.red("No virtualenv has been created for this project yet!"), + "{}({}){}".format( + crayons.red("No virtualenv has been created for this project"), + crayons.white(project.project_directory, bold=True), + crayons.red(" yet!") + ), err=True, ) ctx.abort() @@ -194,7 +195,7 @@ def cli( ) ctx.abort() # --two / --three was passed… - if (state.python or state.three is not None) or site_packages: + if (state.python or state.three is not None) or state.site_packages: ensure_project( three=state.three, python=state.python, @@ -211,7 +212,7 @@ def cli( @cli.command( short_help="Installs provided packages and adds them to Pipfile, or (if no packages are given), installs all packages from Pipfile.", - context_settings=dict(ignore_unknown_options=True, allow_extra_args=True), + context_settings=subcommand_context, ) @system_option @code_option @@ -253,8 +254,10 @@ def install( ctx.abort() -@cli.command(short_help="Un-installs a provided package and removes it from Pipfile.") -@option("--skip-lock/--lock", is_flag=True, default=False, help="Lock afterwards.") +@cli.command( + short_help="Un-installs a provided package and removes it from Pipfile.", + context_settings=subcommand_context +) @option( "--all-dev", is_flag=True, @@ -273,7 +276,6 @@ def install( def uninstall( ctx, state, - skip_lock=False, all_dev=False, all=False, **kwargs @@ -296,7 +298,8 @@ def uninstall( if retcode: sys.exit(retcode) -@cli.command(short_help="Generates Pipfile.lock.") + +@cli.command(short_help="Generates Pipfile.lock.", context_settings=CONTEXT_SETTINGS) @lock_options @pass_state @pass_context @@ -309,7 +312,10 @@ def lock( from ..core import ensure_project, do_init, do_lock # Ensure that virtualenv is available. - ensure_project(three=state.three, python=state.python, pypi_mirror=state.pypi_mirror) + ensure_project( + three=state.three, python=state.python, pypi_mirror=state.pypi_mirror, + warn=(not state.quiet) + ) if state.installstate.requirementstxt: do_init( dev=state.installstate.dev, @@ -323,12 +329,13 @@ def lock( pre=state.installstate.pre, keep_outdated=state.installstate.keep_outdated, pypi_mirror=state.pypi_mirror, + write=not state.quiet, ) @cli.command( short_help="Spawns a shell within the virtualenv.", - context_settings=dict(ignore_unknown_options=True, allow_extra_args=True), + context_settings=subcommand_context, ) @option( "--fancy", @@ -387,11 +394,7 @@ def shell( @cli.command( add_help_option=False, short_help="Spawns a command installed into the virtualenv.", - context_settings=dict( - ignore_unknown_options=True, - allow_interspersed_args=False, - allow_extra_args=True, - ), + context_settings=subcommand_context_no_interspersion, ) @common_options @argument("command") @@ -400,7 +403,6 @@ def shell( def run(state, command, args): """Spawns a command installed into the virtualenv.""" from ..core import do_run - do_run( command=command, args=args, three=state.three, python=state.python, pypi_mirror=state.pypi_mirror ) @@ -408,7 +410,7 @@ def run(state, command, args): @cli.command( short_help="Checks for security vulnerabilities and against PEP 508 markers provided in Pipfile.", - context_settings=dict(ignore_unknown_options=True, allow_extra_args=True), + context_settings=subcommand_context ) @option( "--unused", @@ -448,7 +450,7 @@ def check( ) -@cli.command(short_help="Runs lock, then sync.") +@cli.command(short_help="Runs lock, then sync.", context_settings=CONTEXT_SETTINGS) @option("--bare", is_flag=True, default=False, help="Minimal output.") @option( "--outdated", is_flag=True, default=False, help=u"List out-of-date dependencies." @@ -525,7 +527,10 @@ def update( ) -@cli.command(short_help=u"Displays currently-installed dependency graph information.") +@cli.command( + short_help=u"Displays currently-installed dependency graph information.", + context_settings=CONTEXT_SETTINGS +) @option("--bare", is_flag=True, default=False, help="Minimal output.") @option("--json", is_flag=True, default=False, help="Output JSON.") @option("--json-tree", is_flag=True, default=False, help="Output JSON in nested tree.") @@ -537,7 +542,10 @@ def graph(bare=False, json=False, json_tree=False, reverse=False): do_graph(bare=bare, json=json, json_tree=json_tree, reverse=reverse) -@cli.command(short_help="View a given module in your editor.", name="open") +@cli.command( + short_help="View a given module in your editor.", name="open", + context_settings=CONTEXT_SETTINGS +) @common_options @argument("module", nargs=1) @pass_state @@ -549,7 +557,7 @@ def run_open(state, module, *args, **kwargs): EDITOR=atom pipenv open requests """ - from ..core import which, ensure_project + from ..core import which, ensure_project, inline_activate_virtual_environment # Ensure that virtualenv is available. ensure_project( @@ -569,11 +577,15 @@ def run_open(state, module, *args, **kwargs): else: p = c.out.strip().rstrip("cdo") echo(crayons.normal("Opening {0!r} in your EDITOR.".format(p), bold=True)) + inline_activate_virtual_environment() edit(filename=p) return 0 -@cli.command(short_help="Installs all packages specified in Pipfile.lock.") +@cli.command( + short_help="Installs all packages specified in Pipfile.lock.", + context_settings=CONTEXT_SETTINGS +) @option("--bare", is_flag=True, default=False, help="Minimal output.") @sync_options @pass_state @@ -606,7 +618,10 @@ def sync( ctx.abort() -@cli.command(short_help="Uninstalls all packages not specified in Pipfile.lock.") +@cli.command( + short_help="Uninstalls all packages not specified in Pipfile.lock.", + context_settings=CONTEXT_SETTINGS +) @option("--bare", is_flag=True, default=False, help="Minimal output.") @option("--dry-run", is_flag=True, default=False, help="Just output unneeded packages.") @verbose_option @@ -617,11 +632,9 @@ def sync( def clean(ctx, state, dry_run=False, bare=False, user=False): """Uninstalls all packages not specified in Pipfile.lock.""" from ..core import do_clean - do_clean(ctx=ctx, three=state.three, python=state.python, dry_run=dry_run) + do_clean(ctx=ctx, three=state.three, python=state.python, dry_run=dry_run, + system=state.system) -# Only invoke the "did you mean" when an argument wasn't passed (it breaks those). -if "-" not in "".join(sys.argv) and len(sys.argv) > 1: - cli = DYMCommandCollection(sources=[cli]) if __name__ == "__main__": cli() diff --git a/pipenv/cli/options.py b/pipenv/cli/options.py index 606454dd..56b0827e 100644 --- a/pipenv/cli/options.py +++ b/pipenv/cli/options.py @@ -4,18 +4,23 @@ from __future__ import absolute_import import os import click.types + from click import ( - BadParameter, Group, Option, argument, echo, make_pass_decorator, option + BadParameter, BadArgumentUsage, Group, Option, argument, echo, make_pass_decorator, option ) +from click_didyoumean import DYMMixin from .. import environments from ..utils import is_valid_url -CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"]) +CONTEXT_SETTINGS = { + "help_option_names": ["-h", "--help"], + "auto_envvar_prefix": "PIPENV" +} -class PipenvGroup(Group): +class PipenvGroup(DYMMixin, Group): """Custom Group class provides formatted main help""" def get_help_option(self, ctx): @@ -51,6 +56,7 @@ class State(object): self.index = None self.extra_index_urls = [] self.verbose = False + self.quiet = False self.pypi_mirror = None self.python = None self.two = None @@ -117,7 +123,7 @@ def sequential_option(f): return value return option("--sequential", is_flag=True, default=False, expose_value=False, help="Install dependencies one-at-a-time, instead of concurrently.", - callback=callback, type=click.types.BOOL)(f) + callback=callback, type=click.types.BOOL, show_envvar=True)(f) def skip_lock_option(f): @@ -127,7 +133,8 @@ def skip_lock_option(f): return value return option("--skip-lock", is_flag=True, default=False, expose_value=False, help=u"Skip locking mechanisms and use the Pipfile instead during operation.", - envvar="PIPENV_SKIP_LOCK", callback=callback, type=click.types.BOOL)(f) + envvar="PIPENV_SKIP_LOCK", callback=callback, type=click.types.BOOL, + show_envvar=True)(f) def keep_outdated_option(f): @@ -137,7 +144,7 @@ def keep_outdated_option(f): return value return option("--keep-outdated", is_flag=True, default=False, expose_value=False, help=u"Keep out-dated dependencies from being updated in Pipfile.lock.", - callback=callback, type=click.types.BOOL)(f) + callback=callback, type=click.types.BOOL, show_envvar=True)(f) def selective_upgrade_option(f): @@ -157,7 +164,7 @@ def ignore_pipfile_option(f): return value return option("--ignore-pipfile", is_flag=True, default=False, expose_value=False, help="Ignore Pipfile when installing, using the Pipfile.lock.", - callback=callback, type=click.types.BOOL)(f) + callback=callback, type=click.types.BOOL, show_envvar=True)(f) def dev_option(f): @@ -167,7 +174,7 @@ def dev_option(f): return value return option("--dev", "-d", is_flag=True, default=False, type=click.types.BOOL, help="Install both develop and default packages.", callback=callback, - expose_value=False)(f) + expose_value=False, show_envvar=True)(f) def pre_option(f): @@ -208,7 +215,7 @@ def python_option(f): return value return option("--python", default=False, nargs=1, callback=callback, help="Specify which version of Python virtualenv should use.", - expose_value=False)(f) + expose_value=False, allow_from_autoenv=False)(f) def pypi_mirror_option(f): @@ -225,12 +232,32 @@ def verbose_option(f): def callback(ctx, param, value): state = ctx.ensure_object(State) if value: + if state.quiet: + raise BadArgumentUsage( + "--verbose and --quiet are mutually exclusive! Please choose one!", + ctx=ctx + ) state.verbose = True - setup_verbosity(ctx, param, value) + setup_verbosity(ctx, param, 1) return option("--verbose", "-v", is_flag=True, expose_value=False, callback=callback, help="Verbose mode.", type=click.types.BOOL)(f) +def quiet_option(f): + def callback(ctx, param, value): + state = ctx.ensure_object(State) + if value: + if state.verbose: + raise BadArgumentUsage( + "--verbose and --quiet are mutually exclusive! Please choose one!", + ctx=ctx + ) + state.quiet = True + setup_verbosity(ctx, param, -1) + return option("--quiet", "-q", is_flag=True, expose_value=False, + callback=callback, help="Quiet mode.", type=click.types.BOOL)(f) + + def site_packages_option(f): def callback(ctx, param, value): state = ctx.ensure_object(State) @@ -238,7 +265,7 @@ def site_packages_option(f): return value return option("--site-packages", is_flag=True, default=False, type=click.types.BOOL, help="Enable site-packages for the virtualenv.", callback=callback, - expose_value=False)(f) + expose_value=False, show_envvar=True)(f) def clear_option(f): @@ -248,7 +275,7 @@ def clear_option(f): return value return option("--clear", is_flag=True, callback=callback, type=click.types.BOOL, help="Clears caches (pipenv, pip, and pip-tools).", - expose_value=False)(f) + expose_value=False, show_envvar=True)(f) def system_option(f): @@ -258,7 +285,8 @@ def system_option(f): state.system = value return value return option("--system", is_flag=True, default=False, help="System pip management.", - callback=callback, type=click.types.BOOL, expose_value=False)(f) + callback=callback, type=click.types.BOOL, expose_value=False, + show_envvar=True)(f) def requirementstxt_option(f): @@ -287,8 +315,9 @@ def code_option(f): if value: state.installstate.code = value return value - return option("--code", "-c", nargs=1, default=False, help="Import from codebase.", - callback=callback, expose_value=False)(f) + return option("--code", "-c", nargs=1, default=False, help="Install packages " + "automatically discovered from import statements.", callback=callback, + expose_value=False)(f) def deploy_option(f): @@ -298,15 +327,21 @@ def deploy_option(f): return value return option("--deploy", is_flag=True, default=False, type=click.types.BOOL, help=u"Abort if the Pipfile.lock is out-of-date, or Python version is" - " wrong.", callback=callback, expose_value=False)(f) + " wrong.", callback=callback, expose_value=False)(f) def setup_verbosity(ctx, param, value): if not value: return import logging - logging.getLogger("pip").setLevel(logging.INFO) - environments.PIPENV_VERBOSITY = 1 + loggers = ("pip", "piptools") + if value == 1: + for logger in loggers: + logging.getLogger(logger).setLevel(logging.INFO) + elif value == -1: + for logger in loggers: + logging.getLogger(logger).setLevel(logging.CRITICAL) + environments.PIPENV_VERBOSITY = value def validate_python_path(ctx, param, value): @@ -369,7 +404,6 @@ def install_options(f): f = index_option(f) f = extra_index_option(f) f = requirementstxt_option(f) - f = pre_option(f) f = selective_upgrade_option(f) f = ignore_pipfile_option(f) f = editable_option(f) diff --git a/pipenv/cmdparse.py b/pipenv/cmdparse.py index cec19273..2e550e22 100644 --- a/pipenv/cmdparse.py +++ b/pipenv/cmdparse.py @@ -10,7 +10,7 @@ class ScriptEmptyError(ValueError): def _quote_if_contains(value, pattern): - if next(re.finditer(pattern, value), None): + if next(iter(re.finditer(pattern, value)), None): return '"{0}"'.format(re.sub(r'(\\*)"', r'\1\1\\"', value)) return value @@ -70,14 +70,29 @@ class Script(object): Foul characters include: * Whitespaces. + * Carets (^). (pypa/pipenv#3307) * Parentheses in the command. (pypa/pipenv#3168) + Carets introduce a difficult situation since they are essentially + "lossy" when parsed. Consider this in cmd.exe:: + + > echo "foo^bar" + "foo^bar" + > echo foo^^bar + foo^bar + + The two commands produce different results, but are both parsed by the + shell as `foo^bar`, and there's essentially no sensible way to tell + what was actually passed in. This implementation assumes the quoted + variation (the first) since it is easier to implement, and arguably + the more common case. + The intended use of this function is to pre-process an argument list before passing it into ``subprocess.Popen(..., shell=True)``. See also: https://docs.python.org/3/library/subprocess.html#converting-argument-sequence """ return " ".join(itertools.chain( - [_quote_if_contains(self.command, r'[\s()]')], - (_quote_if_contains(arg, r'\s') for arg in self.args), + [_quote_if_contains(self.command, r'[\s^()]')], + (_quote_if_contains(arg, r'[\s^]') for arg in self.args), )) diff --git a/pipenv/core.py b/pipenv/core.py index 6f1a7a2d..cf427728 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1,62 +1,52 @@ # -*- coding=utf-8 -*- - +from __future__ import absolute_import, print_function +import io +import json as simplejson import logging import os -import sys import shutil +import sys import time -import json as simplejson -import click -import click_completion -import crayons -import dotenv -import delegator -import pipfile -import vistir import warnings + +import click import six - import urllib3.util as urllib3_util +import vistir +from click_completion import init as init_completion +import delegator +import dotenv +import pipfile + +from .patched import crayons +from . import environments, exceptions, pep508checker, progress +from ._compat import fix_utf8, decode_for_output from .cmdparse import Script +from .environments import ( + PIPENV_CACHE_DIR, PIPENV_COLORBLIND, PIPENV_DEFAULT_PYTHON_VERSION, + PIPENV_DONT_USE_PYENV, PIPENV_HIDE_EMOJIS, PIPENV_MAX_SUBPROCESS, + PIPENV_PYUP_API_KEY, PIPENV_SHELL_FANCY, PIPENV_SKIP_VALIDATION, + PIPENV_YES, SESSION_IS_INTERACTIVE, PIP_EXISTS_ACTION, PIPENV_RESOLVE_VCS, + is_type_checking +) from .project import Project, SourceNotFound from .utils import ( - convert_deps_to_pip, - is_required_version, - proper_case, - pep423_name, - venv_resolve_deps, - escape_grouped_arguments, - python_version, - find_windows_executable, - prepare_pip_source_args, - is_valid_url, - is_pypi_url, - create_mirror_source, - download_file, - is_pinned, - is_star, - parse_indexes, - escape_cmd, - create_spinner, - get_canonical_names + convert_deps_to_pip, create_mirror_source, create_spinner, download_file, + escape_cmd, escape_grouped_arguments, find_windows_executable, + get_canonical_names, is_pinned, is_pypi_url, is_required_version, is_star, + is_valid_url, parse_indexes, pep423_name, prepare_pip_source_args, + proper_case, python_version, venv_resolve_deps, run_command, + is_python_command, find_python, make_posix, interrupt_handled_subprocess, + get_indexes_from_requirement, get_source_list, get_project_index, ) -from . import environments, pep508checker, progress -from .environments import ( - PIPENV_COLORBLIND, - PIPENV_SHELL_FANCY, - PIPENV_SKIP_VALIDATION, - PIPENV_HIDE_EMOJIS, - PIPENV_YES, - PIPENV_DEFAULT_PYTHON_VERSION, - PIPENV_MAX_SUBPROCESS, - PIPENV_DONT_USE_PYENV, - SESSION_IS_INTERACTIVE, - PIPENV_CACHE_DIR, - PIPENV_PYUP_API_KEY, -) -from ._compat import fix_utf8 -from . import exceptions + + +if is_type_checking(): + from typing import Dict, List, Mapping, Optional, Union, Text + from pipenv.vendor.requirementslib.models.requirements import Requirement + TSourceDict = Dict[Text, Union[Text, bool]] + # Packages that should be ignored later. BAD_PACKAGES = ( @@ -92,7 +82,7 @@ else: INSTALL_LABEL2 = " " STARTING_LABEL = " " # Enable shell completion. -click_completion.init() +init_completion() # Disable colors, for the color blind and others who do not prefer colors. if PIPENV_COLORBLIND: crayons.disable() @@ -106,16 +96,19 @@ def which(command, location=None, allow_global=False): location = os.environ.get("VIRTUAL_ENV", None) if not (location and os.path.exists(location)) and not allow_global: raise RuntimeError("location not created nor specified") + + version_str = "python{0}".format(".".join([str(v) for v in sys.version_info[:2]])) + is_python = command in ("python", os.path.basename(sys.executable), version_str) if not allow_global: if os.name == "nt": p = find_windows_executable(os.path.join(location, "Scripts"), command) else: p = os.path.join(location, "bin", command) else: - if command == "python": + if is_python: p = sys.executable if not os.path.exists(p): - if command == "python": + if is_python: p = sys.executable or system_which("python") else: p = system_which(command) @@ -258,7 +251,9 @@ def import_from_code(path="."): rs = [] try: - for r in pipreqs.get_all_imports(path): + for r in pipreqs.get_all_imports( + path, encoding="utf-8", extra_ignore_dirs=[".venv"] + ): if r not in BAD_PACKAGES: rs.append(r) pkg_names = pipreqs.get_pkg_names(rs) @@ -343,26 +338,16 @@ def find_a_system_python(line): * Search for "python" and "pythonX.Y" executables in PATH to find a match. * Nothing fits, return None. """ - if not line: - return None - if os.path.isabs(line): - return line - from .vendor.pythonfinder import Finder + from .vendor.pythonfinder import Finder finder = Finder(system=False, global_search=True) + if not line: + return next(iter(finder.find_all_python_versions()), None) + # Use the windows finder executable if (line.startswith("py ") or line.startswith("py.exe ")) and os.name == "nt": line = line.split(" ", 1)[1].lstrip("-") - elif line.startswith("py"): - python_entry = finder.which(line) - if python_entry: - return python_entry.path.as_posix() - return None - python_entry = finder.find_python_version(line) - if not python_entry: - python_entry = finder.which("python{0}".format(line)) - if python_entry: - return python_entry.path.as_posix() - return None + python_entry = find_python(finder, line) + return python_entry def ensure_python(three=None, python=None): @@ -395,6 +380,9 @@ def ensure_python(three=None, python=None): if not python: python = PIPENV_DEFAULT_PYTHON_VERSION path_to_python = find_a_system_python(python) + if environments.is_verbose(): + click.echo(u"Using python: {0}".format(python), err=True) + click.echo(u"Path to python: {0}".format(path_to_python), err=True) if not path_to_python and python is not None: # We need to install Python. click.echo( @@ -455,6 +443,11 @@ def ensure_python(three=None, python=None): sp.ok(environments.PIPENV_SPINNER_OK_TEXT.format("Success!")) # Print the results, in a beautiful blue… click.echo(crayons.blue(c.out), err=True) + # Clear the pythonfinder caches + from .vendor.pythonfinder import Finder + finder = Finder(system=False, global_search=True) + finder.find_python_version.cache_clear() + finder.find_all_python_versions.cache_clear() # Find the newly installed Python, hopefully. version = str(version) path_to_python = find_a_system_python(version) @@ -487,6 +480,8 @@ def ensure_virtualenv(three=None, python=None, site_packages=False, pypi_mirror= ensure_environment() # Ensure Python is available. python = ensure_python(three=three, python=python) + if python is not None and not isinstance(python, six.string_types): + python = python.path.as_posix() # Create the virtualenv. # Abort if --system (or running in a virtualenv). if PIPENV_USE_SYSTEM: @@ -508,7 +503,10 @@ def ensure_virtualenv(three=None, python=None, site_packages=False, pypi_mirror= elif (python) or (three is not None) or (site_packages is not False): USING_DEFAULT_PYTHON = False # Ensure python is installed before deleting existing virtual env - ensure_python(three=three, python=python) + python = ensure_python(three=three, python=python) + if python is not None and not isinstance(python, six.string_types): + python = python.path.as_posix() + click.echo(crayons.red("Virtualenv already exists!"), err=True) # If VIRTUAL_ENV is set, there is a possibility that we are # going to remove the active virtualenv that the user cares @@ -555,11 +553,16 @@ def ensure_project( # Automatically use an activated virtualenv. if PIPENV_USE_SYSTEM: system = True - if not project.pipfile_exists: - if deploy is True: - raise exceptions.PipfileNotFound - else: - project.touch_pipfile() + if not project.pipfile_exists and deploy: + raise exceptions.PipfileNotFound + # Fail if working under / + if not project.name: + click.echo( + "{0}: Pipenv is not intended to work under the root directory, " + "please choose another path.".format(crayons.red("ERROR")), + err=True + ) + sys.exit(1) # Skip virtualenv creation when --system was used. if not system: ensure_virtualenv( @@ -581,7 +584,7 @@ def ensure_project( crayons.red("Warning", bold=True), crayons.normal("python_version", bold=True), crayons.blue(project.required_python_version), - crayons.blue(python_version(path_to_python)), + crayons.blue(python_version(path_to_python) or "unknown"), crayons.green(shorten_path(path_to_python)), ), err=True, @@ -622,24 +625,23 @@ def shorten_path(location, bold=False): def do_where(virtualenv=False, bare=True): """Executes the where functionality.""" if not virtualenv: - location = project.pipfile_location - # Shorten the virtual display of the path to the virtualenv. - if not bare: - location = shorten_path(location) - if not location: + if not project.pipfile_exists: click.echo( "No Pipfile present at project home. Consider running " "{0} first to automatically generate a Pipfile for you." "".format(crayons.green("`pipenv install`")), err=True, ) - elif not bare: + return + location = project.pipfile_location + # Shorten the virtual display of the path to the virtualenv. + if not bare: + location = shorten_path(location) click.echo( "Pipfile found at {0}.\n Considering this to be the project home." "".format(crayons.green(location)), err=True, ) - pass else: click.echo(project.project_directory) else: @@ -652,10 +654,10 @@ def do_where(virtualenv=False, bare=True): click.echo(location) -def _cleanup_procs(procs, concurrent, failed_deps_queue, retry=True): +def _cleanup_procs(procs, failed_deps_queue, retry=True): while not procs.empty(): c = procs.get() - if concurrent: + if not c.blocking: c.block() failed = False if c.return_code != 0: @@ -671,7 +673,7 @@ def _cleanup_procs(procs, concurrent, failed_deps_queue, retry=True): # We echo both c.out and c.err because pip returns error details on out. err = c.err.strip().splitlines() if c.err else [] out = c.out.strip().splitlines() if c.out else [] - err_lines = [line for line in [out, err]] + err_lines = [line for message in [out, err] for line in message] # Return the subprocess' return code. raise exceptions.InstallError(c.dep.name, extra=err_lines) # Save the Failed Dependency for later. @@ -687,66 +689,83 @@ def _cleanup_procs(procs, concurrent, failed_deps_queue, retry=True): def batch_install(deps_list, procs, failed_deps_queue, - requirements_dir, no_deps=False, ignore_hashes=False, + requirements_dir, no_deps=True, ignore_hashes=False, allow_global=False, blocking=False, pypi_mirror=None, - nprocs=PIPENV_MAX_SUBPROCESS, retry=True): - + retry=True, sequential_deps=None): + from .vendor.requirementslib.models.utils import strip_extras_markers_from_requirement + if sequential_deps is None: + sequential_deps = [] failed = (not retry) + install_deps = not no_deps if not failed: - label = INSTALL_LABEL if os.name != "nt" else "" + label = INSTALL_LABEL if not PIPENV_HIDE_EMOJIS else "" else: label = INSTALL_LABEL2 + deps_to_install = deps_list[:] + deps_to_install.extend(sequential_deps) + sequential_dep_names = [d.name for d in sequential_deps] + deps_list_bar = progress.bar( - deps_list, width=32, + deps_to_install, width=32, label=label ) + + indexes = [] trusted_hosts = [] # Install these because for dep in deps_list_bar: - index = None - if dep.index: - index = project.find_source(dep.index) - indexes.append(index) - if not index.get("verify_ssl", False): - trusted_hosts.append(urllib3_util.parse_url(index.get("url")).host) + extra_indexes = [] + if dep.req.req: + dep.req.req = strip_extras_markers_from_requirement(dep.req.req) + if dep.markers: + dep.markers = str(strip_extras_markers_from_requirement(dep.get_markers())) # Install the module. is_artifact = False + if no_deps: + link = getattr(dep.req, "link", None) + is_wheel = getattr(link, "is_wheel", False) if link else False if dep.is_file_or_url and (dep.is_direct_url or any( dep.req.uri.endswith(ext) for ext in ["zip", "tar.gz"] )): is_artifact = True - - extra_indexes = [] - if not index and indexes: - index = next(iter(indexes)) - if len(indexes) > 1: - extra_indexes = indexes[1:] + elif dep.is_vcs: + is_artifact = True + if not PIPENV_RESOLVE_VCS and is_artifact and not dep.editable: + install_deps = True + no_deps = False with vistir.contextmanagers.temp_environ(): if not allow_global: os.environ["PIP_USER"] = vistir.compat.fs_str("0") + if "PYTHONHOME" in os.environ: + del os.environ["PYTHONHOME"] + if "GIT_CONFIG" in os.environ and dep.is_vcs: + del os.environ["GIT_CONFIG"] + c = pip_install( dep, ignore_hashes=any([ignore_hashes, dep.editable, dep.is_vcs]), allow_global=allow_global, - no_deps=False if is_artifact else no_deps, + no_deps=not install_deps, block=any([dep.editable, dep.is_vcs, blocking]), - index=index, + index=dep.index, requirements_dir=requirements_dir, pypi_mirror=pypi_mirror, trusted_hosts=trusted_hosts, - extra_indexes=extra_indexes + extra_indexes=extra_indexes, + use_pep517=not failed, ) - if dep.is_vcs: + c.dep = dep + # if dep.is_vcs or dep.editable: + is_sequential = sequential_deps and dep.name in sequential_dep_names + if is_sequential: c.block() - if procs.qsize() < nprocs: - c.dep = dep - procs.put(c) - if procs.full() or procs.qsize() == len(deps_list): - _cleanup_procs(procs, not blocking, failed_deps_queue, retry=retry) + procs.put(c) + if procs.full() or procs.qsize() == len(deps_list) or is_sequential: + _cleanup_procs(procs, failed_deps_queue, retry=retry) def do_install_dependencies( @@ -770,14 +789,14 @@ def do_install_dependencies( from six.moves import queue if requirements: bare = True - blocking = not concurrent # Load the lockfile if it exists, or if only is being used (e.g. lock is being used). if skip_lock or only or not project.lockfile_exists: if not bare: click.echo( crayons.normal(fix_utf8("Installing dependencies from Pipfile…"), bold=True) ) - lockfile = project.get_or_create_lockfile() + # skip_lock should completely bypass the lockfile (broken in 4dac1676) + lockfile = project.get_or_create_lockfile(from_pipfile=True) else: lockfile = project.get_or_create_lockfile() if not bare: @@ -790,7 +809,7 @@ def do_install_dependencies( ) ) # Allow pip to resolve dependencies when in skip-lock mode. - no_deps = not skip_lock + no_deps = not skip_lock # skip_lock true, no_deps False, pip resolves deps deps_list = list(lockfile.get_requirements(dev=dev, only=requirements)) if requirements: index_args = prepare_pip_source_args(project.sources) @@ -805,24 +824,40 @@ def do_install_dependencies( ) sys.exit(0) - procs = queue.Queue(maxsize=PIPENV_MAX_SUBPROCESS) + if concurrent: + nprocs = PIPENV_MAX_SUBPROCESS + else: + nprocs = 1 + procs = queue.Queue(maxsize=nprocs) failed_deps_queue = queue.Queue() - + if skip_lock: + ignore_hashes = True + editable_or_vcs_deps = [dep for dep in deps_list if (dep.editable or dep.vcs)] + normal_deps = [dep for dep in deps_list if not (dep.editable or dep.vcs)] install_kwargs = { "no_deps": no_deps, "ignore_hashes": ignore_hashes, "allow_global": allow_global, - "blocking": blocking, "pypi_mirror": pypi_mirror + "blocking": not concurrent, "pypi_mirror": pypi_mirror, + "sequential_deps": editable_or_vcs_deps } - if concurrent: - install_kwargs["nprocs"] = PIPENV_MAX_SUBPROCESS - else: - install_kwargs["nprocs"] = 1 batch_install( - deps_list, procs, failed_deps_queue, requirements_dir, **install_kwargs + normal_deps, procs, failed_deps_queue, requirements_dir, **install_kwargs ) if not procs.empty(): - _cleanup_procs(procs, concurrent, failed_deps_queue) + _cleanup_procs(procs, failed_deps_queue) + + # click.echo(crayons.normal( + # decode_for_output("Installing editable and vcs dependencies…"), bold=True + # )) + + # install_kwargs.update({"blocking": True}) + # # XXX: All failed and editable/vcs deps should be installed in sequential mode! + # procs = queue.Queue(maxsize=1) + # batch_install( + # editable_or_vcs_deps, procs, failed_deps_queue, requirements_dir, + # **install_kwargs + # ) # Iterate over the hopefully-poorly-packaged dependencies… if not failed_deps_queue.empty(): @@ -833,16 +868,12 @@ def do_install_dependencies( while not failed_deps_queue.empty(): failed_dep = failed_deps_queue.get() retry_list.append(failed_dep) - install_kwargs.update({ - "nprocs": 1, - "retry": False, - "blocking": True, - }) + install_kwargs.update({"retry": False}) batch_install( retry_list, procs, failed_deps_queue, requirements_dir, **install_kwargs ) if not procs.empty(): - _cleanup_procs(procs, False, failed_deps_queue, retry=False) + _cleanup_procs(procs, failed_deps_queue, retry=False) def convert_three_to_python(three, python): @@ -862,6 +893,7 @@ def convert_three_to_python(three, python): def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None): """Creates a virtualenv.""" + click.echo( crayons.normal(fix_utf8("Creating a virtualenv for this project…"), bold=True), err=True ) @@ -871,11 +903,13 @@ def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None): ) # Default to using sys.executable, if Python wasn't provided. + using_string = u"Using" if not python: python = sys.executable + using_string = "Using default python from" click.echo( u"{0} {1} {3} {2}".format( - crayons.normal("Using", bold=True), + crayons.normal(using_string, bold=True), crayons.red(python, bold=True), crayons.normal(fix_utf8("to create virtualenv…"), bold=True), crayons.green("({0})".format(python_version(python))), @@ -905,20 +939,19 @@ def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None): pip_config = {} # Actually create the virtualenv. - nospin = environments.PIPENV_NOSPIN - with create_spinner("Creating virtual environment...") as sp: - c = vistir.misc.run( - cmd, verbose=False, return_object=True, write_to_stdout=False, - combine_stderr=False, block=True, nospin=True, env=pip_config, + error = None + with create_spinner(u"Creating virtual environment...") as sp: + with interrupt_handled_subprocess(cmd, combine_stderr=False, env=pip_config) as c: + click.echo(crayons.blue(u"{0}".format(c.out)), err=True) + if c.returncode != 0: + error = c.err if environments.is_verbose() else exceptions.prettify_exc(c.err) + sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format(u"Failed creating virtual environment")) + else: + sp.green.ok(environments.PIPENV_SPINNER_OK_TEXT.format(u"Successfully created virtual environment!")) + if error is not None: + raise exceptions.VirtualenvCreationException( + extra=crayons.red("{0}".format(error)) ) - click.echo(crayons.blue("{0}".format(c.out)), err=True) - if c.returncode != 0: - sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format("Failed creating virtual environment")) - raise exceptions.VirtualenvCreationException( - extra=[crayons.blue("{0}".format(c.err)),] - ) - else: - sp.green.ok(environments.PIPENV_SPINNER_OK_TEXT.format("Successfully created virtual environment!")) # Associate project directory with the environment. # This mimics Pew's "setproject". @@ -1019,9 +1052,12 @@ def do_lock( dev_packages = overwrite_dev(project.packages, dev_packages) # Resolve dev-package dependencies, with pip-tools. for is_dev in [True, False]: - pipfile_section = "dev_packages" if is_dev else "packages" + pipfile_section = "dev-packages" if is_dev else "packages" lockfile_section = "develop" if is_dev else "default" - packages = getattr(project, pipfile_section) + if project.pipfile_exists: + packages = project.parsed_pipfile.get(pipfile_section, {}) + else: + packages = getattr(project, pipfile_section.replace("-", "_")) if write: # Alert the user of progress. @@ -1034,12 +1070,9 @@ def do_lock( err=True, ) - deps = convert_deps_to_pip( - packages, project, r=False, include_index=True - ) # Mutates the lockfile venv_resolve_deps( - deps, + packages, which=which, project=project, dev=is_dev, @@ -1048,7 +1081,8 @@ def do_lock( allow_global=system, pypi_mirror=pypi_mirror, pipfile=packages, - lockfile=lockfile + lockfile=lockfile, + keep_outdated=keep_outdated ) # Support for --keep-outdated… @@ -1065,6 +1099,12 @@ def do_lock( lockfile[section_name][canonical_name] = cached_lockfile[ section_name ][canonical_name].copy() + for key in ["default", "develop"]: + packages = set(cached_lockfile[key].keys()) + new_lockfile = set(lockfile[key].keys()) + missing = packages - new_lockfile + for missing_pkg in missing: + lockfile[key][missing_pkg] = cached_lockfile[key][missing_pkg].copy() # Overwrite any develop packages with default packages. lockfile["develop"].update(overwrite_dev(lockfile.get("default", {}), lockfile["develop"])) if write: @@ -1143,12 +1183,19 @@ def do_init( pypi_mirror=None, ): """Executes the init functionality.""" - from .environments import PIPENV_VIRTUALENV + from .environments import ( + PIPENV_VIRTUALENV, PIPENV_DEFAULT_PYTHON_VERSION, PIPENV_PYTHON, PIPENV_USE_SYSTEM + ) + python = None + if PIPENV_PYTHON is not None: + python = PIPENV_PYTHON + elif PIPENV_DEFAULT_PYTHON_VERSION is not None: + python = PIPENV_DEFAULT_PYTHON_VERSION - if not system: + if not system and not PIPENV_USE_SYSTEM: if not project.virtualenv_exists: try: - do_create_virtualenv(pypi_mirror=pypi_mirror) + do_create_virtualenv(python=python, three=None, pypi_mirror=pypi_mirror) except KeyboardInterrupt: cleanup_virtualenv(bare=False) sys.exit(1) @@ -1189,9 +1236,9 @@ def do_init( ) else: if old_hash: - msg = fix_utf8("Pipfile.lock ({1}) out of date, updating to ({0})…") + msg = fix_utf8("Pipfile.lock ({0}) out of date, updating to ({1})…") else: - msg = fix_utf8("Pipfile.lock is corrupted, replaced with ({0})…") + msg = fix_utf8("Pipfile.lock is corrupted, replaced with ({1})…") click.echo( crayons.red(msg.format(old_hash[-6:], new_hash[-6:]), bold=True), err=True, @@ -1247,12 +1294,107 @@ def do_init( ) +def get_pip_args( + pre=False, # type: bool + verbose=False, # type: bool, + upgrade=False, # type: bool, + require_hashes=False, # type: bool, + no_build_isolation=False, # type: bool, + no_use_pep517=False, # type: bool, + no_deps=False, # type: bool, + selective_upgrade=False, # type: bool + src_dir=None, # type: Optional[str] +): + # type: (...) -> List[str] + from .vendor.packaging.version import parse as parse_version + arg_map = { + "pre": ["--pre"], + "verbose": ["--verbose"], + "upgrade": ["--upgrade"], + "require_hashes": ["--require-hashes"], + "no_build_isolation": ["--no-build-isolation"], + "no_use_pep517": [], + "no_deps": ["--no-deps"], + "selective_upgrade": [ + "--upgrade-strategy=only-if-needed", "--exists_action={0}".format(PIP_EXISTS_ACTION or "i") + ], + "src_dir": src_dir, + } + if project.environment.pip_version >= parse_version("19.0"): + arg_map["no_use_pep517"].append("--no-use-pep517") + if project.environment.pip_version < parse_version("19.1"): + arg_map["no_use_pep517"].append("--no-build-isolation") + arg_set = [] + for key in arg_map.keys(): + if key in locals() and locals().get(key): + arg_set.extend(arg_map.get(key)) + return list(vistir.misc.dedup(arg_set)) + + +def get_requirement_line( + requirement, # type: Requirement + src_dir=None, # type: Optional[str] + include_hashes=True, # type: bool + format_for_file=False, # type: bool +): + # type: (...) -> Union[List[str], str] + line = None + if requirement.vcs or requirement.is_file_or_url: + if src_dir and requirement.line_instance.wheel_kwargs: + requirement.line_instance._wheel_kwargs.update({ + "src_dir": src_dir + }) + requirement.line_instance.vcsrepo + line = requirement.line_instance.line + if requirement.line_instance.markers: + line = '{0}; {1}'.format(line, requirement.line_instance.markers) + if not format_for_file: + line = '"{0}"'.format(line) + if requirement.editable: + if not format_for_file: + return ["-e", line] + return '-e {0}'.format(line) + if not format_for_file: + return [line,] + return line + return requirement.as_line(include_hashes=include_hashes, as_list=not format_for_file) + + +def write_requirement_to_file( + requirement, # type: Requirement + requirements_dir=None, # type: Optional[str] + src_dir=None, # type: Optional[str] + include_hashes=True # type: bool +): + # type: (...) -> str + if not requirements_dir: + requirements_dir = vistir.path.create_tracked_tempdir( + prefix="pipenv", suffix="requirements") + line = requirement.line_instance.get_line( + with_prefix=True, with_hashes=include_hashes, with_markers=True, as_list=False + ) + + f = vistir.compat.NamedTemporaryFile( + prefix="pipenv-", suffix="-requirement.txt", dir=requirements_dir, + delete=False + ) + if environments.is_verbose(): + click.echo( + "Writing supplied requirement line to temporary file: {0!r}".format(line), + err=True + ) + f.write(vistir.misc.to_bytes(line)) + r = f.name + f.close() + return r + + def pip_install( requirement=None, r=None, allow_global=False, ignore_hashes=False, - no_deps=True, + no_deps=None, block=True, index=None, pre=False, @@ -1260,148 +1402,100 @@ def pip_install( requirements_dir=None, extra_indexes=None, pypi_mirror=None, - trusted_hosts=None + trusted_hosts=None, + use_pep517=True ): from pipenv.patched.notpip._internal import logger as piplogger - from .utils import Mapping - from .vendor.urllib3.util import parse_url - - src = [] - write_to_tmpfile = False - if requirement: - needs_hashes = not requirement.editable and not ignore_hashes and r is None - has_subdir = requirement.is_vcs and requirement.req.subdirectory - write_to_tmpfile = needs_hashes or has_subdir - + src_dir = None if not trusted_hosts: trusted_hosts = [] + trusted_hosts.extend(os.environ.get("PIP_TRUSTED_HOSTS", [])) + if not allow_global: + src_dir = os.getenv("PIP_SRC", os.getenv("PIP_SRC_DIR", project.virtualenv_src_location)) + else: + src_dir = os.getenv("PIP_SRC", os.getenv("PIP_SRC_DIR")) + if requirement: + if requirement.editable or not requirement.hashes: + ignore_hashes = True + elif not (requirement.is_vcs or requirement.editable or requirement.vcs): + ignore_hashes = False + line = None + # Try installing for each source in project.sources. + if not index and requirement.index: + index = requirement.index + if index and not extra_indexes: + extra_indexes = list(project.sources) + if requirement and requirement.vcs or requirement.editable: + requirement.index = None + # Install dependencies when a package is a non-editable VCS dependency. + # Don't specify a source directory when using --system. + if not requirement.editable and no_deps is not True: + # Leave this off becauase old lockfiles don't have all deps included + # TODO: When can it be turned back on? + no_deps = False + elif requirement.editable and no_deps is None: + no_deps = True + + r = write_requirement_to_file( + requirement, requirements_dir=requirements_dir, src_dir=src_dir, + include_hashes=not ignore_hashes + ) + sources = get_source_list( + index, extra_indexes=extra_indexes, trusted_hosts=trusted_hosts, + pypi_mirror=pypi_mirror + ) + if r: + with io.open(r, "r") as fh: + if "--hash" not in fh.read(): + ignore_hashes = True if environments.is_verbose(): - piplogger.setLevel(logging.INFO) + piplogger.setLevel(logging.WARN) if requirement: click.echo( crayons.normal("Installing {0!r}".format(requirement.name), bold=True), err=True, ) - # Create files for hash mode. - if write_to_tmpfile: - if not requirements_dir: - requirements_dir = vistir.path.create_tracked_tempdir( - prefix="pipenv", suffix="requirements") - f = vistir.compat.NamedTemporaryFile( - prefix="pipenv-", suffix="-requirement.txt", dir=requirements_dir, - delete=False - ) - f.write(vistir.misc.to_bytes(requirement.as_line())) - r = f.name - f.close() - # Install dependencies when a package is a VCS dependency. - if requirement and requirement.vcs: - no_deps = False - # Don't specify a source directory when using --system. - if not allow_global and ("PIP_SRC" not in os.environ): - src.extend(["--src", "{0}".format(project.virtualenv_src_location)]) - # Try installing for each source in project.sources. - if index: - if isinstance(index, (Mapping, dict)): - index_source = index - else: - try: - index_source = project.find_source(index) - index_source = index_source.copy() - except SourceNotFound: - src_name = project.src_name_from_url(index) - index_url = parse_url(index) - verify_ssl = index_url.host not in trusted_hosts - index_source = {"url": index, "verify_ssl": verify_ssl, "name": src_name} - sources = [index_source.copy(),] - if extra_indexes: - if isinstance(extra_indexes, six.string_types): - extra_indexes = [extra_indexes,] - for idx in extra_indexes: - extra_src = None - if isinstance(idx, (Mapping, dict)): - extra_src = idx - try: - extra_src = project.find_source(idx) if not extra_src else extra_src - except SourceNotFound: - src_name = project.src_name_from_url(idx) - src_url = parse_url(idx) - verify_ssl = src_url.host not in trusted_hosts - extra_src = {"url": idx, "verify_ssl": verify_ssl, "name": extra_src} - if extra_src["url"] != index_source["url"]: - sources.append(extra_src) - else: - for idx in project.pipfile_sources: - if idx["url"] != sources[0]["url"]: - sources.append(idx) - else: - sources = project.pipfile_sources - if pypi_mirror: - sources = [ - create_mirror_source(pypi_mirror) if is_pypi_url(source["url"]) else source - for source in sources - ] - if (requirement and requirement.editable) and not r: - line_kwargs = {"as_list": True} - if requirement.markers: - line_kwargs["include_markers"] = False - install_reqs = requirement.as_line(**line_kwargs) - if requirement.editable and install_reqs[0].startswith("-e "): - req, install_reqs = install_reqs[0], install_reqs[1:] - editable_opt, req = req.split(" ", 1) - install_reqs = [editable_opt, req] + install_reqs - if not all(item.startswith("--hash") for item in install_reqs): - ignore_hashes = True - elif r: - install_reqs = ["-r", r] - with open(r) as f: - if "--hash" not in f.read(): - ignore_hashes = True - else: - ignore_hashes = True if not requirement.hashes else False - install_reqs = requirement.as_line(as_list=True) - if not requirement.markers: - install_reqs = [escape_cmd(r) for r in install_reqs] - elif len(install_reqs) > 1: - install_reqs = install_reqs[0] + [escape_cmd(r) for r in install_reqs[1:]] pip_command = [which_pip(allow_global=allow_global), "install"] - if pre: - pip_command.append("--pre") - if src: - pip_command.extend(src) - if environments.is_verbose(): - pip_command.append("--verbose") - pip_command.append("--upgrade") - if selective_upgrade: - pip_command.append("--upgrade-strategy=only-if-needed") - if no_deps: - pip_command.append("--no-deps") - pip_command.extend(install_reqs) + pip_args = get_pip_args( + pre=pre, verbose=environments.is_verbose(), upgrade=True, + selective_upgrade=selective_upgrade, no_use_pep517=not use_pep517, + no_deps=no_deps, require_hashes=not ignore_hashes + ) + pip_command.extend(pip_args) + if r: + pip_command.extend(["-r", r]) + elif line: + pip_command.extend(line) pip_command.extend(prepare_pip_source_args(sources)) - if not ignore_hashes: - pip_command.append("--require-hashes") - if environments.is_verbose(): click.echo("$ {0}".format(pip_command), err=True) cache_dir = vistir.compat.Path(PIPENV_CACHE_DIR) + DEFAULT_EXISTS_ACTION = "w" + if selective_upgrade: + DEFAULT_EXISTS_ACTION = "i" + exists_action = vistir.misc.fs_str(PIP_EXISTS_ACTION or DEFAULT_EXISTS_ACTION) pip_config = { "PIP_CACHE_DIR": vistir.misc.fs_str(cache_dir.as_posix()), "PIP_WHEEL_DIR": vistir.misc.fs_str(cache_dir.joinpath("wheels").as_posix()), "PIP_DESTINATION_DIR": vistir.misc.fs_str( cache_dir.joinpath("pkgs").as_posix() ), - "PIP_EXISTS_ACTION": vistir.misc.fs_str("w"), + "PIP_EXISTS_ACTION": exists_action, "PATH": vistir.misc.fs_str(os.environ.get("PATH")), } - if src: + if src_dir: + if environments.is_verbose(): + click.echo("Using source directory: {0!r}".format(src_dir), err=True) pip_config.update( - {"PIP_SRC": vistir.misc.fs_str(project.virtualenv_src_location)} + {"PIP_SRC": vistir.misc.fs_str(src_dir)} ) cmd = Script.parse(pip_command) pip_command = cmd.cmdify() + c = None c = delegator.run(pip_command, block=block, env=pip_config) + c.env = pip_config return c @@ -1428,18 +1522,61 @@ def pip_download(package_name): return c +def fallback_which(command, location=None, allow_global=False, system=False): + """ + A fallback implementation of the `which` utility command that relies exclusively on + searching the path for commands. + + :param str command: The command to search for, optional + :param str location: The search location to prioritize (prepend to path), defaults to None + :param bool allow_global: Whether to search the global path, defaults to False + :param bool system: Whether to use the system python instead of pipenv's python, defaults to False + :raises ValueError: Raised if no command is provided + :raises TypeError: Raised if the command provided is not a string + :return: A path to the discovered command location + :rtype: str + """ + + from .vendor.pythonfinder import Finder + if not command: + raise ValueError("fallback_which: Must provide a command to search for...") + if not isinstance(command, six.string_types): + raise TypeError("Provided command must be a string, received {0!r}".format(command)) + global_search = system or allow_global + if location is None: + global_search = True + finder = Finder(system=False, global_search=global_search, path=location) + if is_python_command(command): + result = find_python(finder, command) + if result: + return result + result = finder.which(command) + if result: + return result.path.as_posix() + return "" + + def which_pip(allow_global=False): """Returns the location of virtualenv-installed pip.""" + + location = None + if "VIRTUAL_ENV" in os.environ: + location = os.environ["VIRTUAL_ENV"] if allow_global: - if "VIRTUAL_ENV" in os.environ: - return which("pip", location=os.environ["VIRTUAL_ENV"]) + if location: + pip = which("pip", location=location) + if pip: + return pip for p in ("pip", "pip3", "pip2"): where = system_which(p) if where: return where - return which("pip") + pip = which("pip") + if not pip: + pip = fallback_which("pip", allow_global=allow_global, location=location) + return pip def system_which(command, mult=False): @@ -1449,6 +1586,7 @@ def system_which(command, mult=False): vistir.compat.fs_str(k): vistir.compat.fs_str(val) for k, val in os.environ.items() } + result = None try: c = delegator.run("{0} {1}".format(_which, command)) try: @@ -1463,21 +1601,19 @@ def system_which(command, mult=False): ) assert c.return_code == 0 except AssertionError: - return None if not mult else [] + result = fallback_which(command, allow_global=True) except TypeError: - from .vendor.pythonfinder import Finder - finder = Finder() - result = finder.which(command) - if result: - return result.path.as_posix() - return + if not result: + result = fallback_which(command, allow_global=True) else: - result = c.out.strip() or c.err.strip() - if mult: - return result.split("\n") - - else: - return result.split("\n")[0] + if not result: + result = next(iter([c.out, c.err]), "").split("\n") + result = next(iter(result)) if not mult else result + return result + if not result: + result = fallback_which(command, allow_global=True) + result = [result] if mult else result + return result def format_help(help): @@ -1614,6 +1750,17 @@ def ensure_lockfile(keep_outdated=False, pypi_mirror=None): def do_py(system=False): + if not project.virtualenv_exists: + click.echo( + "{}({}){}".format( + crayons.red("No virtualenv has been created for this project "), + crayons.white(project.project_directory, bold=True), + crayons.red(" yet!") + ), + err=True, + ) + return + try: click.echo(which("python", allow_global=system)) except AttributeError: @@ -1623,7 +1770,9 @@ def do_py(system=False): def do_outdated(pypi_mirror=None): # TODO: Allow --skip-lock here? from .vendor.requirementslib.models.requirements import Requirement + from .vendor.requirementslib.models.utils import get_version from .vendor.packaging.utils import canonicalize_name + from .vendor.vistir.compat import Mapping from collections import namedtuple packages = {} @@ -1635,6 +1784,7 @@ def do_outdated(pypi_mirror=None): (pkg.project_name, pkg.parsed_version, pkg.latest_version) for pkg in project.environment.get_outdated_packages() } + reverse_deps = project.environment.reverse_dependencies() for result in installed_packages: dep = Requirement.from_line(str(result.as_requirement())) packages.update(dep.as_pipfile()) @@ -1658,10 +1808,26 @@ def do_outdated(pypi_mirror=None): elif canonicalize_name(package) in outdated_packages: skipped.append(outdated_packages[canonicalize_name(package)]) for package, old_version, new_version in skipped: - click.echo(crayons.yellow( - "Skipped Update of Package {0!s}: {1!s} installed, {2!s} available.".format( - package, old_version, new_version - )), err=True + name_in_pipfile = project.get_package_name_in_pipfile(package) + pipfile_version_text = "" + required = "" + version = None + if name_in_pipfile: + version = get_version(project.packages[name_in_pipfile]) + reverse_deps = reverse_deps.get(name_in_pipfile) + if isinstance(reverse_deps, Mapping) and "required" in reverse_deps: + required = " {0} required".format(reverse_deps["required"]) + if version: + pipfile_version_text = " ({0} set in Pipfile)".format(version) + else: + pipfile_version_text = " (Unpinned in Pipfile)" + click.echo( + crayons.yellow( + "Skipped Update of Package {0!s}: {1!s} installed,{2!s}{3!s}, " + "{4!s} available.".format( + package, old_version, required, pipfile_version_text, new_version + ) + ), err=True ) if not outdated: click.echo(crayons.green("All packages are up to date!", bold=True)) @@ -1814,26 +1980,6 @@ def do_install( for req in import_from_code(code): click.echo(" Found {0}!".format(crayons.green(req))) project.add_package_to_pipfile(req) - # Install editable local packages before locking - this gives us access to dist-info - if project.pipfile_exists and ( - # double negatives are for english readability, leave them alone. - (not project.lockfile_exists and not deploy) - or (not project.virtualenv_exists and not system) - ): - section = ( - project.editable_packages if not dev else project.editable_dev_packages - ) - for package in section.keys(): - req = convert_deps_to_pip( - {package: section[package]}, project=project, r=False - ) - if req: - req = req[0] - req = req[len("-e ") :] if req.startswith("-e ") else req - if not editable_packages: - editable_packages = [req] - else: - editable_packages.extend([req]) # Allow more than one package to be provided. package_args = [p for p in packages] + [ "-e {0}".format(pkg) for pkg in editable_packages @@ -1852,7 +1998,7 @@ def do_install( if not is_star(section[package__name]) and is_star(package__val): # Support for VCS dependencies. package_args[i] = convert_deps_to_pip( - {packages: section[package__name]}, project=project, r=False + {package__name: section[package__name]}, project=project, r=False )[0] except KeyError: pass @@ -1877,12 +2023,12 @@ def do_install( keep_outdated=keep_outdated ) - # This is for if the user passed in dependencies, then we want to maek sure we + # This is for if the user passed in dependencies, then we want to make sure we else: - from .vendor.requirementslib import Requirement + from .vendor.requirementslib.models.requirements import Requirement # make a tuple of (display_name, entry) - pkg_list = packages + ["-e {0}".format(pkg) for pkg in editable_packages] + pkg_list = packages + ['-e {0}'.format(pkg) for pkg in editable_packages] if not system and not project.virtualenv_exists: do_init( dev=dev, @@ -1895,6 +2041,7 @@ def do_install( pypi_mirror=pypi_mirror, skip_lock=skip_lock, ) + pip_shims_module = os.environ.pop("PIP_SHIMS_BASE_MODULE", None) for pkg_line in pkg_list: click.echo( crayons.normal( @@ -1906,36 +2053,73 @@ def do_install( with vistir.contextmanagers.temp_environ(), create_spinner("Installing...") as sp: if not system: os.environ["PIP_USER"] = vistir.compat.fs_str("0") + if "PYTHONHOME" in os.environ: + del os.environ["PYTHONHOME"] + sp.text = "Resolving {0}...".format(pkg_line) try: pkg_requirement = Requirement.from_line(pkg_line) except ValueError as e: sp.write_err(vistir.compat.fs_str("{0}: {1}".format(crayons.red("WARNING"), e))) - sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format("Installation Failed")) + sp.red.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format("Installation Failed")) sys.exit(1) if index_url: pkg_requirement.index = index_url + no_deps = False + sp.text = "Installing..." try: + sp.text = "Installing {0}...".format(pkg_requirement.name) + if environments.is_verbose(): + sp.hide_and_write("Installing package: {0}".format(pkg_requirement.as_line(include_hashes=False))) c = pip_install( pkg_requirement, ignore_hashes=True, allow_global=system, selective_upgrade=selective_upgrade, - no_deps=False, + no_deps=no_deps, pre=pre, requirements_dir=requirements_directory, index=index_url, extra_indexes=extra_index_url, pypi_mirror=pypi_mirror, ) + if not c.ok: + sp.write_err(u"{0}: {1}".format( + crayons.red("WARNING"), + vistir.compat.fs_str("Failed installing package {0}".format(pkg_line))) + ) + sp.write_err( + vistir.compat.fs_str(u"Error text: {0}".format(c.out)) + ) + sp.write_err( + vistir.compat.fs_str(u"{0}".format(c.err)) + ) + sp.write_err( + u"{0} An error occurred while installing {1}!".format( + crayons.red(u"Error: ", bold=True), crayons.green(pkg_line) + ), + ) + sp.write_err(crayons.blue(vistir.compat.fs_str(format_pip_error(c.err)))) + if environments.is_verbose(): + sp.write_err(crayons.blue(vistir.compat.fs_str(format_pip_output(c.out)))) + if "setup.py egg_info" in c.err: + sp.write_err(vistir.compat.fs_str( + "This is likely caused by a bug in {0}. " + "Report this to its maintainers.".format( + crayons.green(pkg_requirement.name) + ) + )) + sp.red.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format("Installation Failed")) + sys.exit(1) except (ValueError, RuntimeError) as e: sp.write_err(vistir.compat.fs_str( "{0}: {1}".format(crayons.red("WARNING"), e), )) - sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format( + sp.red.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format( "Installation Failed", )) + sys.exit(1) # Warn if --editable wasn't passed. - if pkg_requirement.is_vcs and not pkg_requirement.editable: + if pkg_requirement.is_vcs and not pkg_requirement.editable and not PIPENV_RESOLVE_VCS: sp.write_err( "{0}: You installed a VCS dependency in non-editable mode. " "This will work fine, but sub-dependencies will not be resolved by {1}." @@ -1945,24 +2129,6 @@ def do_install( crayons.red("$ pipenv lock"), ) ) - click.echo(crayons.blue(format_pip_output(c.out))) - # Ensure that package was successfully installed. - if c.return_code != 0: - sp.write_err(vistir.compat.fs_str( - "{0} An error occurred while installing {1}!".format( - crayons.red("Error: ", bold=True), crayons.green(pkg_line) - ), - )) - sp.write_err(vistir.compat.fs_str(crayons.blue(format_pip_error(c.err)))) - if "setup.py egg_info" in c.err: - sp.write_err(vistir.compat.fs_str( - "This is likely caused by a bug in {0}. " - "Report this to its maintainers.".format( - crayons.green(pkg_requirement.name) - ) - )) - sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format("Installation Failed")) - sys.exit(1) sp.write(vistir.compat.fs_str( u"{0} {1} {2} {3}{4}".format( crayons.normal(u"Adding", bold=True), @@ -1972,15 +2138,25 @@ def do_install( crayons.normal(fix_utf8("…"), bold=True), ) )) + # Add the package to the Pipfile. + try: + project.add_package_to_pipfile(pkg_requirement, dev) + except ValueError: + import traceback + sp.write_err( + "{0} {1}".format( + crayons.red("Error:", bold=True), traceback.format_exc() + ) + ) + sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format( + "Failed adding package to Pipfile" + )) sp.ok(environments.PIPENV_SPINNER_OK_TEXT.format("Installation Succeeded")) - # Add the package to the Pipfile. - try: - project.add_package_to_pipfile(pkg_requirement, dev) - except ValueError as e: - raise exceptions.PipfileException(e) # Update project settings with pre preference. if pre: project.update_settings({"allow_prereleases": pre}) + if pip_shims_module: + os.environ["PIP_SHIMS_BASE_MODULE"] = pip_shims_module do_init( dev=dev, system=system, @@ -2092,6 +2268,7 @@ def do_uninstall( p for normalized, p in selected_pkg_map.items() if normalized in (used_packages - bad_pkgs) ] + pip_path = None for normalized, package_name in selected_pkg_map.items(): click.echo( crayons.white( @@ -2101,12 +2278,10 @@ def do_uninstall( # Uninstall the package. if package_name in packages_to_remove: with project.environment.activated(): - cmd = "{0} uninstall {1} -y".format( - escape_grouped_arguments(which_pip(allow_global=system)), package_name, - ) - if environments.is_verbose(): - click.echo("$ {0}".format(cmd)) - c = delegator.run(cmd) + if pip_path is None: + pip_path = which_pip(allow_global=system) + cmd = [pip_path, "uninstall", package_name, "-y"] + c = run_command(cmd) click.echo(crayons.blue(c.out)) if c.return_code != 0: failure = True @@ -2157,11 +2332,6 @@ def do_shell(three=None, python=False, fancy=False, shell_args=None, pypi_mirror three=three, python=python, validate=False, pypi_mirror=pypi_mirror, ) - # Set an environment variable, so we know we're in the environment. - os.environ["PIPENV_ACTIVE"] = vistir.misc.fs_str("1") - - os.environ.pop("PIP_SHIMS_BASE_MODULE", None) - # Support shell compatibility mode. if PIPENV_SHELL_FANCY: fancy = True @@ -2177,6 +2347,13 @@ def do_shell(three=None, python=False, fancy=False, shell_args=None, pypi_mirror shell_args, ) + # Set an environment variable, so we know we're in the environment. + # Only set PIPENV_ACTIVE after finishing reading virtualenv_location + # otherwise its value will be changed + os.environ["PIPENV_ACTIVE"] = vistir.misc.fs_str("1") + + os.environ.pop("PIP_SHIMS_BASE_MODULE", None) + if fancy: shell.fork(*fork_args) return @@ -2311,22 +2488,44 @@ def do_run(command, args, three=None, python=False, pypi_mirror=None): three=three, python=python, validate=False, pypi_mirror=pypi_mirror, ) - # Set an environment variable, so we know we're in the environment. - os.environ["PIPENV_ACTIVE"] = vistir.misc.fs_str("1") - - os.environ.pop("PIP_SHIMS_BASE_MODULE", None) load_dot_env() + previous_pip_shims_module = os.environ.pop("PIP_SHIMS_BASE_MODULE", None) + # Activate virtualenv under the current interpreter's environment inline_activate_virtual_environment() + + # Set an environment variable, so we know we're in the environment. + # Only set PIPENV_ACTIVE after finishing reading virtualenv_location + # such as in inline_activate_virtual_environment + # otherwise its value will be changed + previous_pipenv_active_value = os.environ.get("PIPENV_ACTIVE") + os.environ["PIPENV_ACTIVE"] = vistir.misc.fs_str("1") + + os.environ.pop("PIP_SHIMS_BASE_MODULE", None) + try: script = project.build_script(command, args) + cmd_string = ' '.join([script.command] + script.args) + if environments.is_verbose(): + click.echo(crayons.normal("$ {0}".format(cmd_string)), err=True) except ScriptEmptyError: click.echo("Can't run script {0!r}-it's empty?", err=True) + run_args = [script] + run_kwargs = {} if os.name == "nt": - do_run_nt(script) + run_fn = do_run_nt else: - do_run_posix(script, command=command) + run_fn = do_run_posix + run_kwargs = {"command": command} + try: + run_fn(*run_args, **run_kwargs) + finally: + os.environ.pop("PIPENV_ACTIVE", None) + if previous_pipenv_active_value is not None: + os.environ["PIPENV_ACTIVE"] = previous_pipenv_active_value + if previous_pip_shims_module is not None: + os.environ["PIP_SHIMS_BASE_MODULE"] = previous_pip_shims_module def do_check( @@ -2338,6 +2537,9 @@ def do_check( args=None, pypi_mirror=None, ): + from pipenv.vendor.vistir.compat import JSONDecodeError + from pipenv.vendor.first import first + if not system: # Ensure that virtualenv is available. ensure_project( @@ -2350,8 +2552,8 @@ def do_check( if not args: args = [] if unused: - deps_required = [k for k in project.packages.keys()] - deps_needed = import_from_code(unused) + deps_required = [k.lower() for k in project.packages.keys()] + deps_needed = [k.lower() for k in import_from_code(unused)] for dep in deps_needed: try: deps_required.remove(dep) @@ -2368,18 +2570,32 @@ def do_check( sys.exit(1) else: sys.exit(0) - click.echo(crayons.normal(fix_utf8("Checking PEP 508 requirements…"), bold=True)) - if system: - python = system_which("python") - else: - python = which("python") - # Run the PEP 508 checker in the virtualenv. - c = delegator.run( - '"{0}" {1}'.format( - python, escape_grouped_arguments(pep508checker.__file__.rstrip("cdo")) - ) + click.echo(crayons.normal(decode_for_output("Checking PEP 508 requirements…"), bold=True)) + pep508checker_path = pep508checker.__file__.rstrip("cdo") + safety_path = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "patched", "safety.zip" ) - results = simplejson.loads(c.out) + if not system: + python = which("python") + else: + python = first(system_which(p) for p in ("python", "python3", "python2")) + if not python: + click.echo(crayons.red("The Python interpreter can't be found."), err=True) + sys.exit(1) + _cmd = [vistir.compat.Path(python).as_posix()] + # Run the PEP 508 checker in the virtualenv. + cmd = _cmd + [vistir.compat.Path(pep508checker_path).as_posix()] + c = run_command(cmd) + if c.return_code is not None: + try: + results = simplejson.loads(c.out.strip()) + except JSONDecodeError: + click.echo("{0}\n{1}\n{2}".format( + crayons.white(decode_for_output("Failed parsing pep508 results: "), bold=True), + c.out.strip(), + c.err.strip() + )) + sys.exit(1) # Load the pipfile. p = pipfile.Pipfile.load(project.pipfile_location) failed = False @@ -2404,15 +2620,13 @@ def do_check( sys.exit(1) else: click.echo(crayons.green("Passed!")) - click.echo(crayons.normal(fix_utf8("Checking installed package safety…"), bold=True)) - path = pep508checker.__file__.rstrip("cdo") - path = os.sep.join(__file__.split(os.sep)[:-1] + ["patched", "safety.zip"]) - if not system: - python = which("python") - else: - python = system_which("python") + click.echo(crayons.normal( + decode_for_output("Checking installed package safety…"), bold=True) + ) if ignore: - ignored = "--ignore {0}".format(" --ignore ".join(ignore)) + if not isinstance(ignore, (tuple, list)): + ignore = [ignore] + ignored = [["--ignore", cve] for cve in ignore] click.echo( crayons.normal( "Notice: Ignoring CVE(s) {0}".format(crayons.yellow(", ".join(ignore))) @@ -2421,17 +2635,21 @@ def do_check( ) else: ignored = "" - c = delegator.run( - '"{0}" {1} check --json --key={2} {3}'.format( - python, escape_grouped_arguments(path), PIPENV_PYUP_API_KEY, ignored - ) - ) + key = "--key={0}".format(PIPENV_PYUP_API_KEY) + cmd = _cmd + [safety_path, "check", "--json", key] + if ignored: + for cve in ignored: + cmd += cve + c = run_command(cmd, catch_exceptions=False) try: results = simplejson.loads(c.out) - except ValueError: - click.echo("An error occurred:", err=True) - click.echo(c.err if len(c.err) > 0 else c.out, err=True) - sys.exit(1) + except (ValueError, JSONDecodeError): + raise exceptions.JSONParseError(c.out, c.err) + except Exception: + raise exceptions.PipenvCmdError(c.cmd, c.out, c.err, c.return_code) + if c.ok: + click.echo(crayons.green("All good!")) + sys.exit(0) for (package, resolved, installed, description, vuln) in results: click.echo( "{0}: {1} {2} resolved ({3} installed)!".format( @@ -2443,14 +2661,14 @@ def do_check( ) click.echo("{0}".format(description)) click.echo() - if not results: - click.echo(crayons.green("All good!")) else: sys.exit(1) def do_graph(bare=False, json=False, json_tree=False, reverse=False): + from pipenv.vendor.vistir.compat import JSONDecodeError import pipdeptree + pipdeptree_path = pipdeptree.__file__.rstrip("cdo") try: python_path = which("python") except AttributeError: @@ -2465,6 +2683,9 @@ def do_graph(bare=False, json=False, json_tree=False, reverse=False): sys.exit(1) except RuntimeError: pass + else: + python_path = vistir.compat.Path(python_path).as_posix() + pipdeptree_path = vistir.compat.Path(pipdeptree_path).as_posix() if reverse and json: click.echo( @@ -2506,7 +2727,7 @@ def do_graph(bare=False, json=False, json_tree=False, reverse=False): if not project.virtualenv_exists: click.echo( u"{0}: No virtualenv has been created for this project yet! Consider " - u"running {1} first to automatically generate one for you or see" + u"running {1} first to automatically generate one for you or see " u"{2} for further instructions.".format( crayons.red("Warning", bold=True), crayons.green("`pipenv install`"), @@ -2515,17 +2736,20 @@ def do_graph(bare=False, json=False, json_tree=False, reverse=False): err=True, ) sys.exit(1) - cmd = '"{0}" {1} {2} -l'.format( - python_path, escape_grouped_arguments(pipdeptree.__file__.rstrip("cdo")), flag - ) + cmd_args = [python_path, pipdeptree_path, flag, "-l"] + c = run_command(cmd_args) # Run dep-tree. - c = delegator.run(cmd) if not bare: if json: data = [] - for d in simplejson.loads(c.out): - if d["package"]["key"] not in BAD_PACKAGES: - data.append(d) + try: + parsed = simplejson.loads(c.out.strip()) + except JSONDecodeError: + raise exceptions.JSONParseError(c.out, c.err) + else: + for d in parsed: + if d["package"]["key"] not in BAD_PACKAGES: + data.append(d) click.echo(simplejson.dumps(data, indent=4)) sys.exit(0) elif json_tree: @@ -2541,12 +2765,18 @@ def do_graph(bare=False, json=False, json_tree=False, reverse=False): obj["dependencies"] = traverse(obj["dependencies"]) return obj - data = traverse(simplejson.loads(c.out)) - click.echo(simplejson.dumps(data, indent=4)) - sys.exit(0) + try: + parsed = simplejson.loads(c.out.strip()) + except JSONDecodeError: + raise exceptions.JSONParseError(c.out, c.err) + else: + data = traverse(parsed) + click.echo(simplejson.dumps(data, indent=4)) + sys.exit(0) else: for line in c.out.strip().split("\n"): # Ignore bad packages as top level. + # TODO: This should probably be a "==" in + line.partition if line.split("==")[0] in BAD_PACKAGES and not reverse: continue @@ -2587,7 +2817,7 @@ def do_sync( ): # The lock file needs to exist because sync won't write to it. if not project.lockfile_exists: - raise exceptions.LockfileNotFound(project.lockfile_location) + raise exceptions.LockfileNotFound("Pipfile.lock") # Ensure that virtualenv is available if not system. ensure_project( @@ -2615,34 +2845,32 @@ def do_sync( click.echo(crayons.green("All dependencies are now up-to-date!")) -def do_clean(ctx, three=None, python=None, dry_run=False, bare=False, pypi_mirror=None): +def do_clean( + ctx, three=None, python=None, dry_run=False, bare=False, pypi_mirror=None, + system=False +): # Ensure that virtualenv is available. from packaging.utils import canonicalize_name ensure_project(three=three, python=python, validate=False, pypi_mirror=pypi_mirror) ensure_lockfile(pypi_mirror=pypi_mirror) # Make sure that the virtualenv's site packages are configured correctly # otherwise we may end up removing from the global site packages directory - installed_package_names = [ - canonicalize_name(pkg.project_name) for pkg - in project.environment.get_installed_packages() - ] + installed_package_names = project.installed_package_names.copy() # Remove known "bad packages" from the list. for bad_package in BAD_PACKAGES: if canonicalize_name(bad_package) in installed_package_names: if environments.is_verbose(): click.echo("Ignoring {0}.".format(bad_package), err=True) - del installed_package_names[installed_package_names.index( - canonicalize_name(bad_package) - )] + installed_package_names.remove(canonicalize_name(bad_package)) # Intelligently detect if --dev should be used or not. - develop = [canonicalize_name(k) for k in project.lockfile_content["develop"].keys()] - default = [canonicalize_name(k) for k in project.lockfile_content["default"].keys()] - for used_package in set(develop + default): + locked_packages = { + canonicalize_name(pkg) for pkg in project.lockfile_package_names["combined"] + } + for used_package in locked_packages: if used_package in installed_package_names: - del installed_package_names[installed_package_names.index( - canonicalize_name(used_package) - )] + installed_package_names.remove(used_package) failure = False + cmd = [which_pip(allow_global=system), "uninstall", "-y", "-qq"] for apparent_bad_package in installed_package_names: if dry_run and not bare: click.echo(apparent_bad_package) @@ -2654,9 +2882,8 @@ def do_clean(ctx, three=None, python=None, dry_run=False, bare=False, pypi_mirro ) ) # Uninstall the package. - c = delegator.run( - "{0} uninstall {1} -y".format(which_pip(), apparent_bad_package) - ) + cmd = [which_pip(), "uninstall", apparent_bad_package, "-y"] + c = run_command(cmd) if c.return_code != 0: failure = True sys.exit(int(failure)) diff --git a/pipenv/environment.py b/pipenv/environment.py index 7558f94c..4a694019 100644 --- a/pipenv/environment.py +++ b/pipenv/environment.py @@ -1,24 +1,29 @@ # -*- coding=utf-8 -*- +from __future__ import absolute_import, print_function import contextlib import importlib +import io import json -import os -import sys import operator -import pkg_resources +import os import site -import six +import sys from distutils.sysconfig import get_python_lib -from sysconfig import get_paths +from sysconfig import get_paths, get_python_version -from cached_property import cached_property +import itertools +import pkg_resources +import six -import vistir import pipenv -from .utils import normalize_path +from .vendor.cached_property import cached_property +from .vendor import vistir + +from .utils import normalize_path, make_posix + BASE_WORKING_SET = pkg_resources.WorkingSet(sys.path) @@ -43,6 +48,9 @@ class Environment(object): self.extra_dists = [] prefix = prefix if prefix else sys.prefix self.prefix = vistir.compat.Path(prefix) + self._base_paths = {} + if self.is_venv: + self._base_paths = self.get_paths() self.sys_paths = get_paths() def safe_import(self, name): @@ -90,22 +98,37 @@ class Environment(object): deps |= cls.resolve_dist(dist, working_set) return deps - def add_dist(self, dist_name): - dist = pkg_resources.get_distribution(pkg_resources.Requirement(dist_name)) + def extend_dists(self, dist): extras = self.resolve_dist(dist, self.base_working_set) + self.extra_dists.append(dist) if extras: self.extra_dists.extend(extras) + def add_dist(self, dist_name): + dist = pkg_resources.get_distribution(pkg_resources.Requirement(dist_name)) + self.extend_dists(dist) + @cached_property def python_version(self): with self.activated(): - from sysconfig import get_python_version - py_version = get_python_version() + sysconfig = self.safe_import("sysconfig") + py_version = sysconfig.get_python_version() return py_version + def find_libdir(self): + libdir = self.prefix / "lib" + return next(iter(list(libdir.iterdir())), None) + @property def python_info(self): include_dir = self.prefix / "include" + if not os.path.exists(include_dir): + include_dirs = self.get_include_path() + if include_dirs: + include_path = include_dirs.get("include", include_dirs.get("platinclude")) + if not include_path: + return {} + include_dir = vistir.compat.Path(include_path) python_path = next(iter(list(include_dir.iterdir())), None) if python_path and python_path.name.startswith("python"): python_version = python_path.name.replace("python", "") @@ -113,6 +136,16 @@ class Environment(object): return {"py_version_short": py_version_short, "abiflags": abiflags} return {} + def _replace_parent_version(self, path, replace_version): + if not os.path.exists(path): + base, leaf = os.path.split(path) + base, parent = os.path.split(base) + leaf = os.path.join(parent, leaf).replace( + replace_version, self.python_info.get("py_version_short", get_python_version()) + ) + return os.path.join(base, leaf) + return path + @cached_property def base_paths(self): """ @@ -143,50 +176,83 @@ class Environment(object): 'stdlib': '/home/hawk/.pyenv/versions/3.7.1/lib/python3.7'} """ - prefix = self.prefix.as_posix() - install_scheme = 'nt' if (os.name == 'nt') else 'posix_prefix' - paths = get_paths(install_scheme, vars={ - 'base': prefix, - 'platbase': prefix, - }) + prefix = make_posix(self.prefix.as_posix()) + paths = {} + if self._base_paths: + paths = self._base_paths.copy() + else: + try: + paths = self.get_paths() + except Exception: + install_scheme = 'nt' if (os.name == 'nt') else 'posix_prefix' + paths = get_paths(install_scheme, vars={ + 'base': prefix, + 'platbase': prefix, + }) + current_version = get_python_version() + try: + for k in list(paths.keys()): + if not os.path.exists(paths[k]): + paths[k] = self._replace_parent_version(paths[k], current_version) + except OSError: + # Sometimes virtualenvs are made using virtualenv interpreters and there is no + # include directory, which will cause this approach to fail. This failsafe + # will make sure we fall back to the shell execution to find the real include path + paths = self.get_include_path() + paths.update(self.get_lib_paths()) + paths["scripts"] = self.script_basedir + if not paths: + install_scheme = 'nt' if (os.name == 'nt') else 'posix_prefix' + paths = get_paths(install_scheme, vars={ + 'base': prefix, + 'platbase': prefix, + }) + if not os.path.exists(paths["purelib"]) and not os.path.exists(paths["platlib"]): + lib_paths = self.get_lib_paths() + paths.update(lib_paths) paths["PATH"] = paths["scripts"] + os.pathsep + os.defpath if "prefix" not in paths: paths["prefix"] = prefix - purelib = get_python_lib(plat_specific=0, prefix=prefix) - platlib = get_python_lib(plat_specific=1, prefix=prefix) + purelib = paths["purelib"] = make_posix(paths["purelib"]) + platlib = paths["platlib"] = make_posix(paths["platlib"]) if purelib == platlib: lib_dirs = purelib else: lib_dirs = purelib + os.pathsep + platlib paths["libdir"] = purelib - paths["purelib"] = purelib - paths["platlib"] = platlib - paths['PYTHONPATH'] = lib_dirs + paths['PYTHONPATH'] = os.pathsep.join(["", ".", lib_dirs]) paths["libdirs"] = lib_dirs return paths @cached_property def script_basedir(self): """Path to the environment scripts dir""" - script_dir = self.base_paths["scripts"] - return script_dir + prefix = make_posix(self.prefix.as_posix()) + install_scheme = 'nt' if (os.name == 'nt') else 'posix_prefix' + paths = get_paths(install_scheme, vars={ + 'base': prefix, + 'platbase': prefix, + }) + return paths["scripts"] @property def python(self): """Path to the environment python""" - py = vistir.compat.Path(self.base_paths["scripts"]).joinpath("python").as_posix() + py = vistir.compat.Path(self.script_basedir).joinpath("python").absolute().as_posix() if not py: return vistir.compat.Path(sys.executable).as_posix() return py @cached_property def sys_path(self): - """The system path inside the environment + """ + The system path inside the environment :return: The :data:`sys.path` from the environment :rtype: list """ + from .vendor.vistir.compat import JSONDecodeError current_executable = vistir.compat.Path(sys.executable).as_posix() if not self.python or self.python == current_executable: return sys.path @@ -194,18 +260,173 @@ class Environment(object): return sys.path cmd_args = [self.python, "-c", "import json, sys; print(json.dumps(sys.path))"] path, _ = vistir.misc.run(cmd_args, return_object=False, nospin=True, block=True, combine_stderr=False, write_to_stdout=False) - path = json.loads(path.strip()) + try: + path = json.loads(path.strip()) + except JSONDecodeError: + path = sys.path return path + def build_command(self, python_lib=False, python_inc=False, scripts=False, py_version=False): + """Build the text for running a command in the given environment + + :param python_lib: Whether to include the python lib dir commands, defaults to False + :type python_lib: bool, optional + :param python_inc: Whether to include the python include dir commands, defaults to False + :type python_inc: bool, optional + :param scripts: Whether to include the scripts directory, defaults to False + :type scripts: bool, optional + :param py_version: Whether to include the python version info, defaults to False + :type py_version: bool, optional + :return: A string representing the command to run + """ + pylib_lines = [] + pyinc_lines = [] + py_command = ( + "import sysconfig, distutils.sysconfig, io, json, sys; paths = {{" + "%s }}; value = u'{{0}}'.format(json.dumps(paths));" + "fh = io.open('{0}', 'w'); fh.write(value); fh.close()" + ) + distutils_line = "distutils.sysconfig.get_python_{0}(plat_specific={1})" + sysconfig_line = "sysconfig.get_path('{0}')" + if python_lib: + for key, var, val in (("pure", "lib", "0"), ("plat", "lib", "1")): + dist_prefix = "{0}lib".format(key) + # XXX: We need to get 'stdlib' or 'platstdlib' + sys_prefix = "{0}stdlib".format("" if key == "pure" else key) + pylib_lines.append("u'%s': u'{{0}}'.format(%s)" % (dist_prefix, distutils_line.format(var, val))) + pylib_lines.append("u'%s': u'{{0}}'.format(%s)" % (sys_prefix, sysconfig_line.format(sys_prefix))) + if python_inc: + for key, var, val in (("include", "inc", "0"), ("platinclude", "inc", "1")): + pylib_lines.append("u'%s': u'{{0}}'.format(%s)" % (key, distutils_line.format(var, val))) + lines = pylib_lines + pyinc_lines + if scripts: + lines.append("u'scripts': u'{{0}}'.format(%s)" % sysconfig_line.format("scripts")) + if py_version: + lines.append("u'py_version_short': u'{{0}}'.format(distutils.sysconfig.get_python_version()),") + lines_as_str = u",".join(lines) + py_command = py_command % lines_as_str + return py_command + + def get_paths(self): + """ + Get the paths for the environment by running a subcommand + + :return: The python paths for the environment + :rtype: Dict[str, str] + """ + tmpfile = vistir.path.create_tracked_tempfile(suffix=".json") + tmpfile.close() + tmpfile_path = make_posix(tmpfile.name) + py_command = self.build_command(python_lib=True, python_inc=True, scripts=True, py_version=True) + command = [self.python, "-c", py_command.format(tmpfile_path)] + c = vistir.misc.run( + command, return_object=True, block=True, nospin=True, write_to_stdout=False + ) + if c.returncode == 0: + paths = {} + with io.open(tmpfile_path, "r", encoding="utf-8") as fh: + paths = json.load(fh) + if "purelib" in paths: + paths["libdir"] = paths["purelib"] = make_posix(paths["purelib"]) + for key in ("platlib", "scripts", "platstdlib", "stdlib", "include", "platinclude"): + if key in paths: + paths[key] = make_posix(paths[key]) + return paths + else: + vistir.misc.echo("Failed to load paths: {0}".format(c.err), fg="yellow") + vistir.misc.echo("Output: {0}".format(c.out), fg="yellow") + return None + + def get_lib_paths(self): + """Get the include path for the environment + + :return: The python include path for the environment + :rtype: Dict[str, str] + """ + tmpfile = vistir.path.create_tracked_tempfile(suffix=".json") + tmpfile.close() + tmpfile_path = make_posix(tmpfile.name) + py_command = self.build_command(python_lib=True) + command = [self.python, "-c", py_command.format(tmpfile_path)] + c = vistir.misc.run( + command, return_object=True, block=True, nospin=True, write_to_stdout=False + ) + paths = None + if c.returncode == 0: + paths = {} + with io.open(tmpfile_path, "r", encoding="utf-8") as fh: + paths = json.load(fh) + if "purelib" in paths: + paths["libdir"] = paths["purelib"] = make_posix(paths["purelib"]) + for key in ("platlib", "platstdlib", "stdlib"): + if key in paths: + paths[key] = make_posix(paths[key]) + return paths + else: + vistir.misc.echo("Failed to load paths: {0}".format(c.err), fg="yellow") + vistir.misc.echo("Output: {0}".format(c.out), fg="yellow") + if not paths: + if not self.prefix.joinpath("lib").exists(): + return {} + stdlib_path = next(iter([ + p for p in self.prefix.joinpath("lib").iterdir() + if p.name.startswith("python") + ]), None) + lib_path = None + if stdlib_path: + lib_path = next(iter([ + p.as_posix() for p in stdlib_path.iterdir() + if p.name == "site-packages" + ])) + paths = {"stdlib": stdlib_path.as_posix()} + if lib_path: + paths["purelib"] = lib_path + return paths + return {} + + def get_include_path(self): + """Get the include path for the environment + + :return: The python include path for the environment + :rtype: Dict[str, str] + """ + tmpfile = vistir.path.create_tracked_tempfile(suffix=".json") + tmpfile.close() + tmpfile_path = make_posix(tmpfile.name) + py_command = ( + "import distutils.sysconfig, io, json, sys; paths = {{u'include': " + "u'{{0}}'.format(distutils.sysconfig.get_python_inc(plat_specific=0)), " + "u'platinclude': u'{{0}}'.format(distutils.sysconfig.get_python_inc(" + "plat_specific=1)) }}; value = u'{{0}}'.format(json.dumps(paths));" + "fh = io.open('{0}', 'w'); fh.write(value); fh.close()" + ) + command = [self.python, "-c", py_command.format(tmpfile_path)] + c = vistir.misc.run( + command, return_object=True, block=True, nospin=True, write_to_stdout=False + ) + if c.returncode == 0: + paths = [] + with io.open(tmpfile_path, "r", encoding="utf-8") as fh: + paths = json.load(fh) + for key in ("include", "platinclude"): + if key in paths: + paths[key] = make_posix(paths[key]) + return paths + else: + vistir.misc.echo("Failed to load paths: {0}".format(c.err), fg="yellow") + vistir.misc.echo("Output: {0}".format(c.out), fg="yellow") + return None + @cached_property def sys_prefix(self): - """The prefix run inside the context of the environment + """ + The prefix run inside the context of the environment :return: The python prefix inside the environment :rtype: :data:`sys.prefix` """ - command = [self.python, "-c" "import sys; print(sys.prefix)"] + command = [self.python, "-c", "import sys; print(sys.prefix)"] c = vistir.misc.run(command, return_object=True, block=True, nospin=True, write_to_stdout=False) sys_prefix = vistir.compat.Path(vistir.misc.to_text(c.out).strip()).as_posix() return sys_prefix @@ -234,15 +455,33 @@ class Environment(object): return "purelib", purelib return "platlib", self.paths["platlib"] + @property + def pip_version(self): + """ + Get the pip version in the environment. Useful for knowing which args we can use + when installing. + """ + from .vendor.packaging.version import parse as parse_version + pip = next(iter( + pkg for pkg in self.get_installed_packages() if pkg.key == "pip" + ), None) + if pip is not None: + return parse_version(pip.version) + return parse_version("18.0") + def get_distributions(self): - """Retrives the distributions installed on the library path of the environment + """ + Retrives the distributions installed on the library path of the environment :return: A set of distributions found on the library path :rtype: iterator """ pkg_resources = self.safe_import("pkg_resources") - return pkg_resources.find_distributions(self.paths["PYTHONPATH"]) + libdirs = self.base_paths["libdirs"].split(os.pathsep) + dists = (pkg_resources.find_distributions(libdir) for libdir in libdirs) + for dist in itertools.chain.from_iterable(dists): + yield dist def find_egg(self, egg_dist): """Find an egg by name in the given environment""" @@ -269,21 +508,30 @@ class Environment(object): def dist_is_in_project(self, dist): """Determine whether the supplied distribution is in the environment.""" from .project import _normalized - prefix = _normalized(self.base_paths["prefix"]) + prefixes = [ + _normalized(prefix) for prefix in self.base_paths["libdirs"].split(os.pathsep) + if _normalized(prefix).startswith(_normalized(self.prefix.as_posix())) + ] location = self.locate_dist(dist) if not location: return False - return _normalized(location).startswith(prefix) + location = _normalized(make_posix(location)) + return any(location.startswith(prefix) for prefix in prefixes) def get_installed_packages(self): """Returns all of the installed packages in a given environment""" workingset = self.get_working_set() - packages = [pkg for pkg in workingset if self.dist_is_in_project(pkg)] + packages = [ + pkg for pkg in workingset + if self.dist_is_in_project(pkg) and pkg.key != "python" + ] return packages @contextlib.contextmanager def get_finder(self, pre=False): - from .vendor.pip_shims import Command, cmdoptions, index_group, PackageFinder + from .vendor.pip_shims.shims import ( + Command, cmdoptions, index_group, PackageFinder, parse_version, pip_version + ) from .environments import PIPENV_CACHE_DIR index_urls = [source.get("url") for source in self.sources] @@ -302,25 +550,35 @@ class Environment(object): pip_options.cache_dir = PIPENV_CACHE_DIR pip_options.pre = self.pipfile.get("pre", pre) with pip_command._build_session(pip_options) as session: - finder = PackageFinder( - find_links=pip_options.find_links, - index_urls=index_urls, allow_all_prereleases=pip_options.pre, - trusted_hosts=pip_options.trusted_hosts, - process_dependency_links=pip_options.process_dependency_links, - session=session - ) + finder_args = { + "find_links": pip_options.find_links, + "index_urls": index_urls, + "allow_all_prereleases": pip_options.pre, + "trusted_hosts": pip_options.trusted_hosts, + "session": session + } + if parse_version(pip_version) < parse_version("19.0"): + finder_args.update( + {"process_dependency_links": pip_options.process_dependency_links} + ) + finder = PackageFinder(**finder_args) yield finder def get_package_info(self, pre=False): + from .vendor.pip_shims.shims import pip_version, parse_version dependency_links = [] packages = self.get_installed_packages() # This code is borrowed from pip's current implementation - for dist in packages: - if dist.has_metadata('dependency_links.txt'): - dependency_links.extend(dist.get_metadata_lines('dependency_links.txt')) + if parse_version(pip_version) < parse_version("19.0"): + for dist in packages: + if dist.has_metadata('dependency_links.txt'): + dependency_links.extend( + dist.get_metadata_lines('dependency_links.txt') + ) with self.get_finder() as finder: - finder.add_dependency_links(dependency_links) + if parse_version(pip_version) < parse_version("19.0"): + finder.add_dependency_links(dependency_links) for dist in packages: typ = 'unknown' @@ -348,36 +606,89 @@ class Environment(object): def get_outdated_packages(self, pre=False): return [ pkg for pkg in self.get_package_info(pre=pre) - if pkg.latest_version._version > pkg.parsed_version._version + if pkg.latest_version._key > pkg.parsed_version._key ] - def get_package_requirements(self): - from .vendor.pipdeptree import flatten, sorted_tree, build_dist_index, construct_tree - dist_index = build_dist_index(self.get_installed_packages()) - tree = sorted_tree(construct_tree(dist_index)) - branch_keys = set(r.key for r in flatten(tree.values())) - nodes = [p for p in tree.keys() if p.key not in branch_keys] - key_tree = dict((k.key, v) for k, v in tree.items()) + @classmethod + def _get_requirements_for_package(cls, node, key_tree, parent=None, chain=None): + if chain is None: + chain = [node.project_name] + + d = node.as_dict() + if parent: + d['required_version'] = node.version_spec if node.version_spec else 'Any' + else: + d['required_version'] = d['installed_version'] + get_children = lambda n: key_tree.get(n.key, []) - def aux(node, parent=None, chain=None): - if chain is None: - chain = [node.project_name] + d['dependencies'] = [ + cls._get_requirements_for_package(c, key_tree, parent=node, + chain=chain+[c.project_name]) + for c in get_children(node) + if c.project_name not in chain + ] - d = node.as_dict() - if parent: - d['required_version'] = node.version_spec if node.version_spec else 'Any' - else: - d['required_version'] = d['installed_version'] + return d - d['dependencies'] = [ - aux(c, parent=node, chain=chain+[c.project_name]) - for c in get_children(node) - if c.project_name not in chain - ] + def get_package_requirements(self, pkg=None): + from .vendor.pipdeptree import flatten, sorted_tree, build_dist_index, construct_tree + packages = self.get_installed_packages() + if pkg: + packages = [p for p in packages if p.key == pkg] + dist_index = build_dist_index(packages) + tree = sorted_tree(construct_tree(dist_index)) + branch_keys = set(r.key for r in flatten(tree.values())) + if pkg is not None: + nodes = [p for p in tree.keys() if p.key == pkg] + else: + nodes = [p for p in tree.keys() if p.key not in branch_keys] + key_tree = dict((k.key, v) for k, v in tree.items()) - return d - return [aux(p) for p in nodes] + return [self._get_requirements_for_package(p, key_tree) for p in nodes] + + @classmethod + def reverse_dependency(cls, node): + new_node = { + "package_name": node["package_name"], + "installed_version": node["installed_version"], + "required_version": node["required_version"] + } + for dependency in node.get("dependencies", []): + for dep in cls.reverse_dependency(dependency): + new_dep = dep.copy() + new_dep["parent"] = (node["package_name"], node["installed_version"]) + yield new_dep + yield new_node + + def reverse_dependencies(self): + from vistir.misc import unnest, chunked + rdeps = {} + for req in self.get_package_requirements(): + for d in self.reverse_dependency(req): + parents = None + name = d["package_name"] + pkg = { + name: { + "installed": d["installed_version"], + "required": d["required_version"] + } + } + parents = tuple(d.get("parent", ())) + pkg[name]["parents"] = parents + if rdeps.get(name): + if not (rdeps[name].get("required") or rdeps[name].get("installed")): + rdeps[name].update(pkg[name]) + rdeps[name]["parents"] = rdeps[name].get("parents", ()) + parents + else: + rdeps[name] = pkg[name] + for k in list(rdeps.keys()): + entry = rdeps[k] + if entry.get("parents"): + rdeps[k]["parents"] = set([ + p for p, version in chunked(2, unnest(entry["parents"])) + ]) + return rdeps def get_working_set(self): """Retrieve the working set of installed packages for the environment. @@ -473,21 +784,34 @@ class Environment(object): vendor_dir = parent_path.joinpath("vendor").as_posix() patched_dir = parent_path.joinpath("patched").as_posix() parent_path = parent_path.as_posix() + self.add_dist("pip") prefix = self.prefix.as_posix() with vistir.contextmanagers.temp_environ(), vistir.contextmanagers.temp_path(): os.environ["PATH"] = os.pathsep.join([ - vistir.compat.fs_str(self.scripts_dir), + vistir.compat.fs_str(self.script_basedir), vistir.compat.fs_str(self.prefix.as_posix()), os.environ.get("PATH", "") ]) os.environ["PYTHONIOENCODING"] = vistir.compat.fs_str("utf-8") os.environ["PYTHONDONTWRITEBYTECODE"] = vistir.compat.fs_str("1") - os.environ["PYTHONPATH"] = self.base_paths["PYTHONPATH"] + from .environments import PIPENV_USE_SYSTEM if self.is_venv: + os.environ["PYTHONPATH"] = self.base_paths["PYTHONPATH"] os.environ["VIRTUAL_ENV"] = vistir.compat.fs_str(prefix) + else: + if not PIPENV_USE_SYSTEM and not os.environ.get("VIRTUAL_ENV"): + os.environ["PYTHONPATH"] = self.base_paths["PYTHONPATH"] + os.environ.pop("PYTHONHOME", None) sys.path = self.sys_path sys.prefix = self.sys_prefix site.addsitedir(self.base_paths["purelib"]) + pip = self.safe_import("pip") + pip_vendor = self.safe_import("pip._vendor") + pep517_dir = os.path.join(os.path.dirname(pip_vendor.__file__), "pep517") + site.addsitedir(pep517_dir) + os.environ["PYTHONPATH"] = os.pathsep.join([ + os.environ.get("PYTHONPATH", self.base_paths["PYTHONPATH"]), pep517_dir + ]) if include_extras: site.addsitedir(parent_path) sys.path.extend([parent_path, patched_dir, vendor_dir]) @@ -591,10 +915,7 @@ class Environment(object): monkey_patch.activate() pip_shims = self.safe_import("pip_shims") pathset_base = pip_shims.UninstallPathSet - import recursive_monkey_patch - recursive_monkey_patch.monkey_patch( - PatchedUninstaller, pathset_base - ) + pathset_base._permitted = PatchedUninstaller._permitted dist = next( iter(filter(lambda d: d.project_name == pkgname, self.get_working_set())), None diff --git a/pipenv/environments.py b/pipenv/environments.py index a4972bbb..e59ed63a 100644 --- a/pipenv/environments.py +++ b/pipenv/environments.py @@ -2,15 +2,28 @@ import os import sys + +from io import UnsupportedOperation + from appdirs import user_cache_dir -from .vendor.vistir.misc import fs_str + from ._compat import fix_utf8 +from .vendor.vistir.misc import _isatty, fs_str # HACK: avoid resolver.py uses the wrong byte code files. # I hope I can remove this one day. os.environ["PYTHONDONTWRITEBYTECODE"] = fs_str("1") + +def _is_env_truthy(name): + """An environment variable is truthy if it exists and isn't one of (0, false, no, off) + """ + if name not in os.environ: + return False + return os.environ.get(name).lower() not in ("0", "false", "no", "off") + + PIPENV_IS_CI = bool("CI" in os.environ or "TF_BUILD" in os.environ) # HACK: Prevent invalid shebangs with Homebrew-installed Python: @@ -68,13 +81,15 @@ Default is to detect emulators automatically. This should be set if your emulator, e.g. Cmder, cannot be detected correctly. """ -PIPENV_HIDE_EMOJIS = bool(os.environ.get("PIPENV_HIDE_EMOJIS")) +PIPENV_HIDE_EMOJIS = ( + os.environ.get("PIPENV_HIDE_EMOJIS") is None + and (os.name == "nt" or PIPENV_IS_CI) + or _is_env_truthy("PIPENV_HIDE_EMOJIS") +) """Disable emojis in output. Default is to show emojis. This is automatically set on Windows. """ -if os.name == "nt" or PIPENV_IS_CI: - PIPENV_HIDE_EMOJIS = True PIPENV_IGNORE_VIRTUALENVS = bool(os.environ.get("PIPENV_IGNORE_VIRTUALENVS")) """If set, Pipenv will always assign a virtual environment for this project. @@ -84,7 +99,7 @@ environment, and reuses it if possible. This is usually the desired behavior, and enables the user to use any user-built environments with Pipenv. """ -PIPENV_INSTALL_TIMEOUT = 60 * 15 +PIPENV_INSTALL_TIMEOUT = int(os.environ.get("PIPENV_INSTALL_TIMEOUT", 60 * 15)) """Max number of seconds to wait for package installation. Defaults to 900 (15 minutes), a very long arbitrary time. @@ -111,7 +126,7 @@ PIPENV_MAX_ROUNDS = int(os.environ.get("PIPENV_MAX_ROUNDS", "16")) Default is 16, an arbitrary number that works most of the time. """ -PIPENV_MAX_SUBPROCESS = int(os.environ.get("PIPENV_MAX_SUBPROCESS", "16")) +PIPENV_MAX_SUBPROCESS = int(os.environ.get("PIPENV_MAX_SUBPROCESS", "8")) """How many subprocesses should Pipenv use when installing. Default is 16, an arbitrary number that seems to work. @@ -135,14 +150,12 @@ environments. if PIPENV_IS_CI: PIPENV_NOSPIN = True -PIPENV_SPINNER = "dots" +PIPENV_SPINNER = "dots" if not os.name == "nt" else "bouncingBar" """Sets the default spinner type. Spinners are identitcal to the node.js spinners and can be found at https://github.com/sindresorhus/cli-spinners """ -if os.name == "nt": - PIPENV_SPINNER = "bouncingBar" PIPENV_PIPFILE = os.environ.get("PIPENV_PIPFILE") """If set, this specifies a custom Pipfile location. @@ -221,6 +234,20 @@ Default is to lock dependencies and update ``Pipfile.lock`` on each run. NOTE: This only affects the ``install`` and ``uninstall`` commands. """ +PIP_EXISTS_ACTION = os.environ.get("PIP_EXISTS_ACTION", "w") +"""Specifies the value for pip's --exists-action option + +Defaullts to (w)ipe +""" + +PIPENV_RESOLVE_VCS = _is_env_truthy(os.environ.get("PIPENV_RESOLVE_VCS", 'true')) +"""Tells Pipenv whether to resolve all VCS dependencies in full. + +As of Pipenv 2018.11.26, only editable VCS dependencies were resolved in full. +To retain this behavior and avoid handling any conflicts that arise from the new +approach, you may set this to '0', 'off', or 'false'. +""" + PIPENV_PYUP_API_KEY = os.environ.get( "PIPENV_PYUP_API_KEY", "1ab8d58f-5122e025-83674263-bc1e79e0" ) @@ -250,7 +277,10 @@ PIPENV_SHELL = ( ) # Internal, to tell whether the command line session is interactive. -SESSION_IS_INTERACTIVE = bool(os.isatty(sys.stdout.fileno())) +try: + SESSION_IS_INTERACTIVE = _isatty(sys.stdout.fileno()) +except UnsupportedOperation: + SESSION_IS_INTERACTIVE = _isatty(sys.stdout) # Internal, consolidated verbosity representation as an integer. The default @@ -278,13 +308,35 @@ def is_quiet(threshold=-1): def is_in_virtualenv(): - pipenv_active = os.environ.get("PIPENV_ACTIVE") - virtual_env = os.environ.get("VIRTUAL_ENV") - return (PIPENV_USE_SYSTEM or virtual_env) and not ( - pipenv_active or PIPENV_IGNORE_VIRTUALENVS - ) + """ + Check virtualenv membership dynamically + + :return: True or false depending on whether we are in a regular virtualenv or not + :rtype: bool + """ + + pipenv_active = os.environ.get("PIPENV_ACTIVE", False) + virtual_env = None + use_system = False + ignore_virtualenvs = bool(os.environ.get("PIPENV_IGNORE_VIRTUALENVS", False)) + + if not pipenv_active and not ignore_virtualenvs: + virtual_env = os.environ.get("VIRTUAL_ENV") + use_system = bool(virtual_env) + return (use_system or virtual_env) and not (pipenv_active or ignore_virtualenvs) PIPENV_SPINNER_FAIL_TEXT = fix_utf8(u"✘ {0}") if not PIPENV_HIDE_EMOJIS else ("{0}") PIPENV_SPINNER_OK_TEXT = fix_utf8(u"✔ {0}") if not PIPENV_HIDE_EMOJIS else ("{0}") + + +def is_type_checking(): + try: + from typing import TYPE_CHECKING + except ImportError: + return False + return TYPE_CHECKING + + +MYPY_RUNNING = is_type_checking() diff --git a/pipenv/exceptions.py b/pipenv/exceptions.py index 62e25d53..c8ca0dc8 100644 --- a/pipenv/exceptions.py +++ b/pipenv/exceptions.py @@ -1,48 +1,67 @@ # -*- coding=utf-8 -*- import itertools +import re import sys -from traceback import format_exception -from pprint import pformat +from collections import namedtuple +from traceback import format_tb import six -from ._compat import fix_utf8 -from .patched import crayons from . import environments -from .vendor.click.utils import echo as click_echo -from .vendor.click._compat import get_text_stderr +from ._compat import decode_for_output +from .patched import crayons from .vendor.click.exceptions import ( - Abort, - BadOptionUsage, - BadParameter, - ClickException, - Exit, - FileError, - MissingParameter, - UsageError, + ClickException, FileError, UsageError ) -from .vendor.click.types import Path +from .vendor.vistir.misc import echo as click_echo +import vistir + +ANSI_REMOVAL_RE = re.compile(r"\033\[((?:\d|;)*)([a-zA-Z])", re.MULTILINE) +STRING_TYPES = (six.string_types, crayons.ColoredString) + +if sys.version_info[:2] >= (3, 7): + KnownException = namedtuple( + 'KnownException', ['exception_name', 'match_string', 'show_from_string', 'prefix'], + defaults=[None, None, None, ""] + ) +else: + KnownException = namedtuple( + 'KnownException', ['exception_name', 'match_string', 'show_from_string', 'prefix'], + ) + KnownException.__new__.__defaults__ = (None, None, None, "") + +KNOWN_EXCEPTIONS = [ + KnownException("PermissionError", prefix="Permission Denied:"), + KnownException( + "VirtualenvCreationException", + match_string="do_create_virtualenv", + show_from_string=None + ) +] def handle_exception(exc_type, exception, traceback, hook=sys.excepthook): if environments.is_verbose() or not issubclass(exc_type, ClickException): hook(exc_type, exception, traceback) else: - exc = format_exception(exc_type, exception, traceback) - lines = itertools.chain.from_iterable([l.splitlines() for l in exc]) - lines = list(lines)[-11:-1] + tb = format_tb(traceback, limit=-6) + lines = itertools.chain.from_iterable([frame.splitlines() for frame in tb]) + formatted_lines = [] for line in lines: line = line.strip("'").strip('"').strip("\n").strip() if not line.startswith("File"): line = " {0}".format(line) else: line = " {0}".format(line) - line = "[pipenv.exceptions.{0!s}]: {1}".format( + line = "[{0!s}]: {1}".format( exception.__class__.__name__, line ) - click_echo(fix_utf8(line), err=True) + formatted_lines.append(line) + # use new exception prettification rules to format exceptions according to + # UX rules + click_echo(decode_for_output(prettify_exc("\n".join(formatted_lines))), err=True) exception.show() @@ -62,16 +81,65 @@ class PipenvException(ClickException): def show(self, file=None): if file is None: - file = get_text_stderr() + file = vistir.misc.get_text_stderr() if self.extra: - if isinstance(self.extra, six.string_types): + if isinstance(self.extra, STRING_TYPES): self.extra = [self.extra,] for extra in self.extra: extra = "[pipenv.exceptions.{0!s}]: {1}".format( self.__class__.__name__, extra ) + extra = decode_for_output(extra, file) click_echo(extra, file=file) - click_echo(fix_utf8("{0}".format(self.message)), file=file) + click_echo(decode_for_output("{0}".format(self.message), file), file=file) + + +class PipenvCmdError(PipenvException): + def __init__(self, cmd, out="", err="", exit_code=1): + self.cmd = cmd + self.out = out + self.err = err + self.exit_code = exit_code + message = "Error running command: {0}".format(cmd) + PipenvException.__init__(self, message) + + def show(self, file=None): + if file is None: + file = vistir.misc.get_text_stderr() + click_echo("{0} {1}".format( + crayons.red("Error running command: "), + crayons.white(decode_for_output("$ {0}".format(self.cmd), file), bold=True) + ), err=True) + if self.out: + click_echo("{0} {1}".format( + crayons.white("OUTPUT: "), + decode_for_output(self.out, file) + ), err=True) + if self.err: + click_echo("{0} {1}".format( + crayons.white("STDERR: "), + decode_for_output(self.err, file) + ), err=True) + + +class JSONParseError(PipenvException): + def __init__(self, contents="", error_text=""): + self.error_text = error_text + PipenvException.__init__(self, contents) + + def show(self, file=None): + if file is None: + file = vistir.misc.get_text_stderr() + message = "{0}\n{1}".format( + crayons.white("Failed parsing JSON results:", bold=True), + decode_for_output(self.message.strip(), file) + ) + click_echo(message, err=True) + if self.error_text: + click_echo("{0} {1}".format( + crayons.white("ERROR TEXT:", bold=True), + decode_for_output(self.error_text, file) + ), err=True) class PipenvUsageError(UsageError): @@ -84,22 +152,22 @@ class PipenvUsageError(UsageError): message = formatted_message.format(msg_prefix, crayons.white(message, bold=True)) self.message = message extra = kwargs.pop("extra", []) - UsageError.__init__(self, fix_utf8(message), ctx) + UsageError.__init__(self, decode_for_output(message), ctx) self.extra = extra def show(self, file=None): if file is None: - file = get_text_stderr() + file = vistir.misc.get_text_stderr() color = None if self.ctx is not None: color = self.ctx.color if self.extra: - if isinstance(self.extra, six.string_types): + if isinstance(self.extra, STRING_TYPES): self.extra = [self.extra,] for extra in self.extra: if color: extra = getattr(crayons, color, "blue")(extra) - click_echo(fix_utf8(extra), file=file) + click_echo(decode_for_output(extra, file), file=file) hint = '' if (self.cmd is not None and self.cmd.get_help_option(self.ctx) is not None): @@ -123,30 +191,33 @@ class PipenvFileError(FileError): crayons.white("{0} not found!".format(filename), bold=True), message ) - FileError.__init__(self, filename=filename, hint=fix_utf8(message), **kwargs) + FileError.__init__(self, filename=filename, hint=decode_for_output(message), **kwargs) self.extra = extra def show(self, file=None): if file is None: - file = get_text_stderr() + file = vistir.misc.get_text_stderr() if self.extra: - if isinstance(self.extra, six.string_types): + if isinstance(self.extra, STRING_TYPES): self.extra = [self.extra,] for extra in self.extra: - click_echo(fix_utf8(extra), file=file) + click_echo(decode_for_output(extra, file), file=file) click_echo(self.message, file=file) class PipfileNotFound(PipenvFileError): def __init__(self, filename="Pipfile", extra=None, **kwargs): extra = kwargs.pop("extra", []) - message = ("{0} {1}".format( + message = ( + "{0} {1}".format( crayons.red("Aborting!", bold=True), - crayons.white("Please ensure that the file exists and is located in your" - " project root directory.", bold=True) + crayons.white( + "Please ensure that the file exists and is located in your" + " project root directory.", bold=True + ) ) ) - super(PipfileNotFound, self).__init__(filename, message=fix_utf8(message), extra=extra, **kwargs) + super(PipfileNotFound, self).__init__(filename, message=message, extra=extra, **kwargs) class LockfileNotFound(PipenvFileError): @@ -157,7 +228,7 @@ class LockfileNotFound(PipenvFileError): crayons.red("$ pipenv lock", bold=True), crayons.white("before you can continue.", bold=True) ) - super(LockfileNotFound, self).__init__(filename, message=fix_utf8(message), extra=extra, **kwargs) + super(LockfileNotFound, self).__init__(filename, message=message, extra=extra, **kwargs) class DeployException(PipenvUsageError): @@ -165,13 +236,13 @@ class DeployException(PipenvUsageError): if not message: message = crayons.normal("Aborting deploy", bold=True) extra = kwargs.pop("extra", []) - PipenvUsageError.__init__(message=fix_utf8(message), extra=extra, **kwargs) + PipenvUsageError.__init__(self, message=message, extra=extra, **kwargs) class PipenvOptionsError(PipenvUsageError): def __init__(self, option_name, message=None, ctx=None, **kwargs): extra = kwargs.pop("extra", []) - PipenvUsageError.__init__(self, message=fix_utf8(message), ctx=ctx, **kwargs) + PipenvUsageError.__init__(self, message=message, ctx=ctx, **kwargs) self.extra = extra self.option_name = option_name @@ -185,7 +256,8 @@ class SystemUsageError(PipenvOptionsError): crayons.red("Warning", bold=True) ), ] - message = crayons.blue("See also: {0}".format(crayons.white("-deploy flag."))) + if message is None: + message = crayons.blue("See also: {0}".format(crayons.white("--deploy flag."))) super(SystemUsageError, self).__init__(option_name, message=message, ctx=ctx, extra=extra, **kwargs) @@ -197,7 +269,7 @@ class PipfileException(PipenvFileError): hint = "{0} {1}".format(crayons.red("ERROR (PACKAGE NOT INSTALLED):"), hint) filename = project.pipfile_location extra = kwargs.pop("extra", []) - PipenvFileError.__init__(self, filename, fix_utf8(hint), extra=extra, **kwargs) + PipenvFileError.__init__(self, filename, hint, extra=extra, **kwargs) class SetupException(PipenvException): @@ -213,7 +285,7 @@ class VirtualenvException(PipenvException): "There was an unexpected error while activating your virtualenv. " "Continuing anyway..." ) - PipenvException.__init__(self, fix_utf8(message), **kwargs) + PipenvException.__init__(self, message, **kwargs) class VirtualenvActivationException(VirtualenvException): @@ -224,7 +296,7 @@ class VirtualenvActivationException(VirtualenvException): "not activated. Continuing anyway…" ) self.message = message - VirtualenvException.__init__(self, fix_utf8(message), **kwargs) + VirtualenvException.__init__(self, message, **kwargs) class VirtualenvCreationException(VirtualenvException): @@ -232,45 +304,70 @@ class VirtualenvCreationException(VirtualenvException): if not message: message = "Failed to create virtual environment." self.message = message - VirtualenvException.__init__(self, fix_utf8(message), **kwargs) + extra = kwargs.pop("extra", None) + if extra is not None and isinstance(extra, STRING_TYPES): + # note we need the format interpolation because ``crayons.ColoredString`` + # is not an actual string type but is only a preparation for interpolation + # so replacement or parsing requires this step + extra = ANSI_REMOVAL_RE.sub("", "{0}".format(extra)) + if "KeyboardInterrupt" in extra: + extra = crayons.red("Virtualenv creation interrupted by user", bold=True) + self.extra = extra = [extra,] + VirtualenvException.__init__(self, message, extra=extra) class UninstallError(PipenvException): def __init__(self, package, command, return_values, return_code, **kwargs): - extra = [crayons.blue("Attempted to run command: {0}".format( - crayons.yellow("$ {0}".format(command), bold=True) - )),] + extra = [ + "{0} {1}".format( + crayons.blue("Attempted to run command: "), + crayons.yellow("$ {0!r}".format(command), bold=True + ) + )] extra.extend([crayons.blue(line.strip()) for line in return_values.splitlines()]) - if isinstance(package, (tuple, list)): + if isinstance(package, (tuple, list, set)): package = " ".join(package) - message = "{0} {1}...".format( + message = "{0!s} {1!s}...".format( crayons.normal("Failed to uninstall package(s)"), - crayons.yellow(package, bold=True) + crayons.yellow("{0}!s".format(package), bold=True) ) self.exit_code = return_code - PipenvException.__init__(self, message=fix_utf8(message), extra=extra) + PipenvException.__init__(self, message=message, extra=extra) self.extra = extra class InstallError(PipenvException): def __init__(self, package, **kwargs): + package_message = "" + if package is not None: + package_message = "Couldn't install package: {0}\n".format( + crayons.white("{0!s}".format(package), bold=True) + ) message = "{0} {1}".format( - crayons.red("ERROR:", bold=True), + "{0}".format(package_message), crayons.yellow("Package installation failed...") ) extra = kwargs.pop("extra", []) - PipenvException.__init__(self, message=fix_utf8(message), extra=extra, **kwargs) + PipenvException.__init__(self, message=message, extra=extra, **kwargs) class CacheError(PipenvException): def __init__(self, path, **kwargs): - message = "{0} {1} {2}\n{0}".format( - crayons.red("ERROR:", bold=True), + message = "{0} {1}\n{2}".format( crayons.blue("Corrupt cache file"), - crayons.white(path), + crayons.white("{0!s}".format(path)), crayons.white('Consider trying "pipenv lock --clear" to clear the cache.') ) - super(PipenvException, self).__init__(message=fix_utf8(message)) + PipenvException.__init__(self, message=message) + + +class DependencyConflict(PipenvException): + def __init__(self, message): + extra = ["{0} {1}".format( + crayons.red("The operation failed...", bold=True), + crayons.red("A dependency conflict was detected and could not be resolved."), + )] + PipenvException.__init__(self, message, extra=extra) class ResolutionFailure(PipenvException): @@ -292,9 +389,7 @@ class ResolutionFailure(PipenvException): ) if "no version found at all" in message: no_version_found = True - message = "{0} {1}".format( - crayons.red("ERROR:", bold=True), crayons.yellow(message) - ) + message = crayons.yellow("{0}".format(message)) if no_version_found: message = "{0}\n{1}".format( message, @@ -303,4 +398,67 @@ class ResolutionFailure(PipenvException): "See PEP440 for more information." ) ) - super(ResolutionFailure, self).__init__(fix_utf8(message), extra=extra) + PipenvException.__init__(self, message, extra=extra) + + +class RequirementError(PipenvException): + + def __init__(self, req=None): + from .utils import VCS_LIST + keys = ("name", "path",) + VCS_LIST + ("line", "uri", "url", "relpath") + if req is not None: + possible_display_values = [getattr(req, value, None) for value in keys] + req_value = next(iter( + val for val in possible_display_values if val is not None + ), None) + if not req_value: + getstate_fn = getattr(req, "__getstate__", None) + slots = getattr(req, "__slots__", None) + keys_fn = getattr(req, "keys", None) + if getstate_fn: + req_value = getstate_fn() + elif slots: + slot_vals = [ + (k, getattr(req, k, None)) for k in slots + if getattr(req, k, None) + ] + req_value = "\n".join([ + " {0}: {1}".format(k, v) for k, v in slot_vals + ]) + elif keys_fn: + values = [(k, req.get(k)) for k in keys_fn() if req.get(k)] + req_value = "\n".join([ + " {0}: {1}".format(k, v) for k, v in values + ]) + else: + req_value = getattr(req.line_instance, "line", None) + message = "{0} {1}".format( + crayons.normal(decode_for_output("Failed creating requirement instance")), + crayons.white(decode_for_output("{0!r}".format(req_value))) + ) + extra = [str(req)] + PipenvException.__init__(self, message, extra=extra) + + +def prettify_exc(error): + """Catch known errors and prettify them instead of showing the + entire traceback, for better UX""" + errors = [] + for exc in KNOWN_EXCEPTIONS: + search_string = exc.match_string if exc.match_string else exc.exception_name + split_string = exc.show_from_string if exc.show_from_string else exc.exception_name + if search_string in error: + # for known exceptions with no display rules and no prefix + # we should simply show nothing + if not exc.show_from_string and not exc.prefix: + errors.append("") + continue + elif exc.prefix and exc.prefix in error: + _, error, info = error.rpartition(exc.prefix) + else: + _, error, info = error.rpartition(split_string) + errors.append("{0} {1}".format(error, info)) + if not errors: + return "{}".format(vistir.misc.decode_for_output(error)) + + return "\n".join(errors) diff --git a/pipenv/patched/notpip/COPYING b/pipenv/patched/notpip/COPYING new file mode 100644 index 00000000..f067af3a --- /dev/null +++ b/pipenv/patched/notpip/COPYING @@ -0,0 +1,14 @@ +Copyright (C) 2008-2011 INADA Naoki + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/pipenv/patched/notpip/LICENSE.BSD b/pipenv/patched/notpip/LICENSE.BSD new file mode 100644 index 00000000..42ce7b75 --- /dev/null +++ b/pipenv/patched/notpip/LICENSE.BSD @@ -0,0 +1,23 @@ +Copyright (c) Donald Stufft and individual contributors. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/pipenv/patched/notpip/__init__.py b/pipenv/patched/notpip/__init__.py index ae265fa7..f48c1ca6 100644 --- a/pipenv/patched/notpip/__init__.py +++ b/pipenv/patched/notpip/__init__.py @@ -1 +1 @@ -__version__ = "18.1" +__version__ = "19.0.3" diff --git a/pipenv/patched/notpip/_internal/build_env.py b/pipenv/patched/notpip/_internal/build_env.py index 6d696fbd..d38adc49 100644 --- a/pipenv/patched/notpip/_internal/build_env.py +++ b/pipenv/patched/notpip/_internal/build_env.py @@ -4,98 +4,173 @@ import logging import os import sys +import textwrap +from collections import OrderedDict from distutils.sysconfig import get_python_lib from sysconfig import get_paths from pipenv.patched.notpip._vendor.pkg_resources import Requirement, VersionConflict, WorkingSet +from pipenv.patched.notpip import __file__ as pip_location from pipenv.patched.notpip._internal.utils.misc import call_subprocess from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING from pipenv.patched.notpip._internal.utils.ui import open_spinner +if MYPY_CHECK_RUNNING: + from typing import Tuple, Set, Iterable, Optional, List # noqa: F401 + from pipenv.patched.notpip._internal.index import PackageFinder # noqa: F401 + logger = logging.getLogger(__name__) +class _Prefix: + + def __init__(self, path): + # type: (str) -> None + self.path = path + self.setup = False + self.bin_dir = get_paths( + 'nt' if os.name == 'nt' else 'posix_prefix', + vars={'base': path, 'platbase': path} + )['scripts'] + # Note: prefer distutils' sysconfig to get the + # library paths so PyPy is correctly supported. + purelib = get_python_lib(plat_specific=False, prefix=path) + platlib = get_python_lib(plat_specific=True, prefix=path) + if purelib == platlib: + self.lib_dirs = [purelib] + else: + self.lib_dirs = [purelib, platlib] + + class BuildEnvironment(object): """Creates and manages an isolated environment to install build deps """ def __init__(self): + # type: () -> None self._temp_dir = TempDirectory(kind="build-env") self._temp_dir.create() - @property - def path(self): - return self._temp_dir.path + self._prefixes = OrderedDict(( + (name, _Prefix(os.path.join(self._temp_dir.path, name))) + for name in ('normal', 'overlay') + )) + + self._bin_dirs = [] # type: List[str] + self._lib_dirs = [] # type: List[str] + for prefix in reversed(list(self._prefixes.values())): + self._bin_dirs.append(prefix.bin_dir) + self._lib_dirs.extend(prefix.lib_dirs) + + # Customize site to: + # - ensure .pth files are honored + # - prevent access to system site packages + system_sites = { + os.path.normcase(site) for site in ( + get_python_lib(plat_specific=False), + get_python_lib(plat_specific=True), + ) + } + self._site_dir = os.path.join(self._temp_dir.path, 'site') + if not os.path.exists(self._site_dir): + os.mkdir(self._site_dir) + with open(os.path.join(self._site_dir, 'sitecustomize.py'), 'w') as fp: + fp.write(textwrap.dedent( + ''' + import os, site, sys + + # First, drop system-sites related paths. + original_sys_path = sys.path[:] + known_paths = set() + for path in {system_sites!r}: + site.addsitedir(path, known_paths=known_paths) + system_paths = set( + os.path.normcase(path) + for path in sys.path[len(original_sys_path):] + ) + original_sys_path = [ + path for path in original_sys_path + if os.path.normcase(path) not in system_paths + ] + sys.path = original_sys_path + + # Second, add lib directories. + # ensuring .pth file are processed. + for path in {lib_dirs!r}: + assert not path in sys.path + site.addsitedir(path) + ''' + ).format(system_sites=system_sites, lib_dirs=self._lib_dirs)) def __enter__(self): - self.save_path = os.environ.get('PATH', None) - self.save_pythonpath = os.environ.get('PYTHONPATH', None) - self.save_nousersite = os.environ.get('PYTHONNOUSERSITE', None) + self._save_env = { + name: os.environ.get(name, None) + for name in ('PATH', 'PYTHONNOUSERSITE', 'PYTHONPATH') + } - install_scheme = 'nt' if (os.name == 'nt') else 'posix_prefix' - install_dirs = get_paths(install_scheme, vars={ - 'base': self.path, - 'platbase': self.path, + path = self._bin_dirs[:] + old_path = self._save_env['PATH'] + if old_path: + path.extend(old_path.split(os.pathsep)) + + pythonpath = [self._site_dir] + + os.environ.update({ + 'PATH': os.pathsep.join(path), + 'PYTHONNOUSERSITE': '1', + 'PYTHONPATH': os.pathsep.join(pythonpath), }) - scripts = install_dirs['scripts'] - if self.save_path: - os.environ['PATH'] = scripts + os.pathsep + self.save_path - else: - os.environ['PATH'] = scripts + os.pathsep + os.defpath - - # Note: prefer distutils' sysconfig to get the - # library paths so PyPy is correctly supported. - purelib = get_python_lib(plat_specific=0, prefix=self.path) - platlib = get_python_lib(plat_specific=1, prefix=self.path) - if purelib == platlib: - lib_dirs = purelib - else: - lib_dirs = purelib + os.pathsep + platlib - if self.save_pythonpath: - os.environ['PYTHONPATH'] = lib_dirs + os.pathsep + \ - self.save_pythonpath - else: - os.environ['PYTHONPATH'] = lib_dirs - - os.environ['PYTHONNOUSERSITE'] = '1' - - return self.path - def __exit__(self, exc_type, exc_val, exc_tb): - def restore_var(varname, old_value): + for varname, old_value in self._save_env.items(): if old_value is None: os.environ.pop(varname, None) else: os.environ[varname] = old_value - restore_var('PATH', self.save_path) - restore_var('PYTHONPATH', self.save_pythonpath) - restore_var('PYTHONNOUSERSITE', self.save_nousersite) - def cleanup(self): + # type: () -> None self._temp_dir.cleanup() - def missing_requirements(self, reqs): - """Return a list of the requirements from reqs that are not present + def check_requirements(self, reqs): + # type: (Iterable[str]) -> Tuple[Set[Tuple[str, str]], Set[str]] + """Return 2 sets: + - conflicting requirements: set of (installed, wanted) reqs tuples + - missing requirements: set of reqs """ - missing = [] - with self: - ws = WorkingSet(os.environ["PYTHONPATH"].split(os.pathsep)) + missing = set() + conflicting = set() + if reqs: + ws = WorkingSet(self._lib_dirs) for req in reqs: try: if ws.find(Requirement.parse(req)) is None: - missing.append(req) - except VersionConflict: - missing.append(req) - return missing + missing.add(req) + except VersionConflict as e: + conflicting.add((str(e.args[0].as_requirement()), + str(e.args[1]))) + return conflicting, missing - def install_requirements(self, finder, requirements, message): + def install_requirements( + self, + finder, # type: PackageFinder + requirements, # type: Iterable[str] + prefix_as_string, # type: str + message # type: Optional[str] + ): + # type: (...) -> None + prefix = self._prefixes[prefix_as_string] + assert not prefix.setup + prefix.setup = True + if not requirements: + return args = [ - sys.executable, '-m', 'pip', 'install', '--ignore-installed', - '--no-user', '--prefix', self.path, '--no-warn-script-location', - ] + sys.executable, os.path.dirname(pip_location), 'install', + '--ignore-installed', '--no-user', '--prefix', prefix.path, + '--no-warn-script-location', + ] # type: List[str] if logger.getEffectiveLevel() <= logging.DEBUG: args.append('-v') for format_control in ('no_binary', 'only_binary'): @@ -114,8 +189,6 @@ class BuildEnvironment(object): args.extend(['--trusted-host', host]) if finder.allow_all_prereleases: args.append('--pre') - if finder.process_dependency_links: - args.append('--process-dependency-links') args.append('--') args.extend(requirements) with open_spinner(message) as spinner: @@ -138,5 +211,5 @@ class NoOpBuildEnvironment(BuildEnvironment): def cleanup(self): pass - def install_requirements(self, finder, requirements, message): + def install_requirements(self, finder, requirements, prefix, message): raise NotImplementedError() diff --git a/pipenv/patched/notpip/_internal/cache.py b/pipenv/patched/notpip/_internal/cache.py index d91b8170..9f35e83d 100644 --- a/pipenv/patched/notpip/_internal/cache.py +++ b/pipenv/patched/notpip/_internal/cache.py @@ -12,8 +12,13 @@ from pipenv.patched.notpip._internal.download import path_to_url from pipenv.patched.notpip._internal.models.link import Link from pipenv.patched.notpip._internal.utils.compat import expanduser from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING from pipenv.patched.notpip._internal.wheel import InvalidWheelFilename, Wheel +if MYPY_CHECK_RUNNING: + from typing import Optional, Set, List, Any # noqa: F401 + from pipenv.patched.notpip._internal.index import FormatControl # noqa: F401 + logger = logging.getLogger(__name__) @@ -29,6 +34,7 @@ class Cache(object): """ def __init__(self, cache_dir, format_control, allowed_formats): + # type: (str, FormatControl, Set[str]) -> None super(Cache, self).__init__() self.cache_dir = expanduser(cache_dir) if cache_dir else None self.format_control = format_control @@ -38,6 +44,7 @@ class Cache(object): assert self.allowed_formats.union(_valid_formats) == _valid_formats def _get_cache_path_parts(self, link): + # type: (Link) -> List[str] """Get parts of part that must be os.path.joined with cache_dir """ @@ -63,6 +70,7 @@ class Cache(object): return parts def _get_candidates(self, link, package_name): + # type: (Link, Optional[str]) -> List[Any] can_not_cache = ( not self.cache_dir or not package_name or @@ -87,23 +95,27 @@ class Cache(object): raise def get_path_for_link(self, link): + # type: (Link) -> str """Return a directory to store cached items in for link. """ raise NotImplementedError() def get(self, link, package_name): + # type: (Link, Optional[str]) -> Link """Returns a link to a cached item if it exists, otherwise returns the passed link. """ raise NotImplementedError() def _link_for_candidate(self, link, candidate): + # type: (Link, str) -> Link root = self.get_path_for_link(link) path = os.path.join(root, candidate) return Link(path_to_url(path)) def cleanup(self): + # type: () -> None pass @@ -112,11 +124,13 @@ class SimpleWheelCache(Cache): """ def __init__(self, cache_dir, format_control): + # type: (str, FormatControl) -> None super(SimpleWheelCache, self).__init__( cache_dir, format_control, {"binary"} ) def get_path_for_link(self, link): + # type: (Link) -> str """Return a directory to store cached wheels for link Because there are M wheels for any one sdist, we provide a directory @@ -137,6 +151,7 @@ class SimpleWheelCache(Cache): return os.path.join(self.cache_dir, "wheels", *parts) def get(self, link, package_name): + # type: (Link, Optional[str]) -> Link candidates = [] for wheel_name in self._get_candidates(link, package_name): @@ -160,6 +175,7 @@ class EphemWheelCache(SimpleWheelCache): """ def __init__(self, format_control): + # type: (FormatControl) -> None self._temp_dir = TempDirectory(kind="ephem-wheel-cache") self._temp_dir.create() @@ -168,6 +184,7 @@ class EphemWheelCache(SimpleWheelCache): ) def cleanup(self): + # type: () -> None self._temp_dir.cleanup() @@ -179,6 +196,7 @@ class WheelCache(Cache): """ def __init__(self, cache_dir, format_control): + # type: (str, FormatControl) -> None super(WheelCache, self).__init__( cache_dir, format_control, {'binary'} ) @@ -186,17 +204,21 @@ class WheelCache(Cache): self._ephem_cache = EphemWheelCache(format_control) def get_path_for_link(self, link): + # type: (Link) -> str return self._wheel_cache.get_path_for_link(link) def get_ephem_path_for_link(self, link): + # type: (Link) -> str return self._ephem_cache.get_path_for_link(link) def get(self, link, package_name): + # type: (Link, Optional[str]) -> Link retval = self._wheel_cache.get(link, package_name) if retval is link: retval = self._ephem_cache.get(link, package_name) return retval def cleanup(self): + # type: () -> None self._wheel_cache.cleanup() self._ephem_cache.cleanup() diff --git a/pipenv/patched/notpip/_internal/cli/base_command.py b/pipenv/patched/notpip/_internal/cli/base_command.py index 229831f2..4aa16da6 100644 --- a/pipenv/patched/notpip/_internal/cli/base_command.py +++ b/pipenv/patched/notpip/_internal/cli/base_command.py @@ -1,11 +1,13 @@ """Base Command class, and related routines""" -from __future__ import absolute_import +from __future__ import absolute_import, print_function import logging import logging.config import optparse import os +import platform import sys +import traceback from pipenv.patched.notpip._internal.cli import cmdoptions from pipenv.patched.notpip._internal.cli.parser import ( @@ -26,13 +28,19 @@ from pipenv.patched.notpip._internal.req.constructors import ( install_req_from_editable, install_req_from_line, ) from pipenv.patched.notpip._internal.req.req_file import parse_requirements -from pipenv.patched.notpip._internal.utils.logging import setup_logging -from pipenv.patched.notpip._internal.utils.misc import get_prog, normalize_path +from pipenv.patched.notpip._internal.utils.deprecation import deprecated +from pipenv.patched.notpip._internal.utils.logging import BrokenStdoutLoggingError, setup_logging +from pipenv.patched.notpip._internal.utils.misc import ( + get_prog, normalize_path, redact_password_from_url, +) from pipenv.patched.notpip._internal.utils.outdated import pip_version_check from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Optional # noqa: F401 + from typing import Optional, List, Tuple, Any # noqa: F401 + from optparse import Values # noqa: F401 + from pipenv.patched.notpip._internal.cache import WheelCache # noqa: F401 + from pipenv.patched.notpip._internal.req.req_set import RequirementSet # noqa: F401 __all__ = ['Command'] @@ -46,6 +54,7 @@ class Command(object): ignore_require_venv = False # type: bool def __init__(self, isolated=False): + # type: (bool) -> None parser_kw = { 'usage': self.usage, 'prog': '%s %s' % (get_prog(), self.name), @@ -69,7 +78,12 @@ class Command(object): ) self.parser.add_option_group(gen_opts) + def run(self, options, args): + # type: (Values, List[Any]) -> Any + raise NotImplementedError + def _build_session(self, options, retries=None, timeout=None): + # type: (Values, Optional[int], Optional[int]) -> PipSession session = PipSession( cache=( normalize_path(os.path.join(options.cache_dir, "http")) @@ -106,21 +120,43 @@ class Command(object): return session def parse_args(self, args): + # type: (List[str]) -> Tuple # factored out for testability return self.parser.parse_args(args) def main(self, args): + # type: (List[str]) -> int options, args = self.parse_args(args) # Set verbosity so that it can be used elsewhere. self.verbosity = options.verbose - options.quiet - setup_logging( + level_number = setup_logging( verbosity=self.verbosity, no_color=options.no_color, user_log_file=options.log, ) + if sys.version_info[:2] == (3, 4): + deprecated( + "Python 3.4 support has been deprecated. pip 19.1 will be the " + "last one supporting it. Please upgrade your Python as Python " + "3.4 won't be maintained after March 2019 (cf PEP 429).", + replacement=None, + gone_in='19.2', + ) + elif sys.version_info[:2] == (2, 7): + message = ( + "A future version of pip will drop support for Python 2.7." + ) + if platform.python_implementation() == "CPython": + message = ( + "Python 2.7 will reach the end of its life on January " + "1st, 2020. Please upgrade your Python as Python 2.7 " + "won't be maintained after that date. " + ) + message + deprecated(message, replacement=None, gone_in=None) + # TODO: Try to get these passing down from the command? # without resorting to os.environ to hold these. # This also affects isolated builds and it should. @@ -159,6 +195,14 @@ class Command(object): logger.critical('ERROR: %s', exc) logger.debug('Exception information:', exc_info=True) + return ERROR + except BrokenStdoutLoggingError: + # Bypass our logger and write any remaining messages to stderr + # because stdout no longer works. + print('ERROR: Pipe to stdout was broken', file=sys.stderr) + if level_number <= logging.DEBUG: + traceback.print_exc(file=sys.stderr) + return ERROR except KeyboardInterrupt: logger.critical('Operation cancelled by user') @@ -195,8 +239,15 @@ class Command(object): class RequirementCommand(Command): @staticmethod - def populate_requirement_set(requirement_set, args, options, finder, - session, name, wheel_cache): + def populate_requirement_set(requirement_set, # type: RequirementSet + args, # type: List[str] + options, # type: Values + finder, # type: PackageFinder + session, # type: PipSession + name, # type: str + wheel_cache # type: Optional[WheelCache] + ): + # type: (...) -> None """ Marshal cmd line args into a requirement set. """ @@ -214,6 +265,7 @@ class RequirementCommand(Command): for req in args: req_to_add = install_req_from_line( req, None, isolated=options.isolated_mode, + use_pep517=options.use_pep517, wheel_cache=wheel_cache ) req_to_add.is_direct = True @@ -223,6 +275,7 @@ class RequirementCommand(Command): req_to_add = install_req_from_editable( req, isolated=options.isolated_mode, + use_pep517=options.use_pep517, wheel_cache=wheel_cache ) req_to_add.is_direct = True @@ -232,7 +285,8 @@ class RequirementCommand(Command): for req_to_add in parse_requirements( filename, finder=finder, options=options, session=session, - wheel_cache=wheel_cache): + wheel_cache=wheel_cache, + use_pep517=options.use_pep517): req_to_add.is_direct = True requirement_set.add_requirement(req_to_add) # If --require-hashes was a line in a requirements file, tell @@ -251,15 +305,25 @@ class RequirementCommand(Command): 'You must give at least one requirement to %(name)s ' '(see "pip help %(name)s")' % opts) - def _build_package_finder(self, options, session, - platform=None, python_versions=None, - abi=None, implementation=None): + def _build_package_finder( + self, + options, # type: Values + session, # type: PipSession + platform=None, # type: Optional[str] + python_versions=None, # type: Optional[List[str]] + abi=None, # type: Optional[str] + implementation=None # type: Optional[str] + ): + # type: (...) -> PackageFinder """ Create a package finder appropriate to this requirement command. """ index_urls = [options.index_url] + options.extra_index_urls if options.no_index: - logger.debug('Ignoring indexes: %s', ','.join(index_urls)) + logger.debug( + 'Ignoring indexes: %s', + ','.join(redact_password_from_url(url) for url in index_urls), + ) index_urls = [] return PackageFinder( @@ -268,7 +332,6 @@ class RequirementCommand(Command): index_urls=index_urls, trusted_hosts=options.trusted_hosts, allow_all_prereleases=options.pre, - process_dependency_links=options.process_dependency_links, session=session, platform=platform, versions=python_versions, diff --git a/pipenv/patched/notpip/_internal/cli/cmdoptions.py b/pipenv/patched/notpip/_internal/cli/cmdoptions.py index a075a67e..3c5a7084 100644 --- a/pipenv/patched/notpip/_internal/cli/cmdoptions.py +++ b/pipenv/patched/notpip/_internal/cli/cmdoptions.py @@ -9,7 +9,9 @@ pass on state. To be consistent, all options will follow this design. """ from __future__ import absolute_import +import textwrap import warnings +from distutils.util import strtobool from functools import partial from optparse import SUPPRESS_HELP, Option, OptionGroup @@ -22,10 +24,27 @@ from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING from pipenv.patched.notpip._internal.utils.ui import BAR_TYPES if MYPY_CHECK_RUNNING: - from typing import Any # noqa: F401 + from typing import Any, Callable, Dict, List, Optional, Union # noqa: F401 + from optparse import OptionParser, Values # noqa: F401 + from pipenv.patched.notpip._internal.cli.parser import ConfigOptionParser # noqa: F401 + + +def raise_option_error(parser, option, msg): + """ + Raise an option parsing error using parser.error(). + + Args: + parser: an OptionParser instance. + option: an Option instance. + msg: the error text. + """ + msg = '{} error: {}'.format(option, msg) + msg = textwrap.fill(' '.join(msg.split())) + parser.error(msg) def make_option_group(group, parser): + # type: (Dict[str, Any], ConfigOptionParser) -> OptionGroup """ Return an OptionGroup object group -- assumed to be dict with 'name' and 'options' keys @@ -38,6 +57,7 @@ def make_option_group(group, parser): def check_install_build_global(options, check_options=None): + # type: (Values, Optional[Values]) -> None """Disable wheels if per-setup.py call options are set. :param options: The OptionParser options to update. @@ -60,6 +80,7 @@ def check_install_build_global(options, check_options=None): def check_dist_restriction(options, check_target=False): + # type: (Values, bool) -> None """Function for determining if custom platform options are allowed. :param options: The OptionParser options. @@ -108,7 +129,7 @@ help_ = partial( dest='help', action='help', help='Show help.', -) # type: Any +) # type: Callable[..., Option] isolated_mode = partial( Option, @@ -120,7 +141,7 @@ isolated_mode = partial( "Run pip in an isolated mode, ignoring environment variables and user " "configuration." ), -) +) # type: Callable[..., Option] require_virtualenv = partial( Option, @@ -130,7 +151,7 @@ require_virtualenv = partial( action='store_true', default=False, help=SUPPRESS_HELP -) # type: Any +) # type: Callable[..., Option] verbose = partial( Option, @@ -139,7 +160,7 @@ verbose = partial( action='count', default=0, help='Give more output. Option is additive, and can be used up to 3 times.' -) +) # type: Callable[..., Option] no_color = partial( Option, @@ -148,7 +169,7 @@ no_color = partial( action='store_true', default=False, help="Suppress colored output", -) +) # type: Callable[..., Option] version = partial( Option, @@ -156,7 +177,7 @@ version = partial( dest='version', action='store_true', help='Show version and exit.', -) # type: Any +) # type: Callable[..., Option] quiet = partial( Option, @@ -169,7 +190,7 @@ quiet = partial( ' times (corresponding to WARNING, ERROR, and CRITICAL logging' ' levels).' ), -) # type: Any +) # type: Callable[..., Option] progress_bar = partial( Option, @@ -182,7 +203,7 @@ progress_bar = partial( 'Specify type of progress to be displayed [' + '|'.join(BAR_TYPES.keys()) + '] (default: %default)' ), -) # type: Any +) # type: Callable[..., Option] log = partial( Option, @@ -190,7 +211,7 @@ log = partial( dest="log", metavar="path", help="Path to a verbose appending log." -) # type: Any +) # type: Callable[..., Option] no_input = partial( Option, @@ -200,7 +221,7 @@ no_input = partial( action='store_true', default=False, help=SUPPRESS_HELP -) # type: Any +) # type: Callable[..., Option] proxy = partial( Option, @@ -209,7 +230,7 @@ proxy = partial( type='str', default='', help="Specify a proxy in the form [user:passwd@]proxy.server:port." -) # type: Any +) # type: Callable[..., Option] retries = partial( Option, @@ -219,7 +240,7 @@ retries = partial( default=5, help="Maximum number of retries each connection should attempt " "(default %default times).", -) # type: Any +) # type: Callable[..., Option] timeout = partial( Option, @@ -229,7 +250,7 @@ timeout = partial( type='float', default=15, help='Set the socket timeout (default %default seconds).', -) # type: Any +) # type: Callable[..., Option] skip_requirements_regex = partial( Option, @@ -239,10 +260,11 @@ skip_requirements_regex = partial( type='str', default='', help=SUPPRESS_HELP, -) # type: Any +) # type: Callable[..., Option] def exists_action(): + # type: () -> Option return Option( # Option when path already exist '--exists-action', @@ -264,7 +286,7 @@ cert = partial( type='str', metavar='path', help="Path to alternate CA bundle.", -) # type: Any +) # type: Callable[..., Option] client_cert = partial( Option, @@ -275,7 +297,7 @@ client_cert = partial( metavar='path', help="Path to SSL client certificate, a single file containing the " "private key and the certificate in PEM format.", -) # type: Any +) # type: Callable[..., Option] index_url = partial( Option, @@ -287,7 +309,7 @@ index_url = partial( "This should point to a repository compliant with PEP 503 " "(the simple repository API) or a local directory laid out " "in the same format.", -) # type: Any +) # type: Callable[..., Option] def extra_index_url(): @@ -310,10 +332,11 @@ no_index = partial( action='store_true', default=False, help='Ignore package index (only looking at --find-links URLs instead).', -) # type: Any +) # type: Callable[..., Option] def find_links(): + # type: () -> Option return Option( '-f', '--find-links', dest='find_links', @@ -327,6 +350,7 @@ def find_links(): def trusted_host(): + # type: () -> Option return Option( "--trusted-host", dest="trusted_hosts", @@ -338,18 +362,8 @@ def trusted_host(): ) -# Remove after 1.5 -process_dependency_links = partial( - Option, - "--process-dependency-links", - dest="process_dependency_links", - action="store_true", - default=False, - help="Enable the processing of dependency links.", -) # type: Any - - def constraints(): + # type: () -> Option return Option( '-c', '--constraint', dest='constraints', @@ -362,6 +376,7 @@ def constraints(): def requirements(): + # type: () -> Option return Option( '-r', '--requirement', dest='requirements', @@ -374,6 +389,7 @@ def requirements(): def editable(): + # type: () -> Option return Option( '-e', '--editable', dest='editables', @@ -394,15 +410,17 @@ src = partial( help='Directory to check out editable projects into. ' 'The default in a virtualenv is "/src". ' 'The default for global installs is "/src".' -) # type: Any +) # type: Callable[..., Option] def _get_format_control(values, option): + # type: (Values, Option) -> Any """Get a format_control object.""" return getattr(values, option.dest) def _handle_no_binary(option, opt_str, value, parser): + # type: (Option, str, str, OptionParser) -> None existing = _get_format_control(parser.values, option) FormatControl.handle_mutual_excludes( value, existing.no_binary, existing.only_binary, @@ -410,6 +428,7 @@ def _handle_no_binary(option, opt_str, value, parser): def _handle_only_binary(option, opt_str, value, parser): + # type: (Option, str, str, OptionParser) -> None existing = _get_format_control(parser.values, option) FormatControl.handle_mutual_excludes( value, existing.only_binary, existing.no_binary, @@ -417,6 +436,7 @@ def _handle_only_binary(option, opt_str, value, parser): def no_binary(): + # type: () -> Option format_control = FormatControl(set(), set()) return Option( "--no-binary", dest="format_control", action="callback", @@ -432,6 +452,7 @@ def no_binary(): def only_binary(): + # type: () -> Option format_control = FormatControl(set(), set()) return Option( "--only-binary", dest="format_control", action="callback", @@ -454,7 +475,7 @@ platform = partial( default=None, help=("Only use wheels compatible with . " "Defaults to the platform of the running system."), -) +) # type: Callable[..., Option] python_version = partial( @@ -469,7 +490,7 @@ python_version = partial( "version (e.g. '2') can be specified to match all " "minor revs of that major version. A minor version " "(e.g. '34') can also be specified."), -) +) # type: Callable[..., Option] implementation = partial( @@ -483,7 +504,7 @@ implementation = partial( " or 'ip'. If not specified, then the current " "interpreter implementation is used. Use 'py' to force " "implementation-agnostic wheels."), -) +) # type: Callable[..., Option] abi = partial( @@ -498,10 +519,11 @@ abi = partial( "you will need to specify --implementation, " "--platform, and --python-version when using " "this option."), -) +) # type: Callable[..., Option] def prefer_binary(): + # type: () -> Option return Option( "--prefer-binary", dest="prefer_binary", @@ -518,15 +540,44 @@ cache_dir = partial( default=USER_CACHE_DIR, metavar="dir", help="Store the cache data in

." -) +) # type: Callable[..., Option] + + +def no_cache_dir_callback(option, opt, value, parser): + """ + Process a value provided for the --no-cache-dir option. + + This is an optparse.Option callback for the --no-cache-dir option. + """ + # The value argument will be None if --no-cache-dir is passed via the + # command-line, since the option doesn't accept arguments. However, + # the value can be non-None if the option is triggered e.g. by an + # environment variable, like PIP_NO_CACHE_DIR=true. + if value is not None: + # Then parse the string value to get argument error-checking. + try: + strtobool(value) + except ValueError as exc: + raise_option_error(parser, option=option, msg=str(exc)) + + # Originally, setting PIP_NO_CACHE_DIR to a value that strtobool() + # converted to 0 (like "false" or "no") caused cache_dir to be disabled + # rather than enabled (logic would say the latter). Thus, we disable + # the cache directory not just on values that parse to True, but (for + # backwards compatibility reasons) also on values that parse to False. + # In other words, always set it to False if the option is provided in + # some (valid) form. + parser.values.cache_dir = False + no_cache = partial( Option, "--no-cache-dir", dest="cache_dir", - action="store_false", + action="callback", + callback=no_cache_dir_callback, help="Disable the cache.", -) +) # type: Callable[..., Option] no_deps = partial( Option, @@ -535,7 +586,7 @@ no_deps = partial( action='store_true', default=False, help="Don't install package dependencies.", -) # type: Any +) # type: Callable[..., Option] build_dir = partial( Option, @@ -547,7 +598,7 @@ build_dir = partial( 'The location of temporary directories can be controlled by setting ' 'the TMPDIR environment variable (TEMP on Windows) appropriately. ' 'When passed, build directories are not cleaned in case of failures.' -) # type: Any +) # type: Callable[..., Option] ignore_requires_python = partial( Option, @@ -555,7 +606,7 @@ ignore_requires_python = partial( dest='ignore_requires_python', action='store_true', help='Ignore the Requires-Python information.' -) # type: Any +) # type: Callable[..., Option] no_build_isolation = partial( Option, @@ -566,6 +617,50 @@ no_build_isolation = partial( help='Disable isolation when building a modern source distribution. ' 'Build dependencies specified by PEP 518 must be already installed ' 'if this option is used.' +) # type: Callable[..., Option] + + +def no_use_pep517_callback(option, opt, value, parser): + """ + Process a value provided for the --no-use-pep517 option. + + This is an optparse.Option callback for the no_use_pep517 option. + """ + # Since --no-use-pep517 doesn't accept arguments, the value argument + # will be None if --no-use-pep517 is passed via the command-line. + # However, the value can be non-None if the option is triggered e.g. + # by an environment variable, for example "PIP_NO_USE_PEP517=true". + if value is not None: + msg = """A value was passed for --no-use-pep517, + probably using either the PIP_NO_USE_PEP517 environment variable + or the "no-use-pep517" config file option. Use an appropriate value + of the PIP_USE_PEP517 environment variable or the "use-pep517" + config file option instead. + """ + raise_option_error(parser, option=option, msg=msg) + + # Otherwise, --no-use-pep517 was passed via the command-line. + parser.values.use_pep517 = False + + +use_pep517 = partial( + Option, + '--use-pep517', + dest='use_pep517', + action='store_true', + default=None, + help='Use PEP 517 for building source distributions ' + '(use --no-use-pep517 to force legacy behaviour).' +) # type: Any + +no_use_pep517 = partial( + Option, + '--no-use-pep517', + dest='use_pep517', + action='callback', + callback=no_use_pep517_callback, + default=None, + help=SUPPRESS_HELP ) # type: Any install_options = partial( @@ -579,7 +674,7 @@ install_options = partial( "bin\"). Use multiple --install-option options to pass multiple " "options to setup.py install. If you are using an option with a " "directory path, be sure to use absolute path.", -) # type: Any +) # type: Callable[..., Option] global_options = partial( Option, @@ -589,7 +684,7 @@ global_options = partial( metavar='options', help="Extra global options to be supplied to the setup.py " "call before the install command.", -) # type: Any +) # type: Callable[..., Option] no_clean = partial( Option, @@ -597,7 +692,7 @@ no_clean = partial( action='store_true', default=False, help="Don't clean up build directories." -) # type: Any +) # type: Callable[..., Option] pre = partial( Option, @@ -606,7 +701,7 @@ pre = partial( default=False, help="Include pre-release and development versions. By default, " "pip only finds stable versions.", -) # type: Any +) # type: Callable[..., Option] disable_pip_version_check = partial( Option, @@ -616,7 +711,7 @@ disable_pip_version_check = partial( default=False, help="Don't periodically check PyPI to determine whether a new version " "of pip is available for download. Implied with --no-index.", -) # type: Any +) # type: Callable[..., Option] # Deprecated, Remove later @@ -626,14 +721,15 @@ always_unzip = partial( dest='always_unzip', action='store_true', help=SUPPRESS_HELP, -) # type: Any +) # type: Callable[..., Option] def _merge_hash(option, opt_str, value, parser): + # type: (Option, str, str, OptionParser) -> None """Given a value spelled "algo:digest", append the digest to a list pointed to in a dict by the algo name.""" if not parser.values.hashes: - parser.values.hashes = {} + parser.values.hashes = {} # type: ignore try: algo, digest = value.split(':', 1) except ValueError: @@ -657,7 +753,7 @@ hash = partial( type='string', help="Verify that the package's archive matches this " 'hash before installing. Example: --hash=sha256:abcdef...', -) # type: Any +) # type: Callable[..., Option] require_hashes = partial( @@ -669,7 +765,7 @@ require_hashes = partial( help='Require a hash to check each requirement against, for ' 'repeatable installs. This option is implied when any package in a ' 'requirements file has a --hash option.', -) # type: Any +) # type: Callable[..., Option] ########## @@ -700,7 +796,7 @@ general_group = { disable_pip_version_check, no_color, ] -} +} # type: Dict[str, Any] index_group = { 'name': 'Package Index Options', @@ -709,6 +805,5 @@ index_group = { extra_index_url, no_index, find_links, - process_dependency_links, ] -} +} # type: Dict[str, Any] diff --git a/pipenv/patched/notpip/_internal/cli/main_parser.py b/pipenv/patched/notpip/_internal/cli/main_parser.py index abe2f69e..704bf404 100644 --- a/pipenv/patched/notpip/_internal/cli/main_parser.py +++ b/pipenv/patched/notpip/_internal/cli/main_parser.py @@ -14,11 +14,17 @@ from pipenv.patched.notpip._internal.commands import ( ) from pipenv.patched.notpip._internal.exceptions import CommandError from pipenv.patched.notpip._internal.utils.misc import get_prog +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Tuple, List # noqa: F401 + __all__ = ["create_main_parser", "parse_command"] def create_main_parser(): + # type: () -> ConfigOptionParser """Creates and returns the main parser for pip's CLI """ @@ -44,7 +50,8 @@ def create_main_parser(): gen_opts = cmdoptions.make_option_group(cmdoptions.general_group, parser) parser.add_option_group(gen_opts) - parser.main = True # so the help formatter knows + # so the help formatter knows + parser.main = True # type: ignore # create command listing for description command_summaries = get_summaries() @@ -55,6 +62,7 @@ def create_main_parser(): def parse_command(args): + # type: (List[str]) -> Tuple[str, List[str]] parser = create_main_parser() # Note: parser calls disable_interspersed_args(), so the result of this @@ -68,7 +76,7 @@ def parse_command(args): # --version if general_options.version: - sys.stdout.write(parser.version) + sys.stdout.write(parser.version) # type: ignore sys.stdout.write(os.linesep) sys.exit() diff --git a/pipenv/patched/notpip/_internal/commands/check.py b/pipenv/patched/notpip/_internal/commands/check.py index adf4f5e7..cf84f5df 100644 --- a/pipenv/patched/notpip/_internal/commands/check.py +++ b/pipenv/patched/notpip/_internal/commands/check.py @@ -16,7 +16,7 @@ class CheckCommand(Command): summary = 'Verify installed packages have compatible dependencies.' def run(self, options, args): - package_set = create_package_set_from_installed() + package_set, parsing_probs = create_package_set_from_installed() missing, conflicting = check_package_set(package_set) for project_name in missing: @@ -35,7 +35,7 @@ class CheckCommand(Command): project_name, version, req, dep_name, dep_version, ) - if missing or conflicting: + if missing or conflicting or parsing_probs: return 1 else: logger.info("No broken requirements found.") diff --git a/pipenv/patched/notpip/_internal/commands/download.py b/pipenv/patched/notpip/_internal/commands/download.py index e5d87121..133ca135 100644 --- a/pipenv/patched/notpip/_internal/commands/download.py +++ b/pipenv/patched/notpip/_internal/commands/download.py @@ -58,6 +58,8 @@ class DownloadCommand(RequirementCommand): cmd_opts.add_option(cmdoptions.require_hashes()) cmd_opts.add_option(cmdoptions.progress_bar()) cmd_opts.add_option(cmdoptions.no_build_isolation()) + cmd_opts.add_option(cmdoptions.use_pep517()) + cmd_opts.add_option(cmdoptions.no_use_pep517()) cmd_opts.add_option( '-d', '--dest', '--destination-dir', '--destination-directory', diff --git a/pipenv/patched/notpip/_internal/commands/install.py b/pipenv/patched/notpip/_internal/commands/install.py index ddcb4759..68255c85 100644 --- a/pipenv/patched/notpip/_internal/commands/install.py +++ b/pipenv/patched/notpip/_internal/commands/install.py @@ -30,12 +30,6 @@ from pipenv.patched.notpip._internal.utils.misc import ( from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory from pipenv.patched.notpip._internal.wheel import WheelBuilder -try: - import wheel -except ImportError: - wheel = None - - logger = logging.getLogger(__name__) @@ -158,6 +152,8 @@ class InstallCommand(RequirementCommand): cmd_opts.add_option(cmdoptions.ignore_requires_python()) cmd_opts.add_option(cmdoptions.no_build_isolation()) + cmd_opts.add_option(cmdoptions.use_pep517()) + cmd_opts.add_option(cmdoptions.no_use_pep517()) cmd_opts.add_option(cmdoptions.install_options()) cmd_opts.add_option(cmdoptions.global_options()) @@ -314,6 +310,7 @@ class InstallCommand(RequirementCommand): ignore_requires_python=options.ignore_requires_python, ignore_installed=options.ignore_installed, isolated=options.isolated_mode, + use_pep517=options.use_pep517 ) resolver.resolve(requirement_set) @@ -321,21 +318,51 @@ class InstallCommand(RequirementCommand): modifying_pip=requirement_set.has_requirement("pip") ) - # If caching is disabled or wheel is not installed don't - # try to build wheels. - if wheel and options.cache_dir: - # build wheels before install. - wb = WheelBuilder( - finder, preparer, wheel_cache, - build_options=[], global_options=[], - ) - # Ignore the result: a failed wheel will be - # installed from the sdist/vcs whatever. + # Consider legacy and PEP517-using requirements separately + legacy_requirements = [] + pep517_requirements = [] + for req in requirement_set.requirements.values(): + if req.use_pep517: + pep517_requirements.append(req) + else: + legacy_requirements.append(req) + + # We don't build wheels for legacy requirements if we + # don't have wheel installed or we don't have a cache dir + try: + import wheel # noqa: F401 + build_legacy = bool(options.cache_dir) + except ImportError: + build_legacy = False + + wb = WheelBuilder( + finder, preparer, wheel_cache, + build_options=[], global_options=[], + ) + + # Always build PEP 517 requirements + build_failures = wb.build( + pep517_requirements, + session=session, autobuilding=True + ) + + if build_legacy: + # We don't care about failures building legacy + # requirements, as we'll fall through to a direct + # install for those. wb.build( - requirement_set.requirements.values(), + legacy_requirements, session=session, autobuilding=True ) + # If we're using PEP 517, we cannot do a direct install + # so we fail here. + if build_failures: + raise InstallationError( + "Could not build wheels for {} which use" + " PEP 517 and cannot be installed directly".format( + ", ".join(r.name for r in build_failures))) + to_install = resolver.get_installation_order( requirement_set ) @@ -472,7 +499,11 @@ class InstallCommand(RequirementCommand): ) def _warn_about_conflicts(self, to_install): - package_set, _dep_info = check_install_conflicts(to_install) + try: + package_set, _dep_info = check_install_conflicts(to_install) + except Exception: + logger.error("Error checking for conflicts.", exc_info=True) + return missing, conflicting = _dep_info # NOTE: There is some duplication here from pipenv.patched.notpip check diff --git a/pipenv/patched/notpip/_internal/commands/list.py b/pipenv/patched/notpip/_internal/commands/list.py index 577c0b5f..a2bd5be1 100644 --- a/pipenv/patched/notpip/_internal/commands/list.py +++ b/pipenv/patched/notpip/_internal/commands/list.py @@ -118,7 +118,6 @@ class ListCommand(Command): index_urls=index_urls, allow_all_prereleases=options.pre, trusted_hosts=options.trusted_hosts, - process_dependency_links=options.process_dependency_links, session=session, ) @@ -134,14 +133,18 @@ class ListCommand(Command): include_editables=options.include_editable, ) + # get_not_required must be called firstly in order to find and + # filter out all dependencies correctly. Otherwise a package + # can't be identified as requirement because some parent packages + # could be filtered out before. + if options.not_required: + packages = self.get_not_required(packages, options) + if options.outdated: packages = self.get_outdated(packages, options) elif options.uptodate: packages = self.get_uptodate(packages, options) - if options.not_required: - packages = self.get_not_required(packages, options) - self.output_package_listing(packages, options) def get_outdated(self, packages, options): @@ -168,16 +171,8 @@ class ListCommand(Command): logger.debug('Ignoring indexes: %s', ','.join(index_urls)) index_urls = [] - dependency_links = [] - for dist in packages: - if dist.has_metadata('dependency_links.txt'): - dependency_links.extend( - dist.get_metadata_lines('dependency_links.txt'), - ) - with self._build_session(options) as session: finder = self._build_package_finder(options, index_urls, session) - finder.add_dependency_links(dependency_links) for dist in packages: typ = 'unknown' diff --git a/pipenv/patched/notpip/_internal/commands/wheel.py b/pipenv/patched/notpip/_internal/commands/wheel.py index 08d695ab..801efff8 100644 --- a/pipenv/patched/notpip/_internal/commands/wheel.py +++ b/pipenv/patched/notpip/_internal/commands/wheel.py @@ -67,6 +67,8 @@ class WheelCommand(RequirementCommand): help="Extra arguments to be supplied to 'setup.py bdist_wheel'.", ) cmd_opts.add_option(cmdoptions.no_build_isolation()) + cmd_opts.add_option(cmdoptions.use_pep517()) + cmd_opts.add_option(cmdoptions.no_use_pep517()) cmd_opts.add_option(cmdoptions.constraints()) cmd_opts.add_option(cmdoptions.editable()) cmd_opts.add_option(cmdoptions.requirements()) @@ -157,6 +159,7 @@ class WheelCommand(RequirementCommand): ignore_requires_python=options.ignore_requires_python, ignore_installed=True, isolated=options.isolated_mode, + use_pep517=options.use_pep517 ) resolver.resolve(requirement_set) @@ -167,10 +170,10 @@ class WheelCommand(RequirementCommand): global_options=options.global_options or [], no_clean=options.no_clean, ) - wheels_built_successfully = wb.build( + build_failures = wb.build( requirement_set.requirements.values(), session=session, ) - if not wheels_built_successfully: + if len(build_failures) != 0: raise CommandError( "Failed to build one or more wheels" ) diff --git a/pipenv/patched/notpip/_internal/download.py b/pipenv/patched/notpip/_internal/download.py index 8f4c38f5..f593c2f2 100644 --- a/pipenv/patched/notpip/_internal/download.py +++ b/pipenv/patched/notpip/_internal/download.py @@ -19,7 +19,6 @@ from pipenv.patched.notpip._vendor.lockfile import LockError from pipenv.patched.notpip._vendor.requests.adapters import BaseAdapter, HTTPAdapter from pipenv.patched.notpip._vendor.requests.auth import AuthBase, HTTPBasicAuth from pipenv.patched.notpip._vendor.requests.models import CONTENT_CHUNK_SIZE, Response -from pipenv.patched.notpip._vendor.requests.sessions import Session from pipenv.patched.notpip._vendor.requests.structures import CaseInsensitiveDict from pipenv.patched.notpip._vendor.requests.utils import get_netrc_auth # NOTE: XMLRPC Client is not annotated in typeshed as on 2017-07-17, which is @@ -27,7 +26,6 @@ from pipenv.patched.notpip._vendor.requests.utils import get_netrc_auth from pipenv.patched.notpip._vendor.six.moves import xmlrpc_client # type: ignore from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse from pipenv.patched.notpip._vendor.six.moves.urllib import request as urllib_request -from pipenv.patched.notpip._vendor.six.moves.urllib.parse import unquote as urllib_unquote from pipenv.patched.notpip._vendor.urllib3.util import IS_PYOPENSSL import pipenv.patched.notpip @@ -40,14 +38,23 @@ from pipenv.patched.notpip._internal.utils.glibc import libc_ver from pipenv.patched.notpip._internal.utils.logging import indent_log from pipenv.patched.notpip._internal.utils.misc import ( ARCHIVE_EXTENSIONS, ask_path_exists, backup_dir, call_subprocess, consume, - display_path, format_size, get_installed_version, rmtree, splitext, - unpack_file, + display_path, format_size, get_installed_version, rmtree, + split_auth_from_netloc, splitext, unpack_file, ) from pipenv.patched.notpip._internal.utils.setuptools_build import SETUPTOOLS_SHIM from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING from pipenv.patched.notpip._internal.utils.ui import DownloadProgressProvider from pipenv.patched.notpip._internal.vcs import vcs +if MYPY_CHECK_RUNNING: + from typing import ( # noqa: F401 + Optional, Tuple, Dict, IO, Text, Union + ) + from pipenv.patched.notpip._internal.models.link import Link # noqa: F401 + from pipenv.patched.notpip._internal.utils.hashes import Hashes # noqa: F401 + from pipenv.patched.notpip._internal.vcs import AuthInfo # noqa: F401 + try: import ssl # noqa except ImportError: @@ -137,14 +144,15 @@ def user_agent(): class MultiDomainBasicAuth(AuthBase): def __init__(self, prompting=True): + # type: (bool) -> None self.prompting = prompting - self.passwords = {} + self.passwords = {} # type: Dict[str, AuthInfo] def __call__(self, req): parsed = urllib_parse.urlparse(req.url) - # Get the netloc without any embedded credentials - netloc = parsed.netloc.rsplit("@", 1)[-1] + # Split the credentials from the netloc. + netloc, url_user_password = split_auth_from_netloc(parsed.netloc) # Set the url of the request to the url without any credentials req.url = urllib_parse.urlunparse(parsed[:1] + (netloc,) + parsed[2:]) @@ -152,9 +160,9 @@ class MultiDomainBasicAuth(AuthBase): # Use any stored credentials that we have for this netloc username, password = self.passwords.get(netloc, (None, None)) - # Extract credentials embedded in the url if we have none stored + # Use the credentials embedded in the url if we have none stored if username is None: - username, password = self.parse_credentials(parsed.netloc) + username, password = url_user_password # Get creds from netrc if we still don't have them if username is None and password is None: @@ -200,6 +208,7 @@ class MultiDomainBasicAuth(AuthBase): # Add our new username and password to the request req = HTTPBasicAuth(username or "", password or "")(resp.request) + req.register_hook("response", self.warn_on_401) # Send our new request new_resp = resp.connection.send(req, **kwargs) @@ -207,14 +216,11 @@ class MultiDomainBasicAuth(AuthBase): return new_resp - def parse_credentials(self, netloc): - if "@" in netloc: - userinfo = netloc.rsplit("@", 1)[0] - if ":" in userinfo: - user, pwd = userinfo.split(":", 1) - return (urllib_unquote(user), urllib_unquote(pwd)) - return urllib_unquote(userinfo), None - return None, None + def warn_on_401(self, resp, **kwargs): + # warn user that they provided incorrect credentials + if resp.status_code == 401: + logger.warning('401 Error, Credentials not correct for %s', + resp.request.url) class LocalFSAdapter(BaseAdapter): @@ -325,7 +331,7 @@ class InsecureHTTPAdapter(HTTPAdapter): class PipSession(requests.Session): - timeout = None + timeout = None # type: Optional[int] def __init__(self, *args, **kwargs): retries = kwargs.pop("retries", 0) @@ -398,6 +404,7 @@ class PipSession(requests.Session): def get_file_content(url, comes_from=None, session=None): + # type: (str, Optional[str], Optional[PipSession]) -> Tuple[str, Text] """Gets the content of a file; it may be a filename, file: URL, or http: URL. Returns (location, content). Content is unicode. @@ -448,6 +455,7 @@ _url_slash_drive_re = re.compile(r'/*([a-z])\|', re.I) def is_url(name): + # type: (Union[str, Text]) -> bool """Returns true if the name looks like a URL""" if ':' not in name: return False @@ -456,6 +464,7 @@ def is_url(name): def url_to_path(url): + # type: (str) -> str """ Convert a file: URL to a path. """ @@ -473,6 +482,7 @@ def url_to_path(url): def path_to_url(path): + # type: (Union[str, Text]) -> str """ Convert a path to a file: URL. The path will be made absolute and have quoted path parts. @@ -483,6 +493,7 @@ def path_to_url(path): def is_archive_file(name): + # type: (str) -> bool """Return True if `name` is a considered as an archive file.""" ext = splitext(name)[1].lower() if ext in ARCHIVE_EXTENSIONS: @@ -503,14 +514,17 @@ def _get_used_vcs_backend(link): def is_vcs_url(link): + # type: (Link) -> bool return bool(_get_used_vcs_backend(link)) def is_file_url(link): + # type: (Link) -> bool return link.url.lower().startswith('file:') def is_dir_url(link): + # type: (Link) -> bool """Return whether a file:// Link points to a directory. ``link`` must not have any other scheme but file://. Call is_file_url() @@ -525,7 +539,14 @@ def _progress_indicator(iterable, *args, **kwargs): return iterable -def _download_url(resp, link, content_file, hashes, progress_bar): +def _download_url( + resp, # type: Response + link, # type: Link + content_file, # type: IO + hashes, # type: Hashes + progress_bar # type: str +): + # type: (...) -> None try: total_length = int(resp.headers['content-length']) except (ValueError, KeyError, TypeError): @@ -647,8 +668,15 @@ def _copy_file(filename, location, link): logger.info('Saved %s', display_path(download_location)) -def unpack_http_url(link, location, download_dir=None, - session=None, hashes=None, progress_bar="on"): +def unpack_http_url( + link, # type: Link + location, # type: str + download_dir=None, # type: Optional[str] + session=None, # type: Optional[PipSession] + hashes=None, # type: Optional[Hashes] + progress_bar="on" # type: str +): + # type: (...) -> None if session is None: raise TypeError( "unpack_http_url() missing 1 required keyword argument: 'session'" @@ -685,7 +713,13 @@ def unpack_http_url(link, location, download_dir=None, os.unlink(from_path) -def unpack_file_url(link, location, download_dir=None, hashes=None): +def unpack_file_url( + link, # type: Link + location, # type: str + download_dir=None, # type: Optional[str] + hashes=None # type: Optional[Hashes] +): + # type: (...) -> None """Unpack link into location. If download_dir is provided and link points to a file, make a copy @@ -798,9 +832,16 @@ class PipXmlrpcTransport(xmlrpc_client.Transport): raise -def unpack_url(link, location, download_dir=None, - only_download=False, session=None, hashes=None, - progress_bar="on"): +def unpack_url( + link, # type: Optional[Link] + location, # type: Optional[str] + download_dir=None, # type: Optional[str] + only_download=False, # type: bool + session=None, # type: Optional[PipSession] + hashes=None, # type: Optional[Hashes] + progress_bar="on" # type: str +): + # type: (...) -> None """Unpack link. If link is a VCS link: if only_download, export into download_dir and ignore location @@ -840,7 +881,14 @@ def unpack_url(link, location, download_dir=None, write_delete_marker_file(location) -def _download_http_url(link, session, temp_dir, hashes, progress_bar): +def _download_http_url( + link, # type: Link + session, # type: PipSession + temp_dir, # type: str + hashes, # type: Hashes + progress_bar # type: str +): + # type: (...) -> Tuple[str, str] """Download link url into temp_dir using provided session""" target_url = link.url.split('#', 1)[0] try: @@ -900,6 +948,7 @@ def _download_http_url(link, session, temp_dir, hashes, progress_bar): def _check_download_dir(link, download_dir, hashes): + # type: (Link, str, Hashes) -> Optional[str] """ Check download_dir for previously downloaded file with correct hash If a correct file is found return its path else None """ diff --git a/pipenv/patched/notpip/_internal/exceptions.py b/pipenv/patched/notpip/_internal/exceptions.py index 2eadcf28..1342935d 100644 --- a/pipenv/patched/notpip/_internal/exceptions.py +++ b/pipenv/patched/notpip/_internal/exceptions.py @@ -5,6 +5,12 @@ from itertools import chain, groupby, repeat from pipenv.patched.notpip._vendor.six import iteritems +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Optional # noqa: F401 + from pipenv.patched.notpip._internal.req.req_install import InstallRequirement # noqa: F401 + class PipError(Exception): """Base pip exception""" @@ -96,7 +102,7 @@ class HashError(InstallationError): typically available earlier. """ - req = None + req = None # type: Optional[InstallRequirement] head = '' def body(self): diff --git a/pipenv/patched/notpip/_internal/index.py b/pipenv/patched/notpip/_internal/index.py index b4b02373..ad145fe4 100644 --- a/pipenv/patched/notpip/_internal/index.py +++ b/pipenv/patched/notpip/_internal/index.py @@ -16,7 +16,7 @@ from pipenv.patched.notpip._vendor.distlib.compat import unescape from pipenv.patched.notpip._vendor.packaging import specifiers from pipenv.patched.notpip._vendor.packaging.utils import canonicalize_name from pipenv.patched.notpip._vendor.packaging.version import parse as parse_version -from pipenv.patched.notpip._vendor.requests.exceptions import SSLError +from pipenv.patched.notpip._vendor.requests.exceptions import RetryError, SSLError from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse from pipenv.patched.notpip._vendor.six.moves.urllib import request as urllib_request @@ -31,14 +31,29 @@ from pipenv.patched.notpip._internal.models.index import PyPI from pipenv.patched.notpip._internal.models.link import Link from pipenv.patched.notpip._internal.pep425tags import get_supported from pipenv.patched.notpip._internal.utils.compat import ipaddress -from pipenv.patched.notpip._internal.utils.deprecation import deprecated from pipenv.patched.notpip._internal.utils.logging import indent_log from pipenv.patched.notpip._internal.utils.misc import ( - ARCHIVE_EXTENSIONS, SUPPORTED_EXTENSIONS, normalize_path, - remove_auth_from_url, + ARCHIVE_EXTENSIONS, SUPPORTED_EXTENSIONS, WHEEL_EXTENSION, normalize_path, + redact_password_from_url, ) from pipenv.patched.notpip._internal.utils.packaging import check_requires_python -from pipenv.patched.notpip._internal.wheel import Wheel, wheel_ext +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING +from pipenv.patched.notpip._internal.wheel import Wheel + +if MYPY_CHECK_RUNNING: + from logging import Logger # noqa: F401 + from typing import ( # noqa: F401 + Tuple, Optional, Any, List, Union, Callable, Set, Sequence, + Iterable, MutableMapping + ) + from pipenv.patched.notpip._vendor.packaging.version import _BaseVersion # noqa: F401 + from pipenv.patched.notpip._vendor.requests import Response # noqa: F401 + from pipenv.patched.notpip._internal.req import InstallRequirement # noqa: F401 + from pipenv.patched.notpip._internal.download import PipSession # noqa: F401 + + SecureOrigin = Tuple[str, str, Optional[str]] + BuildTag = Tuple[Any, ...] # either emply tuple or Tuple[int, str] + CandidateSortingKey = Tuple[int, _BaseVersion, BuildTag, Optional[int]] __all__ = ['FormatControl', 'PackageFinder'] @@ -53,126 +68,190 @@ SECURE_ORIGINS = [ ("file", "*", None), # ssh is always secure. ("ssh", "*", "*"), -] +] # type: List[SecureOrigin] logger = logging.getLogger(__name__) -def _get_content_type(url, session): - """Get the Content-Type of the given url, using a HEAD request""" +def _match_vcs_scheme(url): + # type: (str) -> Optional[str] + """Look for VCS schemes in the URL. + + Returns the matched VCS scheme, or None if there's no match. + """ + from pipenv.patched.notpip._internal.vcs import VcsSupport + for scheme in VcsSupport.schemes: + if url.lower().startswith(scheme) and url[len(scheme)] in '+:': + return scheme + return None + + +def _is_url_like_archive(url): + # type: (str) -> bool + """Return whether the URL looks like an archive. + """ + filename = Link(url).filename + for bad_ext in ARCHIVE_EXTENSIONS: + if filename.endswith(bad_ext): + return True + return False + + +class _NotHTML(Exception): + def __init__(self, content_type, request_desc): + # type: (str, str) -> None + super(_NotHTML, self).__init__(content_type, request_desc) + self.content_type = content_type + self.request_desc = request_desc + + +def _ensure_html_header(response): + # type: (Response) -> None + """Check the Content-Type header to ensure the response contains HTML. + + Raises `_NotHTML` if the content type is not text/html. + """ + content_type = response.headers.get("Content-Type", "") + if not content_type.lower().startswith("text/html"): + raise _NotHTML(content_type, response.request.method) + + +class _NotHTTP(Exception): + pass + + +def _ensure_html_response(url, session): + # type: (str, PipSession) -> None + """Send a HEAD request to the URL, and ensure the response contains HTML. + + Raises `_NotHTTP` if the URL is not available for a HEAD request, or + `_NotHTML` if the content type is not text/html. + """ scheme, netloc, path, query, fragment = urllib_parse.urlsplit(url) if scheme not in {'http', 'https'}: - # FIXME: some warning or something? - # assertion error? - return '' + raise _NotHTTP() resp = session.head(url, allow_redirects=True) resp.raise_for_status() - return resp.headers.get("Content-Type", "") + _ensure_html_header(resp) -def _handle_get_page_fail(link, reason, url, meth=None): +def _get_html_response(url, session): + # type: (str, PipSession) -> Response + """Access an HTML page with GET, and return the response. + + This consists of three parts: + + 1. If the URL looks suspiciously like an archive, send a HEAD first to + check the Content-Type is HTML, to avoid downloading a large file. + Raise `_NotHTTP` if the content type cannot be determined, or + `_NotHTML` if it is not HTML. + 2. Actually perform the request. Raise HTTP exceptions on network failures. + 3. Check the Content-Type header to make sure we got HTML, and raise + `_NotHTML` otherwise. + """ + if _is_url_like_archive(url): + _ensure_html_response(url, session=session) + + logger.debug('Getting page %s', url) + + resp = session.get( + url, + headers={ + "Accept": "text/html", + # We don't want to blindly returned cached data for + # /simple/, because authors generally expecting that + # twine upload && pip install will function, but if + # they've done a pip install in the last ~10 minutes + # it won't. Thus by setting this to zero we will not + # blindly use any cached data, however the benefit of + # using max-age=0 instead of no-cache, is that we will + # still support conditional requests, so we will still + # minimize traffic sent in cases where the page hasn't + # changed at all, we will just always incur the round + # trip for the conditional GET now instead of only + # once per 10 minutes. + # For more information, please see pypa/pip#5670. + "Cache-Control": "max-age=0", + }, + ) + resp.raise_for_status() + + # The check for archives above only works if the url ends with + # something that looks like an archive. However that is not a + # requirement of an url. Unless we issue a HEAD request on every + # url we cannot know ahead of time for sure if something is HTML + # or not. However we can check after we've downloaded it. + _ensure_html_header(resp) + + return resp + + +def _handle_get_page_fail( + link, # type: Link + reason, # type: Union[str, Exception] + meth=None # type: Optional[Callable[..., None]] +): + # type: (...) -> None if meth is None: meth = logger.debug meth("Could not fetch URL %s: %s - skipping", link, reason) def _get_html_page(link, session=None): + # type: (Link, Optional[PipSession]) -> Optional[HTMLPage] if session is None: raise TypeError( "_get_html_page() missing 1 required keyword argument: 'session'" ) - url = link.url - url = url.split('#', 1)[0] + url = link.url.split('#', 1)[0] # Check for VCS schemes that do not support lookup as web pages. - from pipenv.patched.notpip._internal.vcs import VcsSupport - for scheme in VcsSupport.schemes: - if url.lower().startswith(scheme) and url[len(scheme)] in '+:': - logger.debug('Cannot look at %s URL %s', scheme, link) - return None + vcs_scheme = _match_vcs_scheme(url) + if vcs_scheme: + logger.debug('Cannot look at %s URL %s', vcs_scheme, link) + return None + + # Tack index.html onto file:// URLs that point to directories + scheme, _, path, _, _, _ = urllib_parse.urlparse(url) + if (scheme == 'file' and os.path.isdir(urllib_request.url2pathname(path))): + # add trailing slash if not present so urljoin doesn't trim + # final segment + if not url.endswith('/'): + url += '/' + url = urllib_parse.urljoin(url, 'index.html') + logger.debug(' file: URL is directory, getting %s', url) try: - filename = link.filename - for bad_ext in ARCHIVE_EXTENSIONS: - if filename.endswith(bad_ext): - content_type = _get_content_type(url, session=session) - if content_type.lower().startswith('text/html'): - break - else: - logger.debug( - 'Skipping page %s because of Content-Type: %s', - link, - content_type, - ) - return - - logger.debug('Getting page %s', url) - - # Tack index.html onto file:// URLs that point to directories - (scheme, netloc, path, params, query, fragment) = \ - urllib_parse.urlparse(url) - if (scheme == 'file' and - os.path.isdir(urllib_request.url2pathname(path))): - # add trailing slash if not present so urljoin doesn't trim - # final segment - if not url.endswith('/'): - url += '/' - url = urllib_parse.urljoin(url, 'index.html') - logger.debug(' file: URL is directory, getting %s', url) - - resp = session.get( - url, - headers={ - "Accept": "text/html", - # We don't want to blindly returned cached data for - # /simple/, because authors generally expecting that - # twine upload && pip install will function, but if - # they've done a pip install in the last ~10 minutes - # it won't. Thus by setting this to zero we will not - # blindly use any cached data, however the benefit of - # using max-age=0 instead of no-cache, is that we will - # still support conditional requests, so we will still - # minimize traffic sent in cases where the page hasn't - # changed at all, we will just always incur the round - # trip for the conditional GET now instead of only - # once per 10 minutes. - # For more information, please see pypa/pip#5670. - "Cache-Control": "max-age=0", - }, + resp = _get_html_response(url, session=session) + except _NotHTTP as exc: + logger.debug( + 'Skipping page %s because it looks like an archive, and cannot ' + 'be checked by HEAD.', link, + ) + except _NotHTML as exc: + logger.debug( + 'Skipping page %s because the %s request got Content-Type: %s', + link, exc.request_desc, exc.content_type, ) - resp.raise_for_status() - - # The check for archives above only works if the url ends with - # something that looks like an archive. However that is not a - # requirement of an url. Unless we issue a HEAD request on every - # url we cannot know ahead of time for sure if something is HTML - # or not. However we can check after we've downloaded it. - content_type = resp.headers.get('Content-Type', 'unknown') - if not content_type.lower().startswith("text/html"): - logger.debug( - 'Skipping page %s because of Content-Type: %s', - link, - content_type, - ) - return - - inst = HTMLPage(resp.content, resp.url, resp.headers) except requests.HTTPError as exc: - _handle_get_page_fail(link, exc, url) + _handle_get_page_fail(link, exc) + except RetryError as exc: + _handle_get_page_fail(link, exc) except SSLError as exc: reason = "There was a problem confirming the ssl certificate: " reason += str(exc) - _handle_get_page_fail(link, reason, url, meth=logger.info) + _handle_get_page_fail(link, reason, meth=logger.info) except requests.ConnectionError as exc: - _handle_get_page_fail(link, "connection error: %s" % exc, url) + _handle_get_page_fail(link, "connection error: %s" % exc) except requests.Timeout: - _handle_get_page_fail(link, "timed out", url) + _handle_get_page_fail(link, "timed out") else: - return inst + return HTMLPage(resp.content, resp.url, resp.headers) + return None class PackageFinder(object): @@ -182,11 +261,21 @@ class PackageFinder(object): packages, by reading pages and looking for appropriate links. """ - def __init__(self, find_links, index_urls, allow_all_prereleases=False, - trusted_hosts=None, process_dependency_links=False, - session=None, format_control=None, platform=None, - versions=None, abi=None, implementation=None, - prefer_binary=False): + def __init__( + self, + find_links, # type: List[str] + index_urls, # type: List[str] + allow_all_prereleases=False, # type: bool + trusted_hosts=None, # type: Optional[Iterable[str]] + session=None, # type: Optional[PipSession] + format_control=None, # type: Optional[FormatControl] + platform=None, # type: Optional[str] + versions=None, # type: Optional[List[str]] + abi=None, # type: Optional[str] + implementation=None, # type: Optional[str] + prefer_binary=False # type: bool + ): + # type: (...) -> None """Create a PackageFinder. :param format_control: A FormatControl object or None. Used to control @@ -215,7 +304,7 @@ class PackageFinder(object): # it and if it exists, use the normalized version. # This is deliberately conservative - it might be fine just to # blindly normalize anything starting with a ~... - self.find_links = [] + self.find_links = [] # type: List[str] for link in find_links: if link.startswith('~'): new_link = normalize_path(link) @@ -224,10 +313,9 @@ class PackageFinder(object): self.find_links.append(link) self.index_urls = index_urls - self.dependency_links = [] # These are boring links that have already been logged somehow: - self.logged_links = set() + self.logged_links = set() # type: Set[Link] self.format_control = format_control or FormatControl(set(), set()) @@ -235,14 +323,11 @@ class PackageFinder(object): self.secure_origins = [ ("*", host, "*") for host in (trusted_hosts if trusted_hosts else []) - ] + ] # type: List[SecureOrigin] # Do we want to allow _all_ pre-releases? self.allow_all_prereleases = allow_all_prereleases - # Do we process dependency links? - self.process_dependency_links = process_dependency_links - # The Session we'll use to make requests self.session = session @@ -274,11 +359,12 @@ class PackageFinder(object): break def get_formatted_locations(self): + # type: () -> str lines = [] if self.index_urls and self.index_urls != [PyPI.simple_url]: lines.append( "Looking in indexes: {}".format(", ".join( - remove_auth_from_url(url) for url in self.index_urls)) + redact_password_from_url(url) for url in self.index_urls)) ) if self.find_links: lines.append( @@ -286,21 +372,6 @@ class PackageFinder(object): ) return "\n".join(lines) - def add_dependency_links(self, links): - # FIXME: this shouldn't be global list this, it should only - # apply to requirements of the package that specifies the - # dependency_links value - # FIXME: also, we should track comes_from (i.e., use Link) - if self.process_dependency_links: - deprecated( - "Dependency Links processing has been deprecated and will be " - "removed in a future release.", - replacement="PEP 508 URL dependencies", - gone_in="18.2", - issue=4187, - ) - self.dependency_links.extend(links) - @staticmethod def get_extras_links(links): requires = [] @@ -316,12 +387,11 @@ class PackageFinder(object): extras[link[1:-1]] = current_list else: current_list.append(link) - return extras - @staticmethod def _sort_locations(locations, expand_dir=False): + # type: (Sequence[str], bool) -> Tuple[List[str], List[str]] """ Sort locations into "files" (archives) and "urls", and return a pair of lists (files,urls) @@ -354,6 +424,11 @@ class PackageFinder(object): sort_path(os.path.join(path, item)) elif is_file_url: urls.append(url) + else: + logger.warning( + "Path '{0}' is ignored: " + "it is a directory.".format(path), + ) elif os.path.isfile(path): sort_path(path) else: @@ -373,6 +448,7 @@ class PackageFinder(object): return files, urls def _candidate_sort_key(self, candidate, ignore_compatibility=True): + # type: (InstallationCandidate, bool) -> CandidateSortingKey """ Function used to generate link sort key for link tuples. The greater the return value, the more preferred it is. @@ -387,7 +463,7 @@ class PackageFinder(object): with the same version, would have to be considered equal """ support_num = len(self.valid_tags) - build_tag = tuple() + build_tag = tuple() # type: BuildTag binary_preference = 0 if candidate.location.is_wheel: # can raise InvalidWheelFilename @@ -404,7 +480,6 @@ class PackageFinder(object): pri = -(wheel.support_index_min(tags=tags)) except TypeError: pri = -(support_num) - if wheel.build_tag is not None: match = re.match(r'^(\d+)(.*)$', wheel.build_tag) build_tag_groups = match.groups() @@ -414,6 +489,7 @@ class PackageFinder(object): return (binary_preference, candidate.version, build_tag, pri) def _validate_secure_origin(self, logger, location): + # type: (Logger, Link) -> bool # Determine if this url used a secure transport mechanism parsed = urllib_parse.urlparse(str(location)) origin = (parsed.scheme, parsed.hostname, parsed.port) @@ -445,7 +521,9 @@ class PackageFinder(object): network = ipaddress.ip_network( secure_origin[1] if isinstance(secure_origin[1], six.text_type) - else secure_origin[1].decode("utf8") + # setting secure_origin[1] to proper Union[bytes, str] + # creates problems in other places + else secure_origin[1].decode("utf8") # type: ignore ) except ValueError: # We don't have both a valid address or a valid network, so @@ -485,6 +563,7 @@ class PackageFinder(object): return False def _get_index_urls_locations(self, project_name): + # type: (str) -> List[str] """Returns the locations found via self.index_urls Checks the url_name on the main (first in the list) index and @@ -507,9 +586,10 @@ class PackageFinder(object): return [mkurl_pypi_url(url) for url in self.index_urls] def find_all_candidates(self, project_name): + # type: (str) -> List[Optional[InstallationCandidate]] """Find all available InstallationCandidate for project_name - This checks index_urls, find_links and dependency_links. + This checks index_urls and find_links. All versions found are returned as an InstallationCandidate list. See _link_package_versions for details on which files are accepted @@ -519,21 +599,18 @@ class PackageFinder(object): fl_file_loc, fl_url_loc = self._sort_locations( self.find_links, expand_dir=True, ) - dep_file_loc, dep_url_loc = self._sort_locations(self.dependency_links) file_locations = (Link(url) for url in itertools.chain( - index_file_loc, fl_file_loc, dep_file_loc, + index_file_loc, fl_file_loc, )) # We trust every url that the user has given us whether it was given - # via --index-url or --find-links - # We explicitly do not trust links that came from dependency_links + # via --index-url or --find-links. # We want to filter out any thing which does not have a secure origin. url_locations = [ link for link in itertools.chain( (Link(url) for url in index_url_loc), (Link(url) for url in fl_url_loc), - (Link(url) for url in dep_url_loc), ) if self._validate_secure_origin(logger, link) ] @@ -564,17 +641,6 @@ class PackageFinder(object): self._package_versions(page.iter_links(), search) ) - dependency_versions = self._package_versions( - (Link(url) for url in self.dependency_links), search - ) - if dependency_versions: - logger.debug( - 'dependency_links found: %s', - ', '.join([ - version.location.url for version in dependency_versions - ]) - ) - file_versions = self._package_versions(file_locations, search) if file_versions: file_versions.sort(reverse=True) @@ -587,12 +653,10 @@ class PackageFinder(object): ) # This is an intentional priority ordering - return ( - file_versions + find_links_versions + page_versions + - dependency_versions - ) + return file_versions + find_links_versions + page_versions def find_requirement(self, req, upgrade, ignore_compatibility=False): + # type: (InstallRequirement, bool, bool) -> Optional[Link] """Try to find a Link matching req Expects req, an InstallRequirement and upgrade, a boolean @@ -692,20 +756,18 @@ class PackageFinder(object): return best_candidate.location def _get_pages(self, locations, project_name): + # type: (Iterable[Link], str) -> Iterable[HTMLPage] """ Yields (page, page_url) from the given locations, skipping locations that have errors. """ - seen = set() + seen = set() # type: Set[Link] for location in locations: if location in seen: continue seen.add(location) - try: - page = self._get_page(location) - except requests.HTTPError: - continue + page = _get_html_page(location, session=self.session) if page is None: continue @@ -714,12 +776,13 @@ class PackageFinder(object): _py_version_re = re.compile(r'-py([123]\.?[0-9]?)$') def _sort_links(self, links): + # type: (Iterable[Link]) -> List[Link] """ Returns elements of links in order, non-egg links first, egg links second, while eliminating duplicates """ eggs, no_eggs = [], [] - seen = set() + seen = set() # type: Set[Link] for link in links: if link not in seen: seen.add(link) @@ -729,7 +792,12 @@ class PackageFinder(object): no_eggs.append(link) return no_eggs + eggs - def _package_versions(self, links, search): + def _package_versions( + self, + links, # type: Iterable[Link] + search # type: Search + ): + # type: (...) -> List[Optional[InstallationCandidate]] result = [] for link in self._sort_links(links): v = self._link_package_versions(link, search) @@ -738,11 +806,13 @@ class PackageFinder(object): return result def _log_skipped_link(self, link, reason): + # type: (Link, str) -> None if link not in self.logged_links: logger.debug('Skipping link %s; %s', link, reason) self.logged_links.add(link) def _link_package_versions(self, link, search, ignore_compatibility=True): + # type: (Link, Search, bool) -> Optional[InstallationCandidate] """Return an InstallationCandidate or None""" version = None if link.egg_fragment: @@ -752,51 +822,51 @@ class PackageFinder(object): egg_info, ext = link.splitext() if not ext: self._log_skipped_link(link, 'not a file') - return + return None if ext not in SUPPORTED_EXTENSIONS: self._log_skipped_link( link, 'unsupported archive format: %s' % ext, ) - return - if "binary" not in search.formats and ext == wheel_ext and not ignore_compatibility: + return None + if "binary" not in search.formats and ext == WHEEL_EXTENSION and not ignore_compatibility: self._log_skipped_link( link, 'No binaries permitted for %s' % search.supplied, ) - return + return None if "macosx10" in link.path and ext == '.zip' and not ignore_compatibility: self._log_skipped_link(link, 'macosx10 one') - return - if ext == wheel_ext: + return None + if ext == WHEEL_EXTENSION: try: wheel = Wheel(link.filename) except InvalidWheelFilename: self._log_skipped_link(link, 'invalid wheel filename') - return + return None if canonicalize_name(wheel.name) != search.canonical: self._log_skipped_link( link, 'wrong project name (not %s)' % search.supplied) - return + return None if not wheel.supported(self.valid_tags) and not ignore_compatibility: self._log_skipped_link( link, 'it is not compatible with this Python') - return + return None version = wheel.version # This should be up by the search.ok_binary check, but see issue 2700. - if "source" not in search.formats and ext != wheel_ext: + if "source" not in search.formats and ext != WHEEL_EXTENSION: self._log_skipped_link( link, 'No sources permitted for %s' % search.supplied, ) - return + return None if not version: - version = egg_info_matches(egg_info, search.supplied, link) - if version is None: + version = _egg_info_matches(egg_info, search.canonical) + if not version: self._log_skipped_link( link, 'Missing project version for %s' % search.supplied) - return + return None match = self._py_version_re.search(version) if match: @@ -805,7 +875,7 @@ class PackageFinder(object): if py_version != sys.version[:3]: self._log_skipped_link( link, 'Python version is incorrect') - return + return None try: support_this_python = check_requires_python(link.requires_python) except specifiers.InvalidSpecifier: @@ -814,45 +884,57 @@ class PackageFinder(object): support_this_python = True if not support_this_python and not ignore_compatibility: - logger.debug("The package %s is incompatible with the python" - "version in use. Acceptable python versions are:%s", + logger.debug("The package %s is incompatible with the python " + "version in use. Acceptable python versions are: %s", link, link.requires_python) - return + return None logger.debug('Found link %s, version: %s', link, version) return InstallationCandidate(search.supplied, version, link, link.requires_python) - def _get_page(self, link): - return _get_html_page(link, session=self.session) + +def _find_name_version_sep(egg_info, canonical_name): + # type: (str, str) -> int + """Find the separator's index based on the package's canonical name. + + `egg_info` must be an egg info string for the given package, and + `canonical_name` must be the package's canonical name. + + This function is needed since the canonicalized name does not necessarily + have the same length as the egg info's name part. An example:: + + >>> egg_info = 'foo__bar-1.0' + >>> canonical_name = 'foo-bar' + >>> _find_name_version_sep(egg_info, canonical_name) + 8 + """ + # Project name and version must be separated by one single dash. Find all + # occurrences of dashes; if the string in front of it matches the canonical + # name, this is the one separating the name and version parts. + for i, c in enumerate(egg_info): + if c != "-": + continue + if canonicalize_name(egg_info[:i]) == canonical_name: + return i + raise ValueError("{} does not match {}".format(egg_info, canonical_name)) -def egg_info_matches( - egg_info, search_name, link, - _egg_info_re=re.compile(r'([a-z0-9_.]+)-([a-z0-9_.!+-]+)', re.I)): +def _egg_info_matches(egg_info, canonical_name): + # type: (str, str) -> Optional[str] """Pull the version part out of a string. :param egg_info: The string to parse. E.g. foo-2.1 - :param search_name: The name of the package this belongs to. None to - infer the name. Note that this cannot unambiguously parse strings - like foo-2-2 which might be foo, 2-2 or foo-2, 2. - :param link: The link the string came from, for logging on failure. + :param canonical_name: The canonicalized name of the package this + belongs to. """ - match = _egg_info_re.search(egg_info) - if not match: - logger.debug('Could not parse version from link: %s', link) + try: + version_start = _find_name_version_sep(egg_info, canonical_name) + 1 + except ValueError: return None - if search_name is None: - full_match = match.group(0) - return full_match.split('-', 1)[-1] - name = match.group(0).lower() - # To match the "safe" name that pkg_resources creates: - name = name.replace('_', '-') - # project name and version must be separated by a dash - look_for = search_name.lower() + "-" - if name.startswith(look_for): - return match.group(0)[len(look_for):] - else: + version = egg_info[version_start:] + if not version: return None + return version def _determine_base_url(document, page_url): @@ -888,6 +970,7 @@ _CLEAN_LINK_RE = re.compile(r'[^a-z0-9$&+,/:;=?@.#%_\\|-]', re.I) def _clean_link(url): + # type: (str) -> str """Makes sure a link is fully encoded. That is, if a ' ' shows up in the link, it will be rewritten to %20 (while not over-quoting % or other characters).""" @@ -898,14 +981,16 @@ class HTMLPage(object): """Represents one page, along with its URL""" def __init__(self, content, url, headers=None): + # type: (bytes, str, MutableMapping[str, str]) -> None self.content = content self.url = url self.headers = headers def __str__(self): - return self.url + return redact_password_from_url(self.url) def iter_links(self): + # type: () -> Iterable[Link] """Yields all links in the page""" document = html5lib.parse( self.content, diff --git a/pipenv/patched/notpip/_internal/locations.py b/pipenv/patched/notpip/_internal/locations.py index 3c7d5bd8..e8f5a268 100644 --- a/pipenv/patched/notpip/_internal/locations.py +++ b/pipenv/patched/notpip/_internal/locations.py @@ -12,6 +12,11 @@ from distutils.command.install import SCHEME_KEYS # type: ignore from pipenv.patched.notpip._internal.utils import appdirs from pipenv.patched.notpip._internal.utils.compat import WINDOWS, expanduser +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Any, Union, Dict, List, Optional # noqa: F401 + # Application Directories USER_CACHE_DIR = appdirs.user_cache_dir("pip") @@ -28,6 +33,7 @@ PIP_DELETE_MARKER_FILENAME = 'pip-delete-this-directory.txt' def write_delete_marker_file(directory): + # type: (str) -> None """ Write the pip delete marker file into this directory. """ @@ -37,6 +43,7 @@ def write_delete_marker_file(directory): def running_under_virtualenv(): + # type: () -> bool """ Return True if we're running inside a virtualenv, False otherwise. @@ -50,6 +57,7 @@ def running_under_virtualenv(): def virtualenv_no_global(): + # type: () -> bool """ Return True if in a venv and no system site packages. """ @@ -59,6 +67,8 @@ def virtualenv_no_global(): no_global_file = os.path.join(site_mod_dir, 'no-global-site-packages.txt') if running_under_virtualenv() and os.path.isfile(no_global_file): return True + else: + return False if running_under_virtualenv(): @@ -80,7 +90,8 @@ src_prefix = os.path.abspath(src_prefix) # FIXME doesn't account for venv linked to global site-packages -site_packages = sysconfig.get_path("purelib") +site_packages = sysconfig.get_path("purelib") # type: Optional[str] + # This is because of a bug in PyPy's sysconfig module, see # https://bitbucket.org/pypy/pypy/issues/2506/sysconfig-returns-incorrect-paths # for more information. @@ -135,6 +146,7 @@ new_config_file = os.path.join(appdirs.user_config_dir("pip"), config_basename) def distutils_scheme(dist_name, user=False, home=None, root=None, isolated=False, prefix=None): + # type:(str, bool, str, str, bool, str) -> dict """ Return a distutils install scheme """ @@ -146,12 +158,15 @@ def distutils_scheme(dist_name, user=False, home=None, root=None, extra_dist_args = {"script_args": ["--no-user-cfg"]} else: extra_dist_args = {} - dist_args = {'name': dist_name} + dist_args = {'name': dist_name} # type: Dict[str, Union[str, List[str]]] dist_args.update(extra_dist_args) d = Distribution(dist_args) + # Ignoring, typeshed issue reported python/typeshed/issues/2567 d.parse_config_files() - i = d.get_command_obj('install', create=True) + # NOTE: Ignoring type since mypy can't find attributes on 'Command' + i = d.get_command_obj('install', create=True) # type: Any + assert i is not None # NOTE: setting user or home has the side-effect of creating the home dir # or user base for installations during finalize_options() # ideally, we'd prefer a scheme class that has no side-effects. @@ -171,7 +186,9 @@ def distutils_scheme(dist_name, user=False, home=None, root=None, # platlib). Note, i.install_lib is *always* set after # finalize_options(); we only want to override here if the user # has explicitly requested it hence going back to the config - if 'install_lib' in d.get_option_dict('install'): + + # Ignoring, typeshed issue reported python/typeshed/issues/2567 + if 'install_lib' in d.get_option_dict('install'): # type: ignore scheme.update(dict(purelib=i.install_lib, platlib=i.install_lib)) if running_under_virtualenv(): diff --git a/pipenv/patched/notpip/_internal/models/candidate.py b/pipenv/patched/notpip/_internal/models/candidate.py index 9627589e..adc3550e 100644 --- a/pipenv/patched/notpip/_internal/models/candidate.py +++ b/pipenv/patched/notpip/_internal/models/candidate.py @@ -1,6 +1,12 @@ from pipenv.patched.notpip._vendor.packaging.version import parse as parse_version from pipenv.patched.notpip._internal.utils.models import KeyBasedCompareMixin +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from pipenv.patched.notpip._vendor.packaging.version import _BaseVersion # noqa: F401 + from pipenv.patched.notpip._internal.models.link import Link # noqa: F401 + from typing import Any, Union # noqa: F401 class InstallationCandidate(KeyBasedCompareMixin): @@ -8,8 +14,9 @@ class InstallationCandidate(KeyBasedCompareMixin): """ def __init__(self, project, version, location, requires_python=None): + # type: (Any, str, Link, Any) -> None self.project = project - self.version = parse_version(version) + self.version = parse_version(version) # type: _BaseVersion self.location = location self.requires_python = requires_python @@ -19,6 +26,7 @@ class InstallationCandidate(KeyBasedCompareMixin): ) def __repr__(self): + # type: () -> str return "".format( self.project, self.version, self.location, ) diff --git a/pipenv/patched/notpip/_internal/models/format_control.py b/pipenv/patched/notpip/_internal/models/format_control.py index caad3cba..7172ad9f 100644 --- a/pipenv/patched/notpip/_internal/models/format_control.py +++ b/pipenv/patched/notpip/_internal/models/format_control.py @@ -1,16 +1,24 @@ from pipenv.patched.notpip._vendor.packaging.utils import canonicalize_name +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Optional, Set, FrozenSet # noqa: F401 + class FormatControl(object): - """A helper class for controlling formats from which packages are installed. - If a field is falsy, it isn't set. If it is {':all:'}, it should match all - packages except those listed in the other field. Only one field can be set - to {':all:'} at a time. The rest of the time exact package name matches - are listed, with any given package only showing up in one field at a time. + """Helper for managing formats from which a package can be installed. """ + def __init__(self, no_binary=None, only_binary=None): - self.no_binary = set() if no_binary is None else no_binary - self.only_binary = set() if only_binary is None else only_binary + # type: (Optional[Set], Optional[Set]) -> None + if no_binary is None: + no_binary = set() + if only_binary is None: + only_binary = set() + + self.no_binary = no_binary + self.only_binary = only_binary def __eq__(self, other): return self.__dict__ == other.__dict__ @@ -27,6 +35,7 @@ class FormatControl(object): @staticmethod def handle_mutual_excludes(value, target, other): + # type: (str, Optional[Set], Optional[Set]) -> None new = value.split(',') while ':all:' in new: other.clear() @@ -45,6 +54,7 @@ class FormatControl(object): target.add(name) def get_allowed_formats(self, canonical_name): + # type: (str) -> FrozenSet result = {"binary", "source"} if canonical_name in self.only_binary: result.discard('source') @@ -57,6 +67,7 @@ class FormatControl(object): return frozenset(result) def disallow_binaries(self): + # type: () -> None self.handle_mutual_excludes( ':all:', self.no_binary, self.only_binary, ) diff --git a/pipenv/patched/notpip/_internal/models/index.py b/pipenv/patched/notpip/_internal/models/index.py index 0983fc9c..b2895dab 100644 --- a/pipenv/patched/notpip/_internal/models/index.py +++ b/pipenv/patched/notpip/_internal/models/index.py @@ -6,6 +6,7 @@ class PackageIndex(object): """ def __init__(self, url, file_storage_domain): + # type: (str, str) -> None super(PackageIndex, self).__init__() self.url = url self.netloc = urllib_parse.urlsplit(url).netloc @@ -18,6 +19,7 @@ class PackageIndex(object): self.file_storage_domain = file_storage_domain def _url_for_path(self, path): + # type: (str) -> str return urllib_parse.urljoin(self.url, path) diff --git a/pipenv/patched/notpip/_internal/models/link.py b/pipenv/patched/notpip/_internal/models/link.py index 686af1d0..ded4de43 100644 --- a/pipenv/patched/notpip/_internal/models/link.py +++ b/pipenv/patched/notpip/_internal/models/link.py @@ -4,9 +4,15 @@ import re from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse from pipenv.patched.notpip._internal.download import path_to_url -from pipenv.patched.notpip._internal.utils.misc import splitext +from pipenv.patched.notpip._internal.utils.misc import ( + WHEEL_EXTENSION, redact_password_from_url, splitext, +) from pipenv.patched.notpip._internal.utils.models import KeyBasedCompareMixin -from pipenv.patched.notpip._internal.wheel import wheel_ext +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Optional, Tuple, Union, Text # noqa: F401 + from pipenv.patched.notpip._internal.index import HTMLPage # noqa: F401 class Link(KeyBasedCompareMixin): @@ -14,6 +20,7 @@ class Link(KeyBasedCompareMixin): """ def __init__(self, url, comes_from=None, requires_python=None): + # type: (str, Optional[Union[str, HTMLPage]], Optional[str]) -> None """ url: url of the resource pointed to (href of the link) @@ -44,15 +51,17 @@ class Link(KeyBasedCompareMixin): else: rp = '' if self.comes_from: - return '%s (from %s)%s' % (self.url, self.comes_from, rp) + return '%s (from %s)%s' % (redact_password_from_url(self.url), + self.comes_from, rp) else: - return str(self.url) + return redact_password_from_url(str(self.url)) def __repr__(self): return '' % self @property def filename(self): + # type: () -> str _, netloc, path, _, _ = urllib_parse.urlsplit(self.url) name = posixpath.basename(path.rstrip('/')) or netloc name = urllib_parse.unquote(name) @@ -61,25 +70,31 @@ class Link(KeyBasedCompareMixin): @property def scheme(self): + # type: () -> str return urllib_parse.urlsplit(self.url)[0] @property def netloc(self): + # type: () -> str return urllib_parse.urlsplit(self.url)[1] @property def path(self): + # type: () -> str return urllib_parse.unquote(urllib_parse.urlsplit(self.url)[2]) def splitext(self): + # type: () -> Tuple[str, str] return splitext(posixpath.basename(self.path.rstrip('/'))) @property def ext(self): + # type: () -> str return self.splitext()[1] @property def url_without_fragment(self): + # type: () -> str scheme, netloc, path, query, fragment = urllib_parse.urlsplit(self.url) return urllib_parse.urlunsplit((scheme, netloc, path, query, None)) @@ -87,6 +102,7 @@ class Link(KeyBasedCompareMixin): @property def egg_fragment(self): + # type: () -> Optional[str] match = self._egg_fragment_re.search(self.url) if not match: return None @@ -96,6 +112,7 @@ class Link(KeyBasedCompareMixin): @property def subdirectory_fragment(self): + # type: () -> Optional[str] match = self._subdirectory_fragment_re.search(self.url) if not match: return None @@ -107,6 +124,7 @@ class Link(KeyBasedCompareMixin): @property def hash(self): + # type: () -> Optional[str] match = self._hash_re.search(self.url) if match: return match.group(2) @@ -114,6 +132,7 @@ class Link(KeyBasedCompareMixin): @property def hash_name(self): + # type: () -> Optional[str] match = self._hash_re.search(self.url) if match: return match.group(1) @@ -121,14 +140,17 @@ class Link(KeyBasedCompareMixin): @property def show_url(self): + # type: () -> Optional[str] return posixpath.basename(self.url.split('#', 1)[0].split('?', 1)[0]) @property def is_wheel(self): - return self.ext == wheel_ext + # type: () -> bool + return self.ext == WHEEL_EXTENSION @property def is_artifact(self): + # type: () -> bool """ Determines if this points to an actual artifact (e.g. a tarball) or if it points to an "abstract" thing like a path or a VCS location. diff --git a/pipenv/patched/notpip/_internal/operations/check.py b/pipenv/patched/notpip/_internal/operations/check.py index 9c8ea08e..a73611d4 100644 --- a/pipenv/patched/notpip/_internal/operations/check.py +++ b/pipenv/patched/notpip/_internal/operations/check.py @@ -1,18 +1,22 @@ """Validation of dependencies of packages """ +import logging from collections import namedtuple from pipenv.patched.notpip._vendor.packaging.utils import canonicalize_name +from pipenv.patched.notpip._vendor.pkg_resources import RequirementParseError from pipenv.patched.notpip._internal.operations.prepare import make_abstract_dist from pipenv.patched.notpip._internal.utils.misc import get_installed_distributions from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING +logger = logging.getLogger(__name__) + if MYPY_CHECK_RUNNING: from pipenv.patched.notpip._internal.req.req_install import InstallRequirement # noqa: F401 from typing import ( # noqa: F401 - Any, Callable, Dict, Iterator, Optional, Set, Tuple, List + Any, Callable, Dict, Optional, Set, Tuple, List ) # Shorthands @@ -28,7 +32,7 @@ PackageDetails = namedtuple('PackageDetails', ['version', 'requires']) def create_package_set_from_installed(**kwargs): - # type: (**Any) -> PackageSet + # type: (**Any) -> Tuple[PackageSet, bool] """Converts a list of distributions into a PackageSet. """ # Default to using all packages installed on the system @@ -36,10 +40,16 @@ def create_package_set_from_installed(**kwargs): kwargs = {"local_only": False, "skip": ()} package_set = {} + problems = False for dist in get_installed_distributions(**kwargs): name = canonicalize_name(dist.project_name) - package_set[name] = PackageDetails(dist.version, dist.requires()) - return package_set + try: + package_set[name] = PackageDetails(dist.version, dist.requires()) + except RequirementParseError as e: + # Don't crash on broken metadata + logging.warning("Error parsing requirements for %s: %s", name, e) + problems = True + return package_set, problems def check_package_set(package_set, should_ignore=None): @@ -95,7 +105,7 @@ def check_install_conflicts(to_install): installing given requirements """ # Start from the current state - package_set = create_package_set_from_installed() + package_set, _ = create_package_set_from_installed() # Install packages would_be_installed = _simulate_installation_of(to_install, package_set) @@ -110,9 +120,6 @@ def check_install_conflicts(to_install): ) -# NOTE from @pradyunsg -# This required a minor update in dependency link handling logic over at -# operations.prepare.IsSDist.dist() to get it working def _simulate_installation_of(to_install, package_set): # type: (List[InstallRequirement], PackageSet) -> Set[str] """Computes the version of packages after installing to_install. @@ -123,7 +130,7 @@ def _simulate_installation_of(to_install, package_set): # Modify it as installing requirement_set would (assuming no errors) for inst_req in to_install: - dist = make_abstract_dist(inst_req).dist(finder=None) + dist = make_abstract_dist(inst_req).dist() name = canonicalize_name(dist.key) package_set[name] = PackageDetails(dist.version, dist.requires()) diff --git a/pipenv/patched/notpip/_internal/operations/freeze.py b/pipenv/patched/notpip/_internal/operations/freeze.py index b18b98e4..8fd755e8 100644 --- a/pipenv/patched/notpip/_internal/operations/freeze.py +++ b/pipenv/patched/notpip/_internal/operations/freeze.py @@ -5,57 +5,61 @@ import logging import os import re -from pipenv.patched.notpip._vendor import pkg_resources, six +from pipenv.patched.notpip._vendor import six from pipenv.patched.notpip._vendor.packaging.utils import canonicalize_name from pipenv.patched.notpip._vendor.pkg_resources import RequirementParseError -from pipenv.patched.notpip._internal.exceptions import InstallationError +from pipenv.patched.notpip._internal.exceptions import BadCommand, InstallationError from pipenv.patched.notpip._internal.req.constructors import ( install_req_from_editable, install_req_from_line, ) from pipenv.patched.notpip._internal.req.req_file import COMMENT_RE -from pipenv.patched.notpip._internal.utils.deprecation import deprecated from pipenv.patched.notpip._internal.utils.misc import ( - dist_is_editable, get_installed_distributions, make_vcs_requirement_url, + dist_is_editable, get_installed_distributions, ) +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import ( # noqa: F401 + Iterator, Optional, List, Container, Set, Dict, Tuple, Iterable, Union + ) + from pipenv.patched.notpip._internal.cache import WheelCache # noqa: F401 + from pipenv.patched.notpip._vendor.pkg_resources import ( # noqa: F401 + Distribution, Requirement + ) + + RequirementInfo = Tuple[Optional[Union[str, Requirement]], bool, List[str]] + logger = logging.getLogger(__name__) def freeze( - requirement=None, - find_links=None, local_only=None, user_only=None, skip_regex=None, - isolated=False, - wheel_cache=None, - exclude_editable=False, - skip=()): + requirement=None, # type: Optional[List[str]] + find_links=None, # type: Optional[List[str]] + local_only=None, # type: Optional[bool] + user_only=None, # type: Optional[bool] + skip_regex=None, # type: Optional[str] + isolated=False, # type: bool + wheel_cache=None, # type: Optional[WheelCache] + exclude_editable=False, # type: bool + skip=() # type: Container[str] +): + # type: (...) -> Iterator[str] find_links = find_links or [] skip_match = None if skip_regex: skip_match = re.compile(skip_regex).search - dependency_links = [] - - for dist in pkg_resources.working_set: - if dist.has_metadata('dependency_links.txt'): - dependency_links.extend( - dist.get_metadata_lines('dependency_links.txt') - ) - for link in find_links: - if '#egg=' in link: - dependency_links.append(link) for link in find_links: yield '-f %s' % link - installations = {} + installations = {} # type: Dict[str, FrozenRequirement] for dist in get_installed_distributions(local_only=local_only, skip=(), user_only=user_only): try: - req = FrozenRequirement.from_dist( - dist, - dependency_links - ) + req = FrozenRequirement.from_dist(dist) except RequirementParseError: logger.warning( "Could not parse requirement: %s", @@ -71,10 +75,10 @@ def freeze( # should only be emitted once, even if the same option is in multiple # requirements files, so we need to keep track of what has been emitted # so that we don't emit it again if it's seen again - emitted_options = set() + emitted_options = set() # type: Set[str] # keep track of which files a requirement is in so that we can # give an accurate warning if a requirement appears multiple times. - req_files = collections.defaultdict(list) + req_files = collections.defaultdict(list) # type: Dict[str, List[str]] for req_file_path in requirement: with open(req_file_path) as req_file: for line in req_file: @@ -128,10 +132,10 @@ def freeze( # but has been processed already if not req_files[line_req.name]: logger.warning( - "Requirement file [%s] contains %s, but that " - "package is not installed", + "Requirement file [%s] contains %s, but " + "package %r is not installed", req_file_path, - COMMENT_RE.sub('', line).strip(), + COMMENT_RE.sub('', line).strip(), line_req.name ) else: req_files[line_req.name].append(req_file_path) @@ -157,105 +161,84 @@ def freeze( yield str(installation).rstrip() +def get_requirement_info(dist): + # type: (Distribution) -> RequirementInfo + """ + Compute and return values (req, editable, comments) for use in + FrozenRequirement.from_dist(). + """ + if not dist_is_editable(dist): + return (None, False, []) + + location = os.path.normcase(os.path.abspath(dist.location)) + + from pipenv.patched.notpip._internal.vcs import vcs, RemoteNotFoundError + vc_type = vcs.get_backend_type(location) + + if not vc_type: + req = dist.as_requirement() + logger.debug( + 'No VCS found for editable requirement {!r} in: {!r}', req, + location, + ) + comments = [ + '# Editable install with no version control ({})'.format(req) + ] + return (location, True, comments) + + try: + req = vc_type.get_src_requirement(location, dist.project_name) + except RemoteNotFoundError: + req = dist.as_requirement() + comments = [ + '# Editable {} install with no remote ({})'.format( + vc_type.__name__, req, + ) + ] + return (location, True, comments) + + except BadCommand: + logger.warning( + 'cannot determine version of editable source in %s ' + '(%s command not found in path)', + location, + vc_type.name, + ) + return (None, True, []) + + except InstallationError as exc: + logger.warning( + "Error when trying to get requirement for VCS system %s, " + "falling back to uneditable format", exc + ) + else: + if req is not None: + return (req, True, []) + + logger.warning( + 'Could not determine repository location of %s', location + ) + comments = ['## !! Could not determine repository location'] + + return (None, False, comments) + + class FrozenRequirement(object): def __init__(self, name, req, editable, comments=()): + # type: (str, Union[str, Requirement], bool, Iterable[str]) -> None self.name = name self.req = req self.editable = editable self.comments = comments - _rev_re = re.compile(r'-r(\d+)$') - _date_re = re.compile(r'-(20\d\d\d\d\d\d)$') - @classmethod - def _init_args_from_dist(cls, dist, dependency_links): - """ - Compute and return arguments (req, editable, comments) to pass to - FrozenRequirement.__init__(). - - This method is for use in FrozenRequirement.from_dist(). - """ - location = os.path.normcase(os.path.abspath(dist.location)) - comments = [] - from pipenv.patched.notpip._internal.vcs import vcs, get_src_requirement - if dist_is_editable(dist) and vcs.get_backend_name(location): - editable = True - try: - req = get_src_requirement(dist, location) - except InstallationError as exc: - logger.warning( - "Error when trying to get requirement for VCS system %s, " - "falling back to uneditable format", exc - ) - req = None - if req is None: - logger.warning( - 'Could not determine repository location of %s', location - ) - comments.append( - '## !! Could not determine repository location' - ) - req = dist.as_requirement() - editable = False - else: - editable = False + def from_dist(cls, dist): + # type: (Distribution) -> FrozenRequirement + req, editable, comments = get_requirement_info(dist) + if req is None: req = dist.as_requirement() - specs = req.specs - assert len(specs) == 1 and specs[0][0] in ["==", "==="], \ - 'Expected 1 spec with == or ===; specs = %r; dist = %r' % \ - (specs, dist) - version = specs[0][1] - ver_match = cls._rev_re.search(version) - date_match = cls._date_re.search(version) - if ver_match or date_match: - svn_backend = vcs.get_backend('svn') - if svn_backend: - svn_location = svn_backend().get_location( - dist, - dependency_links, - ) - if not svn_location: - logger.warning( - 'Warning: cannot find svn location for %s', req, - ) - comments.append( - '## FIXME: could not find svn URL in dependency_links ' - 'for this package:' - ) - else: - deprecated( - "SVN editable detection based on dependency links " - "will be dropped in the future.", - replacement=None, - gone_in="18.2", - issue=4187, - ) - comments.append( - '# Installing as editable to satisfy requirement %s:' % - req - ) - if ver_match: - rev = ver_match.group(1) - else: - rev = '{%s}' % date_match.group(1) - editable = True - egg_name = cls.egg_name(dist) - req = make_vcs_requirement_url(svn_location, rev, egg_name) - return (req, editable, comments) - - @classmethod - def from_dist(cls, dist, dependency_links): - args = cls._init_args_from_dist(dist, dependency_links) - return cls(dist.project_name, *args) - - @staticmethod - def egg_name(dist): - name = dist.egg_name() - match = re.search(r'-py\d\.\d$', name) - if match: - name = name[:match.start()] - return name + return cls(dist.project_name, req, editable, comments=comments) def __str__(self): req = self.req diff --git a/pipenv/patched/notpip/_internal/operations/prepare.py b/pipenv/patched/notpip/_internal/operations/prepare.py index d61270b4..018fca97 100644 --- a/pipenv/patched/notpip/_internal/operations/prepare.py +++ b/pipenv/patched/notpip/_internal/operations/prepare.py @@ -18,12 +18,21 @@ from pipenv.patched.notpip._internal.utils.compat import expanduser from pipenv.patched.notpip._internal.utils.hashes import MissingHashes from pipenv.patched.notpip._internal.utils.logging import indent_log from pipenv.patched.notpip._internal.utils.misc import display_path, normalize_path, rmtree +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING from pipenv.patched.notpip._internal.vcs import vcs +if MYPY_CHECK_RUNNING: + from typing import Any, Optional # noqa: F401 + from pipenv.patched.notpip._internal.req.req_install import InstallRequirement # noqa: F401 + from pipenv.patched.notpip._internal.index import PackageFinder # noqa: F401 + from pipenv.patched.notpip._internal.download import PipSession # noqa: F401 + from pipenv.patched.notpip._internal.req.req_tracker import RequirementTracker # noqa: F401 + logger = logging.getLogger(__name__) def make_abstract_dist(req): + # type: (InstallRequirement) -> DistAbstraction """Factory to make an abstract dist object. Preconditions: Either an editable req with a source_dir, or satisfied_by or @@ -59,40 +68,40 @@ class DistAbstraction(object): """ def __init__(self, req): - self.req = req + # type: (InstallRequirement) -> None + self.req = req # type: InstallRequirement - def dist(self, finder): + def dist(self): + # type: () -> Any """Return a setuptools Dist object.""" - raise NotImplementedError(self.dist) + raise NotImplementedError def prep_for_dist(self, finder, build_isolation): + # type: (PackageFinder, bool) -> Any """Ensure that we can get a Dist for this requirement.""" - raise NotImplementedError(self.dist) + raise NotImplementedError class IsWheel(DistAbstraction): - def dist(self, finder): + def dist(self): + # type: () -> pkg_resources.Distribution return list(pkg_resources.find_distributions( self.req.source_dir))[0] def prep_for_dist(self, finder, build_isolation): + # type: (PackageFinder, bool) -> Any # FIXME:https://github.com/pypa/pip/issues/1112 pass class IsSDist(DistAbstraction): - def dist(self, finder): - dist = self.req.get_dist() - # FIXME: shouldn't be globally added. - if finder and dist.has_metadata('dependency_links.txt'): - finder.add_dependency_links( - dist.get_metadata_lines('dependency_links.txt') - ) - return dist + def dist(self): + return self.req.get_dist() def prep_for_dist(self, finder, build_isolation): + # type: (PackageFinder, bool) -> None # Prepare for building. We need to: # 1. Load pyproject.toml (if it exists) # 2. Set up the build environment @@ -100,43 +109,64 @@ class IsSDist(DistAbstraction): self.req.load_pyproject_toml() should_isolate = self.req.use_pep517 and build_isolation + def _raise_conflicts(conflicting_with, conflicting_reqs): + raise InstallationError( + "Some build dependencies for %s conflict with %s: %s." % ( + self.req, conflicting_with, ', '.join( + '%s is incompatible with %s' % (installed, wanted) + for installed, wanted in sorted(conflicting)))) + if should_isolate: # Isolate in a BuildEnvironment and install the build-time # requirements. self.req.build_env = BuildEnvironment() self.req.build_env.install_requirements( - finder, self.req.pyproject_requires, + finder, self.req.pyproject_requires, 'overlay', "Installing build dependencies" ) - missing = [] - if self.req.requirements_to_check: - check = self.req.requirements_to_check - missing = self.req.build_env.missing_requirements(check) + conflicting, missing = self.req.build_env.check_requirements( + self.req.requirements_to_check + ) + if conflicting: + _raise_conflicts("PEP 517/518 supported requirements", + conflicting) if missing: logger.warning( "Missing build requirements in pyproject.toml for %s.", self.req, ) logger.warning( - "The project does not specify a build backend, and pip " - "cannot fall back to setuptools without %s.", + "The project does not specify a build backend, and " + "pip cannot fall back to setuptools without %s.", " and ".join(map(repr, sorted(missing))) ) + # Install any extra build dependencies that the backend requests. + # This must be done in a second pass, as the pyproject.toml + # dependencies must be installed before we can call the backend. + with self.req.build_env: + # We need to have the env active when calling the hook. + self.req.spin_message = "Getting requirements to build wheel" + reqs = self.req.pep517_backend.get_requires_for_build_wheel() + conflicting, missing = self.req.build_env.check_requirements(reqs) + if conflicting: + _raise_conflicts("the backend dependencies", conflicting) + self.req.build_env.install_requirements( + finder, missing, 'normal', + "Installing backend dependencies" + ) - try: - self.req.run_egg_info() - except (OSError, TypeError): - self.req._correct_build_location() - self.req.run_egg_info() + self.req.prepare_metadata() self.req.assert_source_matches_version() class Installed(DistAbstraction): - def dist(self, finder): + def dist(self): + # type: () -> pkg_resources.Distribution return self.req.satisfied_by def prep_for_dist(self, finder, build_isolation): + # type: (PackageFinder, bool) -> Any pass @@ -144,8 +174,17 @@ class RequirementPreparer(object): """Prepares a Requirement """ - def __init__(self, build_dir, download_dir, src_dir, wheel_download_dir, - progress_bar, build_isolation, req_tracker): + def __init__( + self, + build_dir, # type: str + download_dir, # type: Optional[str] + src_dir, # type: str + wheel_download_dir, # type: Optional[str] + progress_bar, # type: str + build_isolation, # type: bool + req_tracker # type: RequirementTracker + ): + # type: (...) -> None super(RequirementPreparer, self).__init__() self.src_dir = src_dir @@ -175,6 +214,7 @@ class RequirementPreparer(object): @property def _download_should_save(self): + # type: () -> bool # TODO: Modify to reduce indentation needed if self.download_dir: self.download_dir = expanduser(self.download_dir) @@ -187,8 +227,15 @@ class RequirementPreparer(object): % display_path(self.download_dir)) return False - def prepare_linked_requirement(self, req, session, finder, - upgrade_allowed, require_hashes): + def prepare_linked_requirement( + self, + req, # type: InstallRequirement + session, # type: PipSession + finder, # type: PackageFinder + upgrade_allowed, # type: bool + require_hashes # type: bool + ): + # type: (...) -> DistAbstraction """Prepare a requirement that would be obtained from req.link """ # TODO: Breakup into smaller functions @@ -209,6 +256,7 @@ class RequirementPreparer(object): # installation. # FIXME: this won't upgrade when there's an existing # package unpacked in `req.source_dir` + # package unpacked in `req.source_dir` if os.path.exists(os.path.join(req.source_dir, 'setup.py')): rmtree(req.source_dir) req.populate_link(finder, upgrade_allowed, require_hashes) @@ -298,8 +346,14 @@ class RequirementPreparer(object): req.archive(self.download_dir) return abstract_dist - def prepare_editable_requirement(self, req, require_hashes, use_user_site, - finder): + def prepare_editable_requirement( + self, + req, # type: InstallRequirement + require_hashes, # type: bool + use_user_site, # type: bool + finder # type: PackageFinder + ): + # type: (...) -> DistAbstraction """Prepare an editable requirement """ assert req.editable, "cannot prepare a non-editable req as editable" @@ -327,6 +381,7 @@ class RequirementPreparer(object): return abstract_dist def prepare_installed_requirement(self, req, require_hashes, skip_reason): + # type: (InstallRequirement, bool, Optional[str]) -> DistAbstraction """Prepare an already-installed requirement """ assert req.satisfied_by, "req should have been satisfied but isn't" diff --git a/pipenv/patched/notpip/_internal/pep425tags.py b/pipenv/patched/notpip/_internal/pep425tags.py index 182c1c88..4b6eb2bc 100644 --- a/pipenv/patched/notpip/_internal/pep425tags.py +++ b/pipenv/patched/notpip/_internal/pep425tags.py @@ -14,8 +14,15 @@ try: import pipenv.patched.notpip._internal.utils.glibc except ImportError: import pipenv.patched.notpip.utils.glibc - from pipenv.patched.notpip._internal.utils.compat import get_extension_suffixes +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import ( # noqa: F401 + Tuple, Callable, List, Optional, Union, Dict + ) + + Pep425Tag = Tuple[str, str, str] logger = logging.getLogger(__name__) @@ -23,6 +30,7 @@ _osx_arch_pat = re.compile(r'(.+)_(\d+)_(\d+)_(.+)') def get_config_var(var): + # type: (str) -> Optional[str] try: return sysconfig.get_config_var(var) except IOError as e: # Issue #1074 @@ -31,6 +39,7 @@ def get_config_var(var): def get_abbr_impl(): + # type: () -> str """Return abbreviated implementation name.""" if hasattr(sys, 'pypy_version_info'): pyimpl = 'pp' @@ -44,6 +53,7 @@ def get_abbr_impl(): def get_impl_ver(): + # type: () -> str """Return implementation version.""" impl_ver = get_config_var("py_version_nodot") if not impl_ver or get_abbr_impl() == 'pp': @@ -52,17 +62,21 @@ def get_impl_ver(): def get_impl_version_info(): + # type: () -> Tuple[int, ...] """Return sys.version_info-like tuple for use in decrementing the minor version.""" if get_abbr_impl() == 'pp': # as per https://github.com/pypa/pip/issues/2882 - return (sys.version_info[0], sys.pypy_version_info.major, - sys.pypy_version_info.minor) + # attrs exist only on pypy + return (sys.version_info[0], + sys.pypy_version_info.major, # type: ignore + sys.pypy_version_info.minor) # type: ignore else: return sys.version_info[0], sys.version_info[1] def get_impl_tag(): + # type: () -> str """ Returns the Tag for this specific implementation. """ @@ -70,6 +84,7 @@ def get_impl_tag(): def get_flag(var, fallback, expected=True, warn=True): + # type: (str, Callable[..., bool], Union[bool, int], bool) -> bool """Use a fallback method for determining SOABI flags if the needed config var is unset or unavailable.""" val = get_config_var(var) @@ -82,6 +97,7 @@ def get_flag(var, fallback, expected=True, warn=True): def get_abi_tag(): + # type: () -> Optional[str] """Return the ABI tag based on SOABI (if available) or emulate SOABI (CPython 2, PyPy).""" soabi = get_config_var('SOABI') @@ -116,10 +132,12 @@ def get_abi_tag(): def _is_running_32bit(): + # type: () -> bool return sys.maxsize == 2147483647 def get_platform(): + # type: () -> str """Return our platform name 'win32', 'linux_x86_64'""" if sys.platform == 'darwin': # distutils.util.get_platform() returns the release based on the value @@ -146,6 +164,7 @@ def get_platform(): def is_manylinux1_compatible(): + # type: () -> bool # Only Linux, and only x86-64 / i686 if get_platform() not in {"linux_x86_64", "linux_i686"}: return False @@ -162,13 +181,33 @@ def is_manylinux1_compatible(): return pipenv.patched.notpip._internal.utils.glibc.have_compatible_glibc(2, 5) +def is_manylinux2010_compatible(): + # type: () -> bool + # Only Linux, and only x86-64 / i686 + if get_platform() not in {"linux_x86_64", "linux_i686"}: + return False + + # Check for presence of _manylinux module + try: + import _manylinux + return bool(_manylinux.manylinux2010_compatible) + except (ImportError, AttributeError): + # Fall through to heuristic check below + pass + + # Check glibc version. CentOS 6 uses glibc 2.12. + return pipenv.patched.notpip._internal.utils.glibc.have_compatible_glibc(2, 12) + + def get_darwin_arches(major, minor, machine): + # type: (int, int, str) -> List[str] """Return a list of supported arches (including group arches) for the given major, minor and machine architecture of an macOS machine. """ arches = [] def _supports_arch(major, minor, arch): + # type: (int, int, str) -> bool # Looking at the application support for macOS versions in the chart # provided by https://en.wikipedia.org/wiki/OS_X#Versions it appears # our timeline looks roughly like: @@ -209,7 +248,7 @@ def get_darwin_arches(major, minor, machine): ("intel", ("x86_64", "i386")), ("fat64", ("x86_64", "ppc64")), ("fat32", ("x86_64", "i386", "ppc")), - ]) + ]) # type: Dict[str, Tuple[str, ...]] if _supports_arch(major, minor, machine): arches.append(machine) @@ -223,8 +262,24 @@ def get_darwin_arches(major, minor, machine): return arches -def get_supported(versions=None, noarch=False, platform=None, - impl=None, abi=None): +def get_all_minor_versions_as_strings(version_info): + # type: (Tuple[int, ...]) -> List[str] + versions = [] + major = version_info[:-1] + # Support all previous minor Python versions. + for minor in range(version_info[-1], -1, -1): + versions.append(''.join(map(str, major + (minor,)))) + return versions + + +def get_supported( + versions=None, # type: Optional[List[str]] + noarch=False, # type: bool + platform=None, # type: Optional[str] + impl=None, # type: Optional[str] + abi=None # type: Optional[str] +): + # type: (...) -> List[Pep425Tag] """Return a list of supported tags for each version specified in `versions`. @@ -241,16 +296,12 @@ def get_supported(versions=None, noarch=False, platform=None, # Versions must be given with respect to the preference if versions is None: - versions = [] version_info = get_impl_version_info() - major = version_info[:-1] - # Support all previous minor Python versions. - for minor in range(version_info[-1], -1, -1): - versions.append(''.join(map(str, major + (minor,)))) + versions = get_all_minor_versions_as_strings(version_info) impl = impl or get_abbr_impl() - abis = [] + abis = [] # type: List[str] abi = abi or get_abi_tag() if abi: @@ -267,6 +318,7 @@ def get_supported(versions=None, noarch=False, platform=None, if not noarch: arch = platform or get_platform() + arch_prefix, arch_sep, arch_suffix = arch.partition('_') if arch.startswith('macosx'): # support macosx-10.6-intel on macosx-10.9-x86_64 match = _osx_arch_pat.match(arch) @@ -280,8 +332,19 @@ def get_supported(versions=None, noarch=False, platform=None, else: # arch pattern didn't match (?!) arches = [arch] - elif platform is None and is_manylinux1_compatible(): - arches = [arch.replace('linux', 'manylinux1'), arch] + elif arch_prefix == 'manylinux2010': + # manylinux1 wheels run on most manylinux2010 systems with the + # exception of wheels depending on ncurses. PEP 571 states + # manylinux1 wheels should be considered manylinux2010 wheels: + # https://www.python.org/dev/peps/pep-0571/#backwards-compatibility-with-manylinux1-wheels + arches = [arch, 'manylinux1' + arch_sep + arch_suffix] + elif platform is None: + arches = [] + if is_manylinux2010_compatible(): + arches.append('manylinux2010' + arch_sep + arch_suffix) + if is_manylinux1_compatible(): + arches.append('manylinux1' + arch_sep + arch_suffix) + arches.append(arch) else: arches = [arch] diff --git a/pipenv/patched/notpip/_internal/pyproject.py b/pipenv/patched/notpip/_internal/pyproject.py index a47e0f05..8845b2dc 100644 --- a/pipenv/patched/notpip/_internal/pyproject.py +++ b/pipenv/patched/notpip/_internal/pyproject.py @@ -2,20 +2,43 @@ from __future__ import absolute_import import io import os +import sys from pipenv.patched.notpip._vendor import pytoml, six from pipenv.patched.notpip._internal.exceptions import InstallationError +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Any, Tuple, Optional, List # noqa: F401 def _is_list_of_str(obj): + # type: (Any) -> bool return ( isinstance(obj, list) and all(isinstance(item, six.string_types) for item in obj) ) -def load_pyproject_toml(use_pep517, pyproject_toml, setup_py, req_name): +def make_pyproject_path(setup_py_dir): + # type: (str) -> str + path = os.path.join(setup_py_dir, 'pyproject.toml') + + # Python2 __file__ should not be unicode + if six.PY2 and isinstance(path, six.text_type): + path = path.encode(sys.getfilesystemencoding()) + + return path + + +def load_pyproject_toml( + use_pep517, # type: Optional[bool] + pyproject_toml, # type: str + setup_py, # type: str + req_name # type: str +): + # type: (...) -> Optional[Tuple[List[str], str, List[str]]] """Load the pyproject.toml file. Parameters: @@ -46,17 +69,20 @@ def load_pyproject_toml(use_pep517, pyproject_toml, setup_py, req_name): build_system = None # The following cases must use PEP 517 - # We check for use_pep517 equalling False because that - # means the user explicitly requested --no-use-pep517 + # We check for use_pep517 being non-None and falsey because that means + # the user explicitly requested --no-use-pep517. The value 0 as + # opposed to False can occur when the value is provided via an + # environment variable or config file option (due to the quirk of + # strtobool() returning an integer in pip's configuration code). if has_pyproject and not has_setup: - if use_pep517 is False: + if use_pep517 is not None and not use_pep517: raise InstallationError( "Disabling PEP 517 processing is invalid: " "project does not have a setup.py" ) use_pep517 = True elif build_system and "build-backend" in build_system: - if use_pep517 is False: + if use_pep517 is not None and not use_pep517: raise InstallationError( "Disabling PEP 517 processing is invalid: " "project specifies a build backend of {} " @@ -85,11 +111,13 @@ def load_pyproject_toml(use_pep517, pyproject_toml, setup_py, req_name): # section, or the user has no pyproject.toml, but has opted in # explicitly via --use-pep517. # In the absence of any explicit backend specification, we - # assume the setuptools backend, and require wheel and a version - # of setuptools that supports that backend. + # assume the setuptools backend that most closely emulates the + # traditional direct setup.py execution, and require wheel and + # a version of setuptools that supports that backend. + build_system = { - "requires": ["setuptools>=38.2.5", "wheel"], - "build-backend": "setuptools.build_meta", + "requires": ["setuptools>=40.8.0", "wheel"], + "build-backend": "setuptools.build_meta:__legacy__", } # If we're using PEP 517, we have build system information (either @@ -123,22 +151,21 @@ def load_pyproject_toml(use_pep517, pyproject_toml, setup_py, req_name): )) backend = build_system.get("build-backend") - check = [] + check = [] # type: List[str] if backend is None: # If the user didn't specify a backend, we assume they want to use # the setuptools backend. But we can't be sure they have included # a version of setuptools which supplies the backend, or wheel - # (which is neede by the backend) in their requirements. So we + # (which is needed by the backend) in their requirements. So we # make a note to check that those requirements are present once # we have set up the environment. - # TODO: Review this - it's quite a lot of work to check for a very - # specific case. The problem is, that case is potentially quite - # common - projects that adopted PEP 518 early for the ability to - # specify requirements to execute setup.py, but never considered - # needing to mention the build tools themselves. The original PEP - # 518 code had a similar check (but implemented in a different - # way). - backend = "setuptools.build_meta" - check = ["setuptools>=38.2.5", "wheel"] + # This is quite a lot of work to check for a very specific case. But + # the problem is, that case is potentially quite common - projects that + # adopted PEP 518 early for the ability to specify requirements to + # execute setup.py, but never considered needing to mention the build + # tools themselves. The original PEP 518 code had a similar check (but + # implemented in a different way). + backend = "setuptools.build_meta:__legacy__" + check = ["setuptools>=40.8.0", "wheel"] return (requires, backend, check) diff --git a/pipenv/patched/notpip/_internal/req/__init__.py b/pipenv/patched/notpip/_internal/req/__init__.py index 72aecb7c..51606fec 100644 --- a/pipenv/patched/notpip/_internal/req/__init__.py +++ b/pipenv/patched/notpip/_internal/req/__init__.py @@ -6,7 +6,10 @@ from .req_install import InstallRequirement from .req_set import RequirementSet from .req_file import parse_requirements from pipenv.patched.notpip._internal.utils.logging import indent_log +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING +if MYPY_CHECK_RUNNING: + from typing import List, Sequence # noqa: F401 __all__ = [ "RequirementSet", "InstallRequirement", @@ -16,8 +19,13 @@ __all__ = [ logger = logging.getLogger(__name__) -def install_given_reqs(to_install, install_options, global_options=(), - *args, **kwargs): +def install_given_reqs( + to_install, # type: List[InstallRequirement] + install_options, # type: List[str] + global_options=(), # type: Sequence[str] + *args, **kwargs +): + # type: (...) -> List[InstallRequirement] """ Install everything in the given list. diff --git a/pipenv/patched/notpip/_internal/req/constructors.py b/pipenv/patched/notpip/_internal/req/constructors.py index 9fe28d88..a9d8221a 100644 --- a/pipenv/patched/notpip/_internal/req/constructors.py +++ b/pipenv/patched/notpip/_internal/req/constructors.py @@ -11,7 +11,6 @@ InstallRequirement. import logging import os import re -import traceback from pipenv.patched.notpip._vendor.packaging.markers import Marker from pipenv.patched.notpip._vendor.packaging.requirements import InvalidRequirement, Requirement @@ -24,11 +23,20 @@ from pipenv.patched.notpip._internal.download import ( from pipenv.patched.notpip._internal.exceptions import InstallationError from pipenv.patched.notpip._internal.models.index import PyPI, TestPyPI from pipenv.patched.notpip._internal.models.link import Link +from pipenv.patched.notpip._internal.pyproject import make_pyproject_path from pipenv.patched.notpip._internal.req.req_install import InstallRequirement from pipenv.patched.notpip._internal.utils.misc import is_installable_dir +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING from pipenv.patched.notpip._internal.vcs import vcs from pipenv.patched.notpip._internal.wheel import Wheel +if MYPY_CHECK_RUNNING: + from typing import ( # noqa: F401 + Optional, Tuple, Set, Any, Union, Text, Dict, + ) + from pipenv.patched.notpip._internal.cache import WheelCache # noqa: F401 + + __all__ = [ "install_req_from_editable", "install_req_from_line", "parse_editable" @@ -39,6 +47,7 @@ operators = Specifier._operators.keys() def _strip_extras(path): + # type: (str) -> Tuple[str, Optional[str]] m = re.match(r'^(.+)(\[[^\]]+\])$', path) extras = None if m: @@ -51,6 +60,7 @@ def _strip_extras(path): def parse_editable(editable_req): + # type: (str) -> Tuple[Optional[str], str, Optional[Set[str]]] """Parses an editable requirement into: - a requirement name - an URL @@ -68,10 +78,18 @@ def parse_editable(editable_req): if os.path.isdir(url_no_extras): if not os.path.exists(os.path.join(url_no_extras, 'setup.py')): - raise InstallationError( - "Directory %r is not installable. File 'setup.py' not found." % - url_no_extras + msg = ( + 'File "setup.py" not found. Directory cannot be installed ' + 'in editable mode: {}'.format(os.path.abspath(url_no_extras)) ) + pyproject_path = make_pyproject_path(url_no_extras) + if os.path.isfile(pyproject_path): + msg += ( + '\n(A "pyproject.toml" file was found, but editable ' + 'mode currently requires a setup.py based build.)' + ) + raise InstallationError(msg) + # Treating it as code that has already been checked out url_no_extras = path_to_url(url_no_extras) @@ -116,6 +134,7 @@ def parse_editable(editable_req): def deduce_helpful_msg(req): + # type: (str) -> str """Returns helpful msg in case requirements file does not exist, or cannot be parsed. @@ -136,7 +155,7 @@ def deduce_helpful_msg(req): " the packages specified within it." except RequirementParseError: logger.debug("Cannot parse '%s' as requirements \ - file" % (req), exc_info=1) + file" % (req), exc_info=True) else: msg += " File '%s' does not exist." % (req) return msg @@ -146,9 +165,15 @@ def deduce_helpful_msg(req): def install_req_from_editable( - editable_req, comes_from=None, isolated=False, options=None, - wheel_cache=None, constraint=False + editable_req, # type: str + comes_from=None, # type: Optional[str] + use_pep517=None, # type: Optional[bool] + isolated=False, # type: bool + options=None, # type: Optional[Dict[str, Any]] + wheel_cache=None, # type: Optional[WheelCache] + constraint=False # type: bool ): + # type: (...) -> InstallRequirement name, url, extras_override = parse_editable(editable_req) if url.startswith('file:'): source_dir = url_to_path(url) @@ -167,6 +192,7 @@ def install_req_from_editable( editable=True, link=Link(url), constraint=constraint, + use_pep517=use_pep517, isolated=isolated, options=options if options else {}, wheel_cache=wheel_cache, @@ -175,9 +201,15 @@ def install_req_from_editable( def install_req_from_line( - name, comes_from=None, isolated=False, options=None, wheel_cache=None, - constraint=False + name, # type: str + comes_from=None, # type: Optional[Union[str, InstallRequirement]] + use_pep517=None, # type: Optional[bool] + isolated=False, # type: bool + options=None, # type: Optional[Dict[str, Any]] + wheel_cache=None, # type: Optional[WheelCache] + constraint=False # type: bool ): + # type: (...) -> InstallRequirement """Creates an InstallRequirement from a name, which might be a requirement, directory containing 'setup.py', filename, or URL. """ @@ -186,24 +218,24 @@ def install_req_from_line( else: marker_sep = ';' if marker_sep in name: - name, markers = name.split(marker_sep, 1) - markers = markers.strip() - if not markers: + name, markers_as_string = name.split(marker_sep, 1) + markers_as_string = markers_as_string.strip() + if not markers_as_string: markers = None else: - markers = Marker(markers) + markers = Marker(markers_as_string) else: markers = None name = name.strip() - req = None + req_as_string = None path = os.path.normpath(os.path.abspath(name)) link = None - extras = None + extras_as_string = None if is_url(name): link = Link(name) else: - p, extras = _strip_extras(path) + p, extras_as_string = _strip_extras(path) looks_like_dir = os.path.isdir(p) and ( os.path.sep in name or (os.path.altsep is not None and os.path.altsep in name) or @@ -234,38 +266,41 @@ def install_req_from_line( # wheel file if link.is_wheel: wheel = Wheel(link.filename) # can raise InvalidWheelFilename - req = "%s==%s" % (wheel.name, wheel.version) + req_as_string = "%s==%s" % (wheel.name, wheel.version) else: # set the req to the egg fragment. when it's not there, this # will become an 'unnamed' requirement - req = link.egg_fragment + req_as_string = link.egg_fragment # a requirement specifier else: - req = name + req_as_string = name - if extras: - extras = Requirement("placeholder" + extras.lower()).extras + if extras_as_string: + extras = Requirement("placeholder" + extras_as_string.lower()).extras else: extras = () - if req is not None: + if req_as_string is not None: try: - req = Requirement(req) + req = Requirement(req_as_string) except InvalidRequirement: - if os.path.sep in req: + if os.path.sep in req_as_string: add_msg = "It looks like a path." - add_msg += deduce_helpful_msg(req) - elif '=' in req and not any(op in req for op in operators): + add_msg += deduce_helpful_msg(req_as_string) + elif ('=' in req_as_string and + not any(op in req_as_string for op in operators)): add_msg = "= is not a valid operator. Did you mean == ?" else: - add_msg = traceback.format_exc() + add_msg = "" raise InstallationError( - "Invalid requirement: '%s'\n%s" % (req, add_msg) + "Invalid requirement: '%s'\n%s" % (req_as_string, add_msg) ) + else: + req = None return InstallRequirement( req, comes_from, link=link, markers=markers, - isolated=isolated, + use_pep517=use_pep517, isolated=isolated, options=options if options else {}, wheel_cache=wheel_cache, constraint=constraint, @@ -273,11 +308,16 @@ def install_req_from_line( ) -def install_req_from_req( - req, comes_from=None, isolated=False, wheel_cache=None +def install_req_from_req_string( + req_string, # type: str + comes_from=None, # type: Optional[InstallRequirement] + isolated=False, # type: bool + wheel_cache=None, # type: Optional[WheelCache] + use_pep517=None # type: Optional[bool] ): + # type: (...) -> InstallRequirement try: - req = Requirement(req) + req = Requirement(req_string) except InvalidRequirement: raise InstallationError("Invalid requirement: '%s'" % req) @@ -294,5 +334,6 @@ def install_req_from_req( ) return InstallRequirement( - req, comes_from, isolated=isolated, wheel_cache=wheel_cache + req, comes_from, isolated=isolated, wheel_cache=wheel_cache, + use_pep517=use_pep517 ) diff --git a/pipenv/patched/notpip/_internal/req/req_file.py b/pipenv/patched/notpip/_internal/req/req_file.py index 5f23cd3a..30391cb5 100644 --- a/pipenv/patched/notpip/_internal/req/req_file.py +++ b/pipenv/patched/notpip/_internal/req/req_file.py @@ -19,6 +19,18 @@ from pipenv.patched.notpip._internal.exceptions import RequirementsFileParseErro from pipenv.patched.notpip._internal.req.constructors import ( install_req_from_editable, install_req_from_line, ) +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import ( # noqa: F401 + Iterator, Tuple, Optional, List, Callable, Text + ) + from pipenv.patched.notpip._internal.req import InstallRequirement # noqa: F401 + from pipenv.patched.notpip._internal.cache import WheelCache # noqa: F401 + from pipenv.patched.notpip._internal.index import PackageFinder # noqa: F401 + from pipenv.patched.notpip._internal.download import PipSession # noqa: F401 + + ReqFileLines = Iterator[Tuple[int, Text]] __all__ = ['parse_requirements'] @@ -43,24 +55,32 @@ SUPPORTED_OPTIONS = [ cmdoptions.no_binary, cmdoptions.only_binary, cmdoptions.pre, - cmdoptions.process_dependency_links, cmdoptions.trusted_host, cmdoptions.require_hashes, -] +] # type: List[Callable[..., optparse.Option]] # options to be passed to requirements SUPPORTED_OPTIONS_REQ = [ cmdoptions.install_options, cmdoptions.global_options, cmdoptions.hash, -] +] # type: List[Callable[..., optparse.Option]] # the 'dest' string values -SUPPORTED_OPTIONS_REQ_DEST = [o().dest for o in SUPPORTED_OPTIONS_REQ] +SUPPORTED_OPTIONS_REQ_DEST = [str(o().dest) for o in SUPPORTED_OPTIONS_REQ] -def parse_requirements(filename, finder=None, comes_from=None, options=None, - session=None, constraint=False, wheel_cache=None): +def parse_requirements( + filename, # type: str + finder=None, # type: Optional[PackageFinder] + comes_from=None, # type: Optional[str] + options=None, # type: Optional[optparse.Values] + session=None, # type: Optional[PipSession] + constraint=False, # type: bool + wheel_cache=None, # type: Optional[WheelCache] + use_pep517=None # type: Optional[bool] +): + # type: (...) -> Iterator[InstallRequirement] """Parse a requirements file and yield InstallRequirement instances. :param filename: Path or url of requirements file. @@ -71,6 +91,7 @@ def parse_requirements(filename, finder=None, comes_from=None, options=None, :param constraint: If true, parsing a constraint file rather than requirements file. :param wheel_cache: Instance of pip.wheel.WheelCache + :param use_pep517: Value of the --use-pep517 option. """ if session is None: raise TypeError( @@ -87,18 +108,19 @@ def parse_requirements(filename, finder=None, comes_from=None, options=None, for line_number, line in lines_enum: req_iter = process_line(line, filename, line_number, finder, comes_from, options, session, wheel_cache, - constraint=constraint) + use_pep517=use_pep517, constraint=constraint) for req in req_iter: yield req def preprocess(content, options): + # type: (Text, Optional[optparse.Values]) -> ReqFileLines """Split, filter, and join lines, and return a line iterator :param content: the content of the requirements file :param options: cli options """ - lines_enum = enumerate(content.splitlines(), start=1) + lines_enum = enumerate(content.splitlines(), start=1) # type: ReqFileLines lines_enum = join_lines(lines_enum) lines_enum = ignore_comments(lines_enum) lines_enum = skip_regex(lines_enum, options) @@ -106,9 +128,19 @@ def preprocess(content, options): return lines_enum -def process_line(line, filename, line_number, finder=None, comes_from=None, - options=None, session=None, wheel_cache=None, - constraint=False): +def process_line( + line, # type: Text + filename, # type: str + line_number, # type: int + finder=None, # type: Optional[PackageFinder] + comes_from=None, # type: Optional[str] + options=None, # type: Optional[optparse.Values] + session=None, # type: Optional[PipSession] + wheel_cache=None, # type: Optional[WheelCache] + use_pep517=None, # type: Optional[bool] + constraint=False # type: bool +): + # type: (...) -> Iterator[InstallRequirement] """Process a single requirements line; This can result in creating/yielding requirements, or updating the finder. @@ -130,13 +162,15 @@ def process_line(line, filename, line_number, finder=None, comes_from=None, defaults = parser.get_default_values() defaults.index_url = None if finder: - # `finder.format_control` will be updated during parsing defaults.format_control = finder.format_control args_str, options_str = break_args_options(line) + # Prior to 2.7.3, shlex cannot deal with unicode entries if sys.version_info < (2, 7, 3): - # Prior to 2.7.3, shlex cannot deal with unicode entries - options_str = options_str.encode('utf8') - opts, _ = parser.parse_args(shlex.split(options_str), defaults) + # https://github.com/python/mypy/issues/1174 + options_str = options_str.encode('utf8') # type: ignore + # https://github.com/python/mypy/issues/1174 + opts, _ = parser.parse_args( + shlex.split(options_str), defaults) # type: ignore # preserve for the nested code path line_comes_from = '%s %s (line %s)' % ( @@ -155,6 +189,7 @@ def process_line(line, filename, line_number, finder=None, comes_from=None, req_options[dest] = opts.__dict__[dest] yield install_req_from_line( args_str, line_comes_from, constraint=constraint, + use_pep517=use_pep517, isolated=isolated, options=req_options, wheel_cache=wheel_cache ) @@ -163,6 +198,7 @@ def process_line(line, filename, line_number, finder=None, comes_from=None, isolated = options.isolated_mode if options else False yield install_req_from_editable( opts.editables[0], comes_from=line_comes_from, + use_pep517=use_pep517, constraint=constraint, isolated=isolated, wheel_cache=wheel_cache ) @@ -183,11 +219,11 @@ def process_line(line, filename, line_number, finder=None, comes_from=None, # do a join so relative paths work req_path = os.path.join(os.path.dirname(filename), req_path) # TODO: Why not use `comes_from='-r {} (line {})'` here as well? - parser = parse_requirements( + parsed_reqs = parse_requirements( req_path, finder, comes_from, options, session, constraint=nested_constraint, wheel_cache=wheel_cache ) - for req in parser: + for req in parsed_reqs: yield req # percolate hash-checking option upward @@ -214,14 +250,13 @@ def process_line(line, filename, line_number, finder=None, comes_from=None, finder.find_links.append(value) if opts.pre: finder.allow_all_prereleases = True - if opts.process_dependency_links: - finder.process_dependency_links = True if opts.trusted_hosts: finder.secure_origins.extend( ("*", host, "*") for host in opts.trusted_hosts) def break_args_options(line): + # type: (Text) -> Tuple[str, Text] """Break up the line into an args and options string. We only want to shlex (and then optparse) the options, not the args. args can contain markers which are corrupted by shlex. @@ -235,10 +270,11 @@ def break_args_options(line): else: args.append(token) options.pop(0) - return ' '.join(args), ' '.join(options) + return ' '.join(args), ' '.join(options) # type: ignore def build_parser(line): + # type: (Text) -> optparse.OptionParser """ Return a parser for parsing requirement lines """ @@ -255,17 +291,20 @@ def build_parser(line): # add offending line msg = 'Invalid requirement: %s\n%s' % (line, msg) raise RequirementsFileParseError(msg) - parser.exit = parser_exit + # NOTE: mypy disallows assigning to a method + # https://github.com/python/mypy/issues/2427 + parser.exit = parser_exit # type: ignore return parser def join_lines(lines_enum): + # type: (ReqFileLines) -> ReqFileLines """Joins a line ending in '\' with the previous line (except when following comments). The joined line takes on the index of the first line. """ primary_line_number = None - new_line = [] + new_line = [] # type: List[Text] for line_number, line in lines_enum: if not line.endswith('\\') or COMMENT_RE.match(line): if COMMENT_RE.match(line): @@ -290,6 +329,7 @@ def join_lines(lines_enum): def ignore_comments(lines_enum): + # type: (ReqFileLines) -> ReqFileLines """ Strips comments and filter empty lines. """ @@ -301,6 +341,7 @@ def ignore_comments(lines_enum): def skip_regex(lines_enum, options): + # type: (ReqFileLines, Optional[optparse.Values]) -> ReqFileLines """ Skip lines that match '--skip-requirements-regex' pattern @@ -314,6 +355,7 @@ def skip_regex(lines_enum, options): def expand_env_variables(lines_enum): + # type: (ReqFileLines) -> ReqFileLines """Replace all environment variables that can be retrieved via `os.getenv`. The only allowed format for environment variables defined in the diff --git a/pipenv/patched/notpip/_internal/req/req_install.py b/pipenv/patched/notpip/_internal/req/req_install.py index 3f32892c..fd3cead6 100644 --- a/pipenv/patched/notpip/_internal/req/req_install.py +++ b/pipenv/patched/notpip/_internal/req/req_install.py @@ -22,7 +22,7 @@ from pipenv.patched.notpip._internal.locations import ( PIP_DELETE_MARKER_FILENAME, running_under_virtualenv, ) from pipenv.patched.notpip._internal.models.link import Link -from pipenv.patched.notpip._internal.pyproject import load_pyproject_toml +from pipenv.patched.notpip._internal.pyproject import load_pyproject_toml, make_pyproject_path from pipenv.patched.notpip._internal.req.req_uninstall import UninstallPathSet from pipenv.patched.notpip._internal.utils.compat import native_str from pipenv.patched.notpip._internal.utils.hashes import Hashes @@ -30,15 +30,28 @@ from pipenv.patched.notpip._internal.utils.logging import indent_log from pipenv.patched.notpip._internal.utils.misc import ( _make_build_dir, ask_path_exists, backup_dir, call_subprocess, display_path, dist_in_site_packages, dist_in_usersite, ensure_dir, - get_installed_version, rmtree, + get_installed_version, redact_password_from_url, rmtree, ) from pipenv.patched.notpip._internal.utils.packaging import get_metadata from pipenv.patched.notpip._internal.utils.setuptools_build import SETUPTOOLS_SHIM from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING from pipenv.patched.notpip._internal.utils.ui import open_spinner from pipenv.patched.notpip._internal.vcs import vcs from pipenv.patched.notpip._internal.wheel import move_wheel_files +if MYPY_CHECK_RUNNING: + from typing import ( # noqa: F401 + Optional, Iterable, List, Union, Any, Text, Sequence, Dict + ) + from pipenv.patched.notpip._internal.build_env import BuildEnvironment # noqa: F401 + from pipenv.patched.notpip._internal.cache import WheelCache # noqa: F401 + from pipenv.patched.notpip._internal.index import PackageFinder # noqa: F401 + from pipenv.patched.notpip._vendor.pkg_resources import Distribution # noqa: F401 + from pipenv.patched.notpip._vendor.packaging.specifiers import SpecifierSet # noqa: F401 + from pipenv.patched.notpip._vendor.packaging.markers import Marker # noqa: F401 + + logger = logging.getLogger(__name__) @@ -49,10 +62,23 @@ class InstallRequirement(object): installing the said requirement. """ - def __init__(self, req, comes_from, source_dir=None, editable=False, - link=None, update=True, markers=None, - isolated=False, options=None, wheel_cache=None, - constraint=False, extras=()): + def __init__( + self, + req, # type: Optional[Requirement] + comes_from, # type: Optional[Union[str, InstallRequirement]] + source_dir=None, # type: Optional[str] + editable=False, # type: bool + link=None, # type: Optional[Link] + update=True, # type: bool + markers=None, # type: Optional[Marker] + use_pep517=None, # type: Optional[bool] + isolated=False, # type: bool + options=None, # type: Optional[Dict[str, Any]] + wheel_cache=None, # type: Optional[WheelCache] + constraint=False, # type: bool + extras=() # type: Iterable[str] + ): + # type: (...) -> None assert req is None or isinstance(req, Requirement), req self.req = req self.comes_from = comes_from @@ -64,10 +90,10 @@ class InstallRequirement(object): self.editable = editable self._wheel_cache = wheel_cache - if link is not None: - self.link = self.original_link = link - else: - self.link = self.original_link = req and req.url and Link(req.url) + if link is None and req and req.url: + # PEP 508 URL requirement + link = Link(req.url) + self.link = self.original_link = link if extras: self.extras = extras @@ -77,11 +103,11 @@ class InstallRequirement(object): } else: self.extras = set() - if markers is not None: - self.markers = markers - else: - self.markers = req and req.marker - self._egg_info_path = None + if markers is None and req: + markers = req.marker + self.markers = markers + + self._egg_info_path = None # type: Optional[str] # This holds the pkg_resources.Distribution object if this requirement # is already available: self.satisfied_by = None @@ -92,11 +118,11 @@ class InstallRequirement(object): self._temp_build_dir = TempDirectory(kind="req-build") # Used to store the global directory where the _temp_build_dir should # have been created. Cf _correct_build_location method. - self._ideal_build_dir = None + self._ideal_build_dir = None # type: Optional[str] # True if the editable should be updated: self.update = update # Set to True after successful installation - self.install_succeeded = None + self.install_succeeded = None # type: Optional[bool] # UninstallPathSet of uninstalled distribution (for possible rollback) self.uninstalled_pathset = None self.options = options if options else {} @@ -105,32 +131,37 @@ class InstallRequirement(object): self.is_direct = False self.isolated = isolated - self.build_env = NoOpBuildEnvironment() + self.build_env = NoOpBuildEnvironment() # type: BuildEnvironment + + # For PEP 517, the directory where we request the project metadata + # gets stored. We need this to pass to build_wheel, so the backend + # can ensure that the wheel matches the metadata (see the PEP for + # details). + self.metadata_directory = None # type: Optional[str] # The static build requirements (from pyproject.toml) - self.pyproject_requires = None + self.pyproject_requires = None # type: Optional[List[str]] # Build requirements that we will check are available - # TODO: We don't do this for --no-build-isolation. Should we? - self.requirements_to_check = [] + self.requirements_to_check = [] # type: List[str] # The PEP 517 backend we should use to build the project - self.pep517_backend = None + self.pep517_backend = None # type: Optional[Pep517HookCaller] # Are we using PEP 517 for this requirement? # After pyproject.toml has been loaded, the only valid values are True # and False. Before loading, None is valid (meaning "use the default"). # Setting an explicit value before loading pyproject.toml is supported, # but after loading this flag should be treated as read only. - self.use_pep517 = None + self.use_pep517 = use_pep517 def __str__(self): if self.req: s = str(self.req) if self.link: - s += ' from %s' % self.link.url + s += ' from %s' % redact_password_from_url(self.link.url) elif self.link: - s = self.link.url + s = redact_password_from_url(self.link.url) else: s = '' if self.satisfied_by is not None: @@ -149,6 +180,7 @@ class InstallRequirement(object): self.__class__.__name__, str(self), self.editable) def populate_link(self, finder, upgrade, require_hashes): + # type: (PackageFinder, bool, bool) -> None """Ensure that if a link can be found for this, that it is found. Note that self.link may still be None - if Upgrade is False and the @@ -171,16 +203,19 @@ class InstallRequirement(object): # Things that are valid for all kinds of requirements? @property def name(self): + # type: () -> Optional[str] if self.req is None: return None return native_str(pkg_resources.safe_name(self.req.name)) @property def specifier(self): + # type: () -> SpecifierSet return self.req.specifier @property def is_pinned(self): + # type: () -> bool """Return whether I am pinned to an exact version. For example, some-package==1.2 is pinned; some-package>1.2 is not. @@ -194,6 +229,7 @@ class InstallRequirement(object): return get_installed_version(self.name) def match_markers(self, extras_requested=None): + # type: (Optional[Iterable[str]]) -> bool if not extras_requested: # Provide an extra to safely evaluate the markers # without matching any extra @@ -207,6 +243,7 @@ class InstallRequirement(object): @property def has_hash_options(self): + # type: () -> bool """Return whether any known-good hashes are specified as options. These activate --require-hashes mode; hashes specified as part of a @@ -216,6 +253,7 @@ class InstallRequirement(object): return bool(self.options.get('hashes', {})) def hashes(self, trust_internet=True): + # type: (bool) -> Hashes """Return a hash-comparer that considers my option- and URL-based hashes to be known-good. @@ -237,6 +275,7 @@ class InstallRequirement(object): return Hashes(good_hashes) def from_path(self): + # type: () -> Optional[str] """Format a nice indicator to show where this "comes from" """ if self.req is None: @@ -252,6 +291,7 @@ class InstallRequirement(object): return s def build_location(self, build_dir): + # type: (str) -> Optional[str] assert build_dir is not None if self._temp_build_dir.path is not None: return self._temp_build_dir.path @@ -279,6 +319,7 @@ class InstallRequirement(object): return os.path.join(build_dir, name) def _correct_build_location(self): + # type: () -> None """Move self._temp_build_dir to self._ideal_build_dir/self.req.name For some requirements (e.g. a path to a directory), the name of the @@ -292,7 +333,8 @@ class InstallRequirement(object): return assert self.req is not None assert self._temp_build_dir.path - assert self._ideal_build_dir.path + assert (self._ideal_build_dir is not None and + self._ideal_build_dir.path) # type: ignore old_location = self._temp_build_dir.path self._temp_build_dir.path = None @@ -311,7 +353,16 @@ class InstallRequirement(object): self.source_dir = os.path.normpath(os.path.abspath(new_location)) self._egg_info_path = None + # Correct the metadata directory, if it exists + if self.metadata_directory: + old_meta = self.metadata_directory + rel = os.path.relpath(old_meta, start=old_location) + new_meta = os.path.join(new_location, rel) + new_meta = os.path.normpath(os.path.abspath(new_meta)) + self.metadata_directory = new_meta + def remove_temporary_source(self): + # type: () -> None """Remove the source files from this requirement, if they are marked for deletion""" if self.source_dir and os.path.exists( @@ -323,6 +374,7 @@ class InstallRequirement(object): self.build_env.cleanup() def check_if_exists(self, use_user_site): + # type: (bool) -> bool """Find an installed distribution that satisfies or conflicts with this requirement, and set self.satisfied_by or self.conflicts_with appropriately. @@ -366,11 +418,22 @@ class InstallRequirement(object): # Things valid for wheels @property def is_wheel(self): - return self.link and self.link.is_wheel + # type: () -> bool + if not self.link: + return False + return self.link.is_wheel - def move_wheel_files(self, wheeldir, root=None, home=None, prefix=None, - warn_script_location=True, use_user_site=False, - pycompile=True): + def move_wheel_files( + self, + wheeldir, # type: str + root=None, # type: Optional[str] + home=None, # type: Optional[str] + prefix=None, # type: Optional[str] + warn_script_location=True, # type: bool + use_user_site=False, # type: bool + pycompile=True # type: bool + ): + # type: (...) -> None move_wheel_files( self.name, self.req, wheeldir, user=use_user_site, @@ -385,12 +448,14 @@ class InstallRequirement(object): # Things valid for sdists @property def setup_py_dir(self): + # type: () -> str return os.path.join( self.source_dir, self.link and self.link.subdirectory_fragment or '') @property def setup_py(self): + # type: () -> str assert self.source_dir, "No source dir for %s" % self setup_py = os.path.join(self.setup_py_dir, 'setup.py') @@ -403,17 +468,13 @@ class InstallRequirement(object): @property def pyproject_toml(self): + # type: () -> str assert self.source_dir, "No source dir for %s" % self - pp_toml = os.path.join(self.setup_py_dir, 'pyproject.toml') - - # Python2 __file__ should not be unicode - if six.PY2 and isinstance(pp_toml, six.text_type): - pp_toml = pp_toml.encode(sys.getfilesystemencoding()) - - return pp_toml + return make_pyproject_path(self.setup_py_dir) def load_pyproject_toml(self): + # type: () -> None """Load the pyproject.toml file. After calling this routine, all of the attributes related to PEP 517 @@ -437,41 +498,36 @@ class InstallRequirement(object): self.pyproject_requires = requires self.pep517_backend = Pep517HookCaller(self.setup_py_dir, backend) - def run_egg_info(self): + # Use a custom function to call subprocesses + self.spin_message = "" + + def runner(cmd, cwd=None, extra_environ=None): + with open_spinner(self.spin_message) as spinner: + call_subprocess( + cmd, + cwd=cwd, + extra_environ=extra_environ, + show_stdout=False, + spinner=spinner + ) + self.spin_message = "" + + self.pep517_backend._subprocess_runner = runner + + def prepare_metadata(self): + # type: () -> None + """Ensure that project metadata is available. + + Under PEP 517, call the backend hook to prepare the metadata. + Under legacy processing, call setup.py egg-info. + """ assert self.source_dir - if self.name: - logger.debug( - 'Running setup.py (path:%s) egg_info for package %s', - self.setup_py, self.name, - ) - else: - logger.debug( - 'Running setup.py (path:%s) egg_info for package from %s', - self.setup_py, self.link, - ) with indent_log(): - script = SETUPTOOLS_SHIM % self.setup_py - sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) - base_cmd = [sys_executable, '-c', script] - if self.isolated: - base_cmd += ["--no-user-cfg"] - egg_info_cmd = base_cmd + ['egg_info'] - # We can't put the .egg-info files at the root, because then the - # source code will be mistaken for an installed egg, causing - # problems - if self.editable: - egg_base_option = [] + if self.use_pep517: + self.prepare_pep517_metadata() else: - egg_info_dir = os.path.join(self.setup_py_dir, 'pip-egg-info') - ensure_dir(egg_info_dir) - egg_base_option = ['--egg-base', 'pip-egg-info'] - with self.build_env: - call_subprocess( - egg_info_cmd + egg_base_option, - cwd=self.setup_py_dir, - show_stdout=False, - command_desc='python setup.py egg_info') + self.run_egg_info() if not self.req: if isinstance(parse_version(self.metadata["Version"]), Version): @@ -490,15 +546,72 @@ class InstallRequirement(object): metadata_name = canonicalize_name(self.metadata["Name"]) if canonicalize_name(self.req.name) != metadata_name: logger.warning( - 'Running setup.py (path:%s) egg_info for package %s ' + 'Generating metadata for package %s ' 'produced metadata for project name %s. Fix your ' '#egg=%s fragments.', - self.setup_py, self.name, metadata_name, self.name + self.name, metadata_name, self.name ) self.req = Requirement(metadata_name) + def prepare_pep517_metadata(self): + # type: () -> None + assert self.pep517_backend is not None + + metadata_dir = os.path.join( + self.setup_py_dir, + 'pip-wheel-metadata' + ) + ensure_dir(metadata_dir) + + with self.build_env: + # Note that Pep517HookCaller implements a fallback for + # prepare_metadata_for_build_wheel, so we don't have to + # consider the possibility that this hook doesn't exist. + backend = self.pep517_backend + self.spin_message = "Preparing wheel metadata" + distinfo_dir = backend.prepare_metadata_for_build_wheel( + metadata_dir + ) + + self.metadata_directory = os.path.join(metadata_dir, distinfo_dir) + + def run_egg_info(self): + # type: () -> None + if self.name: + logger.debug( + 'Running setup.py (path:%s) egg_info for package %s', + self.setup_py, self.name, + ) + else: + logger.debug( + 'Running setup.py (path:%s) egg_info for package from %s', + self.setup_py, self.link, + ) + script = SETUPTOOLS_SHIM % self.setup_py + sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) + base_cmd = [sys_executable, '-c', script] + if self.isolated: + base_cmd += ["--no-user-cfg"] + egg_info_cmd = base_cmd + ['egg_info'] + # We can't put the .egg-info files at the root, because then the + # source code will be mistaken for an installed egg, causing + # problems + if self.editable: + egg_base_option = [] # type: List[str] + else: + egg_info_dir = os.path.join(self.setup_py_dir, 'pip-egg-info') + ensure_dir(egg_info_dir) + egg_base_option = ['--egg-base', 'pip-egg-info'] + with self.build_env: + call_subprocess( + egg_info_cmd + egg_base_option, + cwd=self.setup_py_dir, + show_stdout=False, + command_desc='python setup.py egg_info') + @property def egg_info_path(self): + # type: () -> str if self._egg_info_path is None: if self.editable: base = self.source_dir @@ -557,18 +670,31 @@ class InstallRequirement(object): return self._metadata def get_dist(self): - """Return a pkg_resources.Distribution built from self.egg_info_path""" - egg_info = self.egg_info_path.rstrip(os.path.sep) - base_dir = os.path.dirname(egg_info) - metadata = pkg_resources.PathMetadata(base_dir, egg_info) - dist_name = os.path.splitext(os.path.basename(egg_info))[0] - return pkg_resources.Distribution( - os.path.dirname(egg_info), + # type: () -> Distribution + """Return a pkg_resources.Distribution for this requirement""" + if self.metadata_directory: + base_dir, distinfo = os.path.split(self.metadata_directory) + metadata = pkg_resources.PathMetadata( + base_dir, self.metadata_directory + ) + dist_name = os.path.splitext(distinfo)[0] + typ = pkg_resources.DistInfoDistribution + else: + egg_info = self.egg_info_path.rstrip(os.path.sep) + base_dir = os.path.dirname(egg_info) + metadata = pkg_resources.PathMetadata(base_dir, egg_info) + dist_name = os.path.splitext(os.path.basename(egg_info))[0] + # https://github.com/python/mypy/issues/1174 + typ = pkg_resources.Distribution # type: ignore + + return typ( + base_dir, project_name=dist_name, metadata=metadata, ) def assert_source_matches_version(self): + # type: () -> None assert self.source_dir version = self.metadata['version'] if self.req.specifier and version not in self.req.specifier: @@ -587,6 +713,7 @@ class InstallRequirement(object): # For both source distributions and editables def ensure_has_source_dir(self, parent_dir): + # type: (str) -> str """Ensure that a source_dir is set. This will create a temporary build dir if the name of the requirement @@ -601,8 +728,13 @@ class InstallRequirement(object): return self.source_dir # For editable installations - def install_editable(self, install_options, - global_options=(), prefix=None): + def install_editable( + self, + install_options, # type: List[str] + global_options=(), # type: Sequence[str] + prefix=None # type: Optional[str] + ): + # type: (...) -> None logger.info('Running setup.py develop for %s', self.name) if self.isolated: @@ -614,8 +746,8 @@ class InstallRequirement(object): with indent_log(): # FIXME: should we do --install-headers here too? - sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) with self.build_env: + sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) call_subprocess( [ sys_executable, @@ -633,6 +765,7 @@ class InstallRequirement(object): self.install_succeeded = True def update_editable(self, obtain=True): + # type: (bool) -> None if not self.link: logger.debug( "Cannot update repository at %s; repository location is " @@ -664,6 +797,7 @@ class InstallRequirement(object): # Top-level Actions def uninstall(self, auto_confirm=False, verbose=False, use_user_site=False): + # type: (bool, bool, bool) -> Optional[UninstallPathSet] """ Uninstall the distribution currently satisfying this requirement. @@ -678,7 +812,7 @@ class InstallRequirement(object): """ if not self.check_if_exists(use_user_site): logger.warning("Skipping %s as it is not installed.", self.name) - return + return None dist = self.satisfied_by or self.conflicts_with uninstalled_pathset = UninstallPathSet.from_dist(dist) @@ -693,9 +827,16 @@ class InstallRequirement(object): name = name.replace(os.path.sep, '/') return name + def _get_archive_name(self, path, parentdir, rootdir): + # type: (str, str, str) -> str + path = os.path.join(parentdir, path) + name = self._clean_zip_name(path, rootdir) + return self.name + '/' + name + # TODO: Investigate if this should be kept in InstallRequirement # Seems to be used only when VCS + downloads def archive(self, build_dir): + # type: (str) -> None assert self.source_dir create_archive = True archive_name = '%s-%s.zip' % (self.name, self.metadata["version"]) @@ -729,23 +870,35 @@ class InstallRequirement(object): if 'pip-egg-info' in dirnames: dirnames.remove('pip-egg-info') for dirname in dirnames: - dirname = os.path.join(dirpath, dirname) - name = self._clean_zip_name(dirname, dir) - zipdir = zipfile.ZipInfo(self.name + '/' + name + '/') + dir_arcname = self._get_archive_name(dirname, + parentdir=dirpath, + rootdir=dir) + zipdir = zipfile.ZipInfo(dir_arcname + '/') zipdir.external_attr = 0x1ED << 16 # 0o755 zip.writestr(zipdir, '') for filename in filenames: if filename == PIP_DELETE_MARKER_FILENAME: continue + file_arcname = self._get_archive_name(filename, + parentdir=dirpath, + rootdir=dir) filename = os.path.join(dirpath, filename) - name = self._clean_zip_name(filename, dir) - zip.write(filename, self.name + '/' + name) + zip.write(filename, file_arcname) zip.close() logger.info('Saved %s', display_path(archive_path)) - def install(self, install_options, global_options=None, root=None, - home=None, prefix=None, warn_script_location=True, - use_user_site=False, pycompile=True): + def install( + self, + install_options, # type: List[str] + global_options=None, # type: Optional[Sequence[str]] + root=None, # type: Optional[str] + home=None, # type: Optional[str] + prefix=None, # type: Optional[str] + warn_script_location=True, # type: bool + use_user_site=False, # type: bool + pycompile=True # type: bool + ): + # type: (...) -> None global_options = global_options if global_options is not None else [] if self.editable: self.install_editable( @@ -775,7 +928,8 @@ class InstallRequirement(object): self.options.get('install_options', []) if self.isolated: - global_options = global_options + ["--no-user-cfg"] + # https://github.com/python/mypy/issues/1174 + global_options = global_options + ["--no-user-cfg"] # type: ignore with TempDirectory(kind="record") as temp_dir: record_filename = os.path.join(temp_dir.path, 'install-record.txt') @@ -834,8 +988,15 @@ class InstallRequirement(object): with open(inst_files_path, 'w') as f: f.write('\n'.join(new_lines) + '\n') - def get_install_args(self, global_options, record_filename, root, prefix, - pycompile): + def get_install_args( + self, + global_options, # type: Sequence[str] + record_filename, # type: str + root, # type: Optional[str] + prefix, # type: Optional[str] + pycompile # type: bool + ): + # type: (...) -> List[str] sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) install_args = [sys_executable, "-u"] install_args.append('-c') diff --git a/pipenv/patched/notpip/_internal/req/req_set.py b/pipenv/patched/notpip/_internal/req/req_set.py index a65851ff..e7da5b71 100644 --- a/pipenv/patched/notpip/_internal/req/req_set.py +++ b/pipenv/patched/notpip/_internal/req/req_set.py @@ -5,29 +5,36 @@ from collections import OrderedDict from pipenv.patched.notpip._internal.exceptions import InstallationError from pipenv.patched.notpip._internal.utils.logging import indent_log +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING from pipenv.patched.notpip._internal.wheel import Wheel +if MYPY_CHECK_RUNNING: + from typing import Optional, List, Tuple, Dict, Iterable # noqa: F401 + from pipenv.patched.notpip._internal.req.req_install import InstallRequirement # noqa: F401 + + logger = logging.getLogger(__name__) class RequirementSet(object): def __init__(self, require_hashes=False, check_supported_wheels=True, ignore_compatibility=True): + # type: (bool, bool) -> None """Create a RequirementSet. """ - self.requirements = OrderedDict() + self.requirements = OrderedDict() # type: Dict[str, InstallRequirement] # noqa: E501 self.require_hashes = require_hashes self.check_supported_wheels = check_supported_wheels if ignore_compatibility: self.check_supported_wheels = False - self.ignore_compatibility = True if (check_supported_wheels is False or ignore_compatibility is True) else False + self.ignore_compatibility = (check_supported_wheels is False or ignore_compatibility is True) # Mapping of alias: real_name - self.requirement_aliases = {} - self.unnamed_requirements = [] - self.successfully_downloaded = [] - self.reqs_to_cleanup = [] + self.requirement_aliases = {} # type: Dict[str, str] + self.unnamed_requirements = [] # type: List[InstallRequirement] + self.successfully_downloaded = [] # type: List[InstallRequirement] + self.reqs_to_cleanup = [] # type: List[InstallRequirement] def __str__(self): reqs = [req for req in self.requirements.values() @@ -42,8 +49,13 @@ class RequirementSet(object): return ('<%s object; %d requirement(s): %s>' % (self.__class__.__name__, len(reqs), reqs_str)) - def add_requirement(self, install_req, parent_req_name=None, - extras_requested=None): + def add_requirement( + self, + install_req, # type: InstallRequirement + parent_req_name=None, # type: Optional[str] + extras_requested=None # type: Optional[Iterable[str]] + ): + # type: (...) -> Tuple[List[InstallRequirement], Optional[InstallRequirement]] # noqa: E501 """Add install_req as a requirement to install. :param parent_req_name: The name of the requirement that needed this @@ -155,6 +167,7 @@ class RequirementSet(object): return [existing_req], existing_req def has_requirement(self, project_name): + # type: (str) -> bool name = project_name.lower() if (name in self.requirements and not self.requirements[name].constraint or @@ -165,10 +178,12 @@ class RequirementSet(object): @property def has_requirements(self): + # type: () -> List[InstallRequirement] return list(req for req in self.requirements.values() if not req.constraint) or self.unnamed_requirements def get_requirement(self, project_name): + # type: (str) -> InstallRequirement for name in project_name, project_name.lower(): if name in self.requirements: return self.requirements[name] @@ -177,6 +192,7 @@ class RequirementSet(object): pass def cleanup_files(self): + # type: () -> None """Clean up files, remove builds.""" logger.debug('Cleaning up...') with indent_log(): diff --git a/pipenv/patched/notpip/_internal/req/req_tracker.py b/pipenv/patched/notpip/_internal/req/req_tracker.py index 6e0201fe..d17a4187 100644 --- a/pipenv/patched/notpip/_internal/req/req_tracker.py +++ b/pipenv/patched/notpip/_internal/req/req_tracker.py @@ -7,6 +7,12 @@ import logging import os from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Set, Iterator # noqa: F401 + from pipenv.patched.notpip._internal.req.req_install import InstallRequirement # noqa: F401 + from pipenv.patched.notpip._internal.models.link import Link # noqa: F401 logger = logging.getLogger(__name__) @@ -14,6 +20,7 @@ logger = logging.getLogger(__name__) class RequirementTracker(object): def __init__(self): + # type: () -> None self._root = os.environ.get('PIP_REQ_TRACKER') if self._root is None: self._temp_dir = TempDirectory(delete=False, kind='req-tracker') @@ -23,7 +30,7 @@ class RequirementTracker(object): else: self._temp_dir = None logger.debug('Re-using requirements tracker %r', self._root) - self._entries = set() + self._entries = set() # type: Set[InstallRequirement] def __enter__(self): return self @@ -32,10 +39,12 @@ class RequirementTracker(object): self.cleanup() def _entry_path(self, link): + # type: (Link) -> str hashed = hashlib.sha224(link.url_without_fragment.encode()).hexdigest() return os.path.join(self._root, hashed) def add(self, req): + # type: (InstallRequirement) -> None link = req.link info = str(req) entry_path = self._entry_path(link) @@ -54,12 +63,14 @@ class RequirementTracker(object): logger.debug('Added %s to build tracker %r', req, self._root) def remove(self, req): + # type: (InstallRequirement) -> None link = req.link self._entries.remove(req) os.unlink(self._entry_path(link)) logger.debug('Removed %s from build tracker %r', req, self._root) def cleanup(self): + # type: () -> None for req in set(self._entries): self.remove(req) remove = self._temp_dir is not None @@ -71,6 +82,7 @@ class RequirementTracker(object): @contextlib.contextmanager def track(self, req): + # type: (InstallRequirement) -> Iterator[None] self.add(req) yield self.remove(req) diff --git a/pipenv/patched/notpip/_internal/req/req_uninstall.py b/pipenv/patched/notpip/_internal/req/req_uninstall.py index 4cd15d84..ce80e6dc 100644 --- a/pipenv/patched/notpip/_internal/req/req_uninstall.py +++ b/pipenv/patched/notpip/_internal/req/req_uninstall.py @@ -15,9 +15,9 @@ from pipenv.patched.notpip._internal.utils.compat import WINDOWS, cache_from_sou from pipenv.patched.notpip._internal.utils.logging import indent_log from pipenv.patched.notpip._internal.utils.misc import ( FakeFile, ask, dist_in_usersite, dist_is_local, egg_link_path, is_local, - normalize_path, renames, + normalize_path, renames, rmtree, ) -from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory +from pipenv.patched.notpip._internal.utils.temp_dir import AdjacentTempDirectory, TempDirectory logger = logging.getLogger(__name__) @@ -86,16 +86,54 @@ def compact(paths): sep = os.path.sep short_paths = set() for path in sorted(paths, key=len): - should_add = any( + should_skip = any( path.startswith(shortpath.rstrip("*")) and path[len(shortpath.rstrip("*").rstrip(sep))] == sep for shortpath in short_paths ) - if not should_add: + if not should_skip: short_paths.add(path) return short_paths +def compress_for_rename(paths): + """Returns a set containing the paths that need to be renamed. + + This set may include directories when the original sequence of paths + included every file on disk. + """ + case_map = dict((os.path.normcase(p), p) for p in paths) + remaining = set(case_map) + unchecked = sorted(set(os.path.split(p)[0] + for p in case_map.values()), key=len) + wildcards = set() + + def norm_join(*a): + return os.path.normcase(os.path.join(*a)) + + for root in unchecked: + if any(os.path.normcase(root).startswith(w) + for w in wildcards): + # This directory has already been handled. + continue + + all_files = set() + all_subdirs = set() + for dirname, subdirs, files in os.walk(root): + all_subdirs.update(norm_join(root, dirname, d) + for d in subdirs) + all_files.update(norm_join(root, dirname, f) + for f in files) + # If all the files we found are in our remaining set of files to + # remove, then remove them from the latter set and add a wildcard + # for the directory. + if not (all_files - remaining): + remaining.difference_update(all_files) + wildcards.add(root + os.sep) + + return set(map(case_map.__getitem__, remaining)) | wildcards + + def compress_for_output_listing(paths): """Returns a tuple of 2 sets of which paths to display to user @@ -145,6 +183,111 @@ def compress_for_output_listing(paths): return will_remove, will_skip +class StashedUninstallPathSet(object): + """A set of file rename operations to stash files while + tentatively uninstalling them.""" + def __init__(self): + # Mapping from source file root to [Adjacent]TempDirectory + # for files under that directory. + self._save_dirs = {} + # (old path, new path) tuples for each move that may need + # to be undone. + self._moves = [] + + def _get_directory_stash(self, path): + """Stashes a directory. + + Directories are stashed adjacent to their original location if + possible, or else moved/copied into the user's temp dir.""" + + try: + save_dir = AdjacentTempDirectory(path) + save_dir.create() + except OSError: + save_dir = TempDirectory(kind="uninstall") + save_dir.create() + self._save_dirs[os.path.normcase(path)] = save_dir + + return save_dir.path + + def _get_file_stash(self, path): + """Stashes a file. + + If no root has been provided, one will be created for the directory + in the user's temp directory.""" + path = os.path.normcase(path) + head, old_head = os.path.dirname(path), None + save_dir = None + + while head != old_head: + try: + save_dir = self._save_dirs[head] + break + except KeyError: + pass + head, old_head = os.path.dirname(head), head + else: + # Did not find any suitable root + head = os.path.dirname(path) + save_dir = TempDirectory(kind='uninstall') + save_dir.create() + self._save_dirs[head] = save_dir + + relpath = os.path.relpath(path, head) + if relpath and relpath != os.path.curdir: + return os.path.join(save_dir.path, relpath) + return save_dir.path + + def stash(self, path): + """Stashes the directory or file and returns its new location. + """ + if os.path.isdir(path): + new_path = self._get_directory_stash(path) + else: + new_path = self._get_file_stash(path) + + self._moves.append((path, new_path)) + if os.path.isdir(path) and os.path.isdir(new_path): + # If we're moving a directory, we need to + # remove the destination first or else it will be + # moved to inside the existing directory. + # We just created new_path ourselves, so it will + # be removable. + os.rmdir(new_path) + renames(path, new_path) + return new_path + + def commit(self): + """Commits the uninstall by removing stashed files.""" + for _, save_dir in self._save_dirs.items(): + save_dir.cleanup() + self._moves = [] + self._save_dirs = {} + + def rollback(self): + """Undoes the uninstall by moving stashed files back.""" + for p in self._moves: + logging.info("Moving to %s\n from %s", *p) + + for new_path, path in self._moves: + try: + logger.debug('Replacing %s from %s', new_path, path) + if os.path.isfile(new_path): + os.unlink(new_path) + elif os.path.isdir(new_path): + rmtree(new_path) + renames(path, new_path) + except OSError as ex: + logger.error("Failed to restore %s", new_path) + logger.debug("Exception: %s", ex) + + self.commit() + + @property + def can_rollback(self): + return bool(self._moves) + + class UninstallPathSet(object): """A set of file paths to be removed in the uninstallation of a requirement.""" @@ -153,8 +296,7 @@ class UninstallPathSet(object): self._refuse = set() self.pth = {} self.dist = dist - self.save_dir = TempDirectory(kind="uninstall") - self._moved_paths = [] + self._moved_paths = StashedUninstallPathSet() def _permitted(self, path): """ @@ -192,11 +334,6 @@ class UninstallPathSet(object): else: self._refuse.add(pth_file) - def _stash(self, path): - return os.path.join( - self.save_dir.path, os.path.splitdrive(path)[1].lstrip(os.path.sep) - ) - def remove(self, auto_confirm=False, verbose=False): """Remove paths in ``self.paths`` with confirmation (unless ``auto_confirm`` is True).""" @@ -215,13 +352,14 @@ class UninstallPathSet(object): with indent_log(): if auto_confirm or self._allowed_to_proceed(verbose): - self.save_dir.create() + moved = self._moved_paths - for path in sorted(compact(self.paths)): - new_path = self._stash(path) + for_rename = compress_for_rename(self.paths) + + for path in sorted(compact(for_rename)): + moved.stash(path) logger.debug('Removing file or directory %s', path) - self._moved_paths.append(path) - renames(path, new_path) + for pth in self.pth.values(): pth.remove() @@ -251,29 +389,27 @@ class UninstallPathSet(object): _display('Would remove:', will_remove) _display('Would not remove (might be manually added):', will_skip) _display('Would not remove (outside of prefix):', self._refuse) + if verbose: + _display('Will actually move:', compress_for_rename(self.paths)) return ask('Proceed (y/n)? ', ('y', 'n')) == 'y' def rollback(self): """Rollback the changes previously made by remove().""" - if self.save_dir.path is None: + if not self._moved_paths.can_rollback: logger.error( "Can't roll back %s; was not uninstalled", self.dist.project_name, ) return False logger.info('Rolling back uninstall of %s', self.dist.project_name) - for path in self._moved_paths: - tmp_path = self._stash(path) - logger.debug('Replacing %s', path) - renames(tmp_path, path) + self._moved_paths.rollback() for pth in self.pth.values(): pth.rollback() def commit(self): """Remove temporary save dir: rollback will no longer be possible.""" - self.save_dir.cleanup() - self._moved_paths = [] + self._moved_paths.commit() @classmethod def from_dist(cls, dist): diff --git a/pipenv/patched/notpip/_internal/resolve.py b/pipenv/patched/notpip/_internal/resolve.py index b0d096f9..e42dd3d4 100644 --- a/pipenv/patched/notpip/_internal/resolve.py +++ b/pipenv/patched/notpip/_internal/resolve.py @@ -18,10 +18,23 @@ from pipenv.patched.notpip._internal.exceptions import ( BestVersionAlreadyInstalled, DistributionNotFound, HashError, HashErrors, UnsupportedPythonVersion, ) -from pipenv.patched.notpip._internal.req.constructors import install_req_from_req +from pipenv.patched.notpip._internal.req.constructors import install_req_from_req_string +from pipenv.patched.notpip._internal.req.req_install import InstallRequirement from pipenv.patched.notpip._internal.utils.logging import indent_log from pipenv.patched.notpip._internal.utils.misc import dist_in_usersite, ensure_dir from pipenv.patched.notpip._internal.utils.packaging import check_dist_requires_python +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Optional, DefaultDict, List, Set # noqa: F401 + from pipenv.patched.notpip._internal.download import PipSession # noqa: F401 + from pipenv.patched.notpip._internal.req.req_install import InstallRequirement # noqa: F401 + from pipenv.patched.notpip._internal.index import PackageFinder # noqa: F401 + from pipenv.patched.notpip._internal.req.req_set import RequirementSet # noqa: F401 + from pipenv.patched.notpip._internal.operations.prepare import ( # noqa: F401 + DistAbstraction, RequirementPreparer + ) + from pipenv.patched.notpip._internal.cache import WheelCache # noqa: F401 logger = logging.getLogger(__name__) @@ -33,9 +46,23 @@ class Resolver(object): _allowed_strategies = {"eager", "only-if-needed", "to-satisfy-only"} - def __init__(self, preparer, session, finder, wheel_cache, use_user_site, - ignore_dependencies, ignore_installed, ignore_requires_python, - force_reinstall, isolated, upgrade_strategy, ignore_compatibility=False): + def __init__( + self, + preparer, # type: RequirementPreparer + session, # type: PipSession + finder, # type: PackageFinder + wheel_cache, # type: Optional[WheelCache] + use_user_site, # type: bool + ignore_dependencies, # type: bool + ignore_installed, # type: bool + ignore_requires_python, # type: bool + force_reinstall, # type: bool + isolated, # type: bool + upgrade_strategy, # type: str + use_pep517=None, # type: Optional[bool] + ignore_compatibility=False, # type: bool + ): + # type: (...) -> None super(Resolver, self).__init__() assert upgrade_strategy in self._allowed_strategies @@ -47,7 +74,8 @@ class Resolver(object): # information about both sdist and wheels transparently. self.wheel_cache = wheel_cache - self.require_hashes = None # This is set in resolve + # This is set in resolve + self.require_hashes = None # type: Optional[bool] self.upgrade_strategy = upgrade_strategy self.force_reinstall = force_reinstall @@ -57,13 +85,16 @@ class Resolver(object): self.ignore_requires_python = ignore_requires_python self.ignore_compatibility = ignore_compatibility self.use_user_site = use_user_site + self.use_pep517 = use_pep517 self.requires_python = None if self.ignore_compatibility: self.ignore_requires_python = True - self._discovered_dependencies = defaultdict(list) + self._discovered_dependencies = \ + defaultdict(list) # type: DefaultDict[str, List] def resolve(self, requirement_set): + # type: (RequirementSet) -> None """Resolve what operations need to be done As a side-effect of this method, the packages (and their dependencies) @@ -98,7 +129,7 @@ class Resolver(object): # exceptions cannot be checked ahead of time, because # req.populate_link() needs to be called before we can make decisions # based on link type. - discovered_reqs = [] + discovered_reqs = [] # type: List[InstallRequirement] hash_errors = HashErrors() for req in chain(root_reqs, discovered_reqs): try: @@ -113,6 +144,7 @@ class Resolver(object): raise hash_errors def _is_upgrade_allowed(self, req): + # type: (InstallRequirement) -> bool if self.upgrade_strategy == "to-satisfy-only": return False elif self.upgrade_strategy == "eager": @@ -122,6 +154,7 @@ class Resolver(object): return req.is_direct def _set_req_to_reinstall(self, req): + # type: (InstallRequirement) -> None """ Set a requirement to be installed. """ @@ -133,6 +166,7 @@ class Resolver(object): # XXX: Stop passing requirement_set for options def _check_skip_installed(self, req_to_install): + # type: (InstallRequirement) -> Optional[str] """Check if req_to_install should be skipped. This will check if the req is installed, and whether we should upgrade @@ -185,6 +219,7 @@ class Resolver(object): return None def _get_abstract_dist_for(self, req): + # type: (InstallRequirement) -> DistAbstraction """Takes a InstallRequirement and returns a single AbstractDist \ representing a prepared variant of the same. """ @@ -241,7 +276,13 @@ class Resolver(object): return abstract_dist - def _resolve_one(self, requirement_set, req_to_install, ignore_requires_python=False): + def _resolve_one( + self, + requirement_set, # type: RequirementSet + req_to_install, # type: InstallRequirement + ignore_requires_python=False # type: bool + ): + # type: (...) -> List[InstallRequirement] """Prepare a single requirements file. :return: A list of additional InstallRequirements to also install. @@ -260,11 +301,11 @@ class Resolver(object): abstract_dist = self._get_abstract_dist_for(req_to_install) # Parse and return dependencies - dist = abstract_dist.dist(self.finder) + dist = abstract_dist.dist() try: check_dist_requires_python(dist) except UnsupportedPythonVersion as err: - if self.ignore_requires_python or self.ignore_compatibility: + if self.ignore_requires_python or ignore_requires_python or self.ignore_compatibility: logger.warning(err.args[0]) else: raise @@ -275,14 +316,16 @@ class Resolver(object): except TypeError: self.requires_python = None - more_reqs = [] + + more_reqs = [] # type: List[InstallRequirement] def add_req(subreq, extras_requested): - sub_install_req = install_req_from_req( + sub_install_req = install_req_from_req_string( str(subreq), req_to_install, isolated=self.isolated, wheel_cache=self.wheel_cache, + use_pep517=self.use_pep517 ) parent_req_name = req_to_install.name to_scan_again, add_to_parent = requirement_set.add_requirement( @@ -300,10 +343,10 @@ class Resolver(object): # We add req_to_install before its dependencies, so that we # can refer to it when adding dependencies. if not requirement_set.has_requirement(req_to_install.name): - # 'unnamed' requirements will get added here available_requested = sorted( set(dist.extras) & set(req_to_install.extras) ) + # 'unnamed' requirements will get added here req_to_install.is_direct = True requirement_set.add_requirement( req_to_install, parent_req_name=None, @@ -335,11 +378,12 @@ class Resolver(object): for available in available_requested: if hasattr(dist, '_DistInfoDistribution__dep_map'): for req in dist._DistInfoDistribution__dep_map[available]: - req = install_req_from_req( - str(req), + req = InstallRequirement( + req, req_to_install, isolated=self.isolated, wheel_cache=self.wheel_cache, + use_pep517=None ) more_reqs.append(req) @@ -353,6 +397,7 @@ class Resolver(object): return more_reqs def get_installation_order(self, req_set): + # type: (RequirementSet) -> List[InstallRequirement] """Create the installation order. The installation order is topological - requirements are installed @@ -363,7 +408,7 @@ class Resolver(object): # installs the user specified things in the order given, except when # dependencies must come earlier to achieve topological order. order = [] - ordered_reqs = set() + ordered_reqs = set() # type: Set[InstallRequirement] def schedule(req): if req.satisfied_by or req in ordered_reqs: diff --git a/pipenv/patched/notpip/_internal/utils/appdirs.py b/pipenv/patched/notpip/_internal/utils/appdirs.py index e8e14526..9ce3a1b3 100644 --- a/pipenv/patched/notpip/_internal/utils/appdirs.py +++ b/pipenv/patched/notpip/_internal/utils/appdirs.py @@ -10,9 +10,16 @@ import sys from pipenv.patched.notpip._vendor.six import PY2, text_type from pipenv.patched.notpip._internal.utils.compat import WINDOWS, expanduser +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import ( # noqa: F401 + List, Union + ) def user_cache_dir(appname): + # type: (str) -> str r""" Return full path to the user-specific cache dir for this application. @@ -61,6 +68,7 @@ def user_cache_dir(appname): def user_data_dir(appname, roaming=False): + # type: (str, bool) -> str r""" Return full path to the user-specific data dir for this application. @@ -113,6 +121,7 @@ def user_data_dir(appname, roaming=False): def user_config_dir(appname, roaming=True): + # type: (str, bool) -> str """Return full path to the user-specific config dir for this application. "appname" is the name of application. @@ -146,6 +155,7 @@ def user_config_dir(appname, roaming=True): # for the discussion regarding site_config_dirs locations # see def site_config_dirs(appname): + # type: (str) -> List[str] r"""Return a list of potential user-shared config dirs for this application. "appname" is the name of application. @@ -186,6 +196,7 @@ def site_config_dirs(appname): # -- Windows support functions -- def _get_win_folder_from_registry(csidl_name): + # type: (str) -> str """ This is a fallback technique at best. I'm not sure if using the registry for this guarantees us the correct answer for all CSIDL_* @@ -208,6 +219,7 @@ def _get_win_folder_from_registry(csidl_name): def _get_win_folder_with_ctypes(csidl_name): + # type: (str) -> str csidl_const = { "CSIDL_APPDATA": 26, "CSIDL_COMMON_APPDATA": 35, diff --git a/pipenv/patched/notpip/_internal/utils/compat.py b/pipenv/patched/notpip/_internal/utils/compat.py index 483bfdc8..1dad56b0 100644 --- a/pipenv/patched/notpip/_internal/utils/compat.py +++ b/pipenv/patched/notpip/_internal/utils/compat.py @@ -11,6 +11,11 @@ import sys from pipenv.patched.notpip._vendor.six import text_type +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Tuple, Text # noqa: F401 + try: import ipaddress except ImportError: @@ -18,8 +23,8 @@ except ImportError: from pipenv.patched.notpip._vendor import ipaddress # type: ignore except ImportError: import ipaddr as ipaddress # type: ignore - ipaddress.ip_address = ipaddress.IPAddress - ipaddress.ip_network = ipaddress.IPNetwork + ipaddress.ip_address = ipaddress.IPAddress # type: ignore + ipaddress.ip_network = ipaddress.IPNetwork # type: ignore __all__ = [ @@ -68,6 +73,7 @@ else: def console_to_str(data): + # type: (bytes) -> Text """Return a string, safe for output, of subprocess output. We assume the data is in the locale preferred encoding. @@ -88,13 +94,13 @@ def console_to_str(data): # Now try to decode the data - if we fail, warn the user and # decode with replacement. try: - s = data.decode(encoding) + decoded_data = data.decode(encoding) except UnicodeDecodeError: logger.warning( "Subprocess output does not appear to be encoded as %s", encoding, ) - s = data.decode(encoding, errors=backslashreplace_decode) + decoded_data = data.decode(encoding, errors=backslashreplace_decode) # Make sure we can print the output, by encoding it to the output # encoding with replacement of unencodable characters, and then @@ -112,20 +118,25 @@ def console_to_str(data): "encoding", None) if output_encoding: - s = s.encode(output_encoding, errors="backslashreplace") - s = s.decode(output_encoding) + output_encoded = decoded_data.encode( + output_encoding, + errors="backslashreplace" + ) + decoded_data = output_encoded.decode(output_encoding) - return s + return decoded_data if sys.version_info >= (3,): def native_str(s, replace=False): + # type: (str, bool) -> str if isinstance(s, bytes): return s.decode('utf-8', 'replace' if replace else 'strict') return s else: def native_str(s, replace=False): + # type: (str, bool) -> str # Replace is ignored -- unicode to UTF-8 can't fail if isinstance(s, text_type): return s.encode('utf-8') @@ -133,6 +144,7 @@ else: def get_path_uid(path): + # type: (str) -> int """ Return path's uid. @@ -174,6 +186,7 @@ else: def expanduser(path): + # type: (str) -> str """ Expand ~ and ~user constructions. @@ -199,6 +212,7 @@ WINDOWS = (sys.platform.startswith("win") or def samefile(file1, file2): + # type: (str, str) -> bool """Provide an alternative for os.path.samefile on Windows/Python2""" if hasattr(os.path, 'samefile'): return os.path.samefile(file1, file2) @@ -210,13 +224,15 @@ def samefile(file1, file2): if hasattr(shutil, 'get_terminal_size'): def get_terminal_size(): + # type: () -> Tuple[int, int] """ Returns a tuple (x, y) representing the width(x) and the height(y) in characters of the terminal window. """ - return tuple(shutil.get_terminal_size()) + return tuple(shutil.get_terminal_size()) # type: ignore else: def get_terminal_size(): + # type: () -> Tuple[int, int] """ Returns a tuple (x, y) representing the width(x) and the height(y) in characters of the terminal window. diff --git a/pipenv/patched/notpip/_internal/utils/deprecation.py b/pipenv/patched/notpip/_internal/utils/deprecation.py index b140ac71..2e309ec2 100644 --- a/pipenv/patched/notpip/_internal/utils/deprecation.py +++ b/pipenv/patched/notpip/_internal/utils/deprecation.py @@ -41,6 +41,7 @@ def _showwarning(message, category, filename, lineno, file=None, line=None): def install_warning_logger(): + # type: () -> None # Enable our Deprecation Warnings warnings.simplefilter("default", PipDeprecationWarning, append=True) diff --git a/pipenv/patched/notpip/_internal/utils/encoding.py b/pipenv/patched/notpip/_internal/utils/encoding.py index 56f60361..f03fc901 100644 --- a/pipenv/patched/notpip/_internal/utils/encoding.py +++ b/pipenv/patched/notpip/_internal/utils/encoding.py @@ -3,6 +3,11 @@ import locale import re import sys +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import List, Tuple, Text # noqa: F401 + BOMS = [ (codecs.BOM_UTF8, 'utf8'), (codecs.BOM_UTF16, 'utf16'), @@ -11,12 +16,13 @@ BOMS = [ (codecs.BOM_UTF32, 'utf32'), (codecs.BOM_UTF32_BE, 'utf32-be'), (codecs.BOM_UTF32_LE, 'utf32-le'), -] +] # type: List[Tuple[bytes, Text]] ENCODING_RE = re.compile(br'coding[:=]\s*([-\w.]+)') def auto_decode(data): + # type: (bytes) -> Text """Check a bytes string for a BOM to correctly detect the encoding Fallback to locale.getpreferredencoding(False) like open() on Python3""" diff --git a/pipenv/patched/notpip/_internal/utils/filesystem.py b/pipenv/patched/notpip/_internal/utils/filesystem.py index e8d6a2bb..d4aae97d 100644 --- a/pipenv/patched/notpip/_internal/utils/filesystem.py +++ b/pipenv/patched/notpip/_internal/utils/filesystem.py @@ -5,6 +5,7 @@ from pipenv.patched.notpip._internal.utils.compat import get_path_uid def check_path_owner(path): + # type: (str) -> bool # If we don't have a way to check the effective uid of this process, then # we'll just assume that we own the directory. if not hasattr(os, "geteuid"): @@ -26,3 +27,4 @@ def check_path_owner(path): return os.access(path, os.W_OK) else: previous, path = path, os.path.dirname(path) + return False # assume we don't own the path diff --git a/pipenv/patched/notpip/_internal/utils/glibc.py b/pipenv/patched/notpip/_internal/utils/glibc.py index ebcfc5be..e2b6d505 100644 --- a/pipenv/patched/notpip/_internal/utils/glibc.py +++ b/pipenv/patched/notpip/_internal/utils/glibc.py @@ -4,8 +4,14 @@ import ctypes import re import warnings +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Optional, Tuple # noqa: F401 + def glibc_version_string(): + # type: () -> Optional[str] "Returns glibc version string, or None if not using glibc." # ctypes.CDLL(None) internally calls dlopen(NULL), and as the dlopen @@ -32,6 +38,7 @@ def glibc_version_string(): # Separated out from have_compatible_glibc for easier unit testing def check_glibc_version(version_str, required_major, minimum_minor): + # type: (str, int, int) -> bool # Parse string and check against requested version. # # We use a regexp instead of str.split because we want to discard any @@ -48,7 +55,8 @@ def check_glibc_version(version_str, required_major, minimum_minor): def have_compatible_glibc(required_major, minimum_minor): - version_str = glibc_version_string() + # type: (int, int) -> bool + version_str = glibc_version_string() # type: Optional[str] if version_str is None: return False return check_glibc_version(version_str, required_major, minimum_minor) @@ -72,6 +80,7 @@ def have_compatible_glibc(required_major, minimum_minor): # misleading. Solution: instead of using platform, use our code that actually # works. def libc_ver(): + # type: () -> Tuple[str, str] """Try to determine the glibc version Returns a tuple of strings (lib, version) which default to empty strings diff --git a/pipenv/patched/notpip/_internal/utils/hashes.py b/pipenv/patched/notpip/_internal/utils/hashes.py index 818f5505..55cb8411 100644 --- a/pipenv/patched/notpip/_internal/utils/hashes.py +++ b/pipenv/patched/notpip/_internal/utils/hashes.py @@ -8,6 +8,18 @@ from pipenv.patched.notpip._internal.exceptions import ( HashMismatch, HashMissing, InstallationError, ) from pipenv.patched.notpip._internal.utils.misc import read_chunks +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import ( # noqa: F401 + Dict, List, BinaryIO, NoReturn, Iterator + ) + from pipenv.patched.notpip._vendor.six import PY3 + if PY3: + from hashlib import _Hash # noqa: F401 + else: + from hashlib import _hash as _Hash # noqa: F401 + # The recommended hash algo of the moment. Change this whenever the state of # the art changes; it won't hurt backward compatibility. @@ -25,6 +37,7 @@ class Hashes(object): """ def __init__(self, hashes=None): + # type: (Dict[str, List[str]]) -> None """ :param hashes: A dict of algorithm names pointing to lists of allowed hex digests @@ -32,6 +45,7 @@ class Hashes(object): self._allowed = {} if hashes is None else hashes def check_against_chunks(self, chunks): + # type: (Iterator[bytes]) -> None """Check good hashes against ones built from iterable of chunks of data. @@ -55,9 +69,11 @@ class Hashes(object): self._raise(gots) def _raise(self, gots): + # type: (Dict[str, _Hash]) -> NoReturn raise HashMismatch(self._allowed, gots) def check_against_file(self, file): + # type: (BinaryIO) -> None """Check good hashes against a file-like object Raise HashMismatch if none match. @@ -66,14 +82,17 @@ class Hashes(object): return self.check_against_chunks(read_chunks(file)) def check_against_path(self, path): + # type: (str) -> None with open(path, 'rb') as file: return self.check_against_file(file) def __nonzero__(self): + # type: () -> bool """Return whether I know any known-good hashes.""" return bool(self._allowed) def __bool__(self): + # type: () -> bool return self.__nonzero__() @@ -85,10 +104,12 @@ class MissingHashes(Hashes): """ def __init__(self): + # type: () -> None """Don't offer the ``hashes`` kwarg.""" # Pass our favorite hash in to generate a "gotten hash". With the # empty list, it will never match, so an error will always raise. super(MissingHashes, self).__init__(hashes={FAVORITE_HASH: []}) def _raise(self, gots): + # type: (Dict[str, _Hash]) -> NoReturn raise HashMissing(gots[FAVORITE_HASH].hexdigest()) diff --git a/pipenv/patched/notpip/_internal/utils/logging.py b/pipenv/patched/notpip/_internal/utils/logging.py index 576c4fa0..638c5ca9 100644 --- a/pipenv/patched/notpip/_internal/utils/logging.py +++ b/pipenv/patched/notpip/_internal/utils/logging.py @@ -1,9 +1,13 @@ from __future__ import absolute_import import contextlib +import errno import logging import logging.handlers import os +import sys + +from pipenv.patched.notpip._vendor.six import PY2 from pipenv.patched.notpip._internal.utils.compat import WINDOWS from pipenv.patched.notpip._internal.utils.misc import ensure_dir @@ -26,6 +30,48 @@ _log_state = threading.local() _log_state.indentation = 0 +class BrokenStdoutLoggingError(Exception): + """ + Raised if BrokenPipeError occurs for the stdout stream while logging. + """ + pass + + +# BrokenPipeError does not exist in Python 2 and, in addition, manifests +# differently in Windows and non-Windows. +if WINDOWS: + # In Windows, a broken pipe can show up as EINVAL rather than EPIPE: + # https://bugs.python.org/issue19612 + # https://bugs.python.org/issue30418 + if PY2: + def _is_broken_pipe_error(exc_class, exc): + """See the docstring for non-Windows Python 3 below.""" + return (exc_class is IOError and + exc.errno in (errno.EINVAL, errno.EPIPE)) + else: + # In Windows, a broken pipe IOError became OSError in Python 3. + def _is_broken_pipe_error(exc_class, exc): + """See the docstring for non-Windows Python 3 below.""" + return ((exc_class is BrokenPipeError) or # noqa: F821 + (exc_class is OSError and + exc.errno in (errno.EINVAL, errno.EPIPE))) +elif PY2: + def _is_broken_pipe_error(exc_class, exc): + """See the docstring for non-Windows Python 3 below.""" + return (exc_class is IOError and exc.errno == errno.EPIPE) +else: + # Then we are in the non-Windows Python 3 case. + def _is_broken_pipe_error(exc_class, exc): + """ + Return whether an exception is a broken pipe error. + + Args: + exc_class: an exception class. + exc: an exception instance. + """ + return (exc_class is BrokenPipeError) # noqa: F821 + + @contextlib.contextmanager def indent_log(num=2): """ @@ -44,15 +90,28 @@ def get_indentation(): class IndentingFormatter(logging.Formatter): + def __init__(self, *args, **kwargs): + """ + A logging.Formatter obeying containing indent_log contexts. + + :param add_timestamp: A bool indicating output lines should be prefixed + with their record's timestamp. + """ + self.add_timestamp = kwargs.pop("add_timestamp", False) + super(IndentingFormatter, self).__init__(*args, **kwargs) def format(self, record): """ Calls the standard formatter, but will indent all of the log messages by our current indentation level. """ - formatted = logging.Formatter.format(self, record) + formatted = super(IndentingFormatter, self).format(record) + prefix = '' + if self.add_timestamp: + prefix = self.formatTime(record, "%Y-%m-%dT%H:%M:%S ") + prefix += " " * get_indentation() formatted = "".join([ - (" " * get_indentation()) + line + prefix + line for line in formatted.splitlines(True) ]) return formatted @@ -83,6 +142,16 @@ class ColorizedStreamHandler(logging.StreamHandler): if WINDOWS and colorama: self.stream = colorama.AnsiToWin32(self.stream) + def _using_stdout(self): + """ + Return whether the handler is using sys.stdout. + """ + if WINDOWS and colorama: + # Then self.stream is an AnsiToWin32 object. + return self.stream.wrapped is sys.stdout + + return self.stream is sys.stdout + def should_color(self): # Don't colorize things if we do not have colorama or if told not to if not colorama or self._no_color: @@ -115,6 +184,19 @@ class ColorizedStreamHandler(logging.StreamHandler): return msg + # The logging module says handleError() can be customized. + def handleError(self, record): + exc_class, exc = sys.exc_info()[:2] + # If a broken pipe occurred while calling write() or flush() on the + # stdout stream in logging's Handler.emit(), then raise our special + # exception so we can handle it in main() instead of logging the + # broken pipe error and continuing. + if (exc_class and self._using_stdout() and + _is_broken_pipe_error(exc_class, exc)): + raise BrokenStdoutLoggingError() + + return super(ColorizedStreamHandler, self).handleError(record) + class BetterRotatingFileHandler(logging.handlers.RotatingFileHandler): @@ -134,6 +216,8 @@ class MaxLevelFilter(logging.Filter): def setup_logging(verbosity, no_color, user_log_file): """Configures and sets up all of the logging + + Returns the requested logging level, as its integer value. """ # Determine the level to be logging at. @@ -148,6 +232,8 @@ def setup_logging(verbosity, no_color, user_log_file): else: level = "INFO" + level_number = getattr(logging, level) + # The "root" logger should match the "console" level *unless* we also need # to log to a user log file. include_user_log = user_log_file is not None @@ -186,6 +272,11 @@ def setup_logging(verbosity, no_color, user_log_file): "()": IndentingFormatter, "format": "%(message)s", }, + "indent_with_timestamp": { + "()": IndentingFormatter, + "format": "%(message)s", + "add_timestamp": True, + }, }, "handlers": { "console": { @@ -208,7 +299,7 @@ def setup_logging(verbosity, no_color, user_log_file): "class": handler_classes["file"], "filename": additional_log_file, "delay": True, - "formatter": "indent", + "formatter": "indent_with_timestamp", }, }, "root": { @@ -223,3 +314,5 @@ def setup_logging(verbosity, no_color, user_log_file): } }, }) + + return level_number diff --git a/pipenv/patched/notpip/_internal/utils/misc.py b/pipenv/patched/notpip/_internal/utils/misc.py index 45e5204c..0a3237a2 100644 --- a/pipenv/patched/notpip/_internal/utils/misc.py +++ b/pipenv/patched/notpip/_internal/utils/misc.py @@ -25,6 +25,7 @@ from pipenv.patched.notpip._vendor.retrying import retry # type: ignore from pipenv.patched.notpip._vendor.six import PY2 from pipenv.patched.notpip._vendor.six.moves import input from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse +from pipenv.patched.notpip._vendor.six.moves.urllib.parse import unquote as urllib_unquote from pipenv.patched.notpip._internal.exceptions import CommandError, InstallationError from pipenv.patched.notpip._internal.locations import ( @@ -34,12 +35,23 @@ from pipenv.patched.notpip._internal.locations import ( from pipenv.patched.notpip._internal.utils.compat import ( WINDOWS, console_to_str, expanduser, stdlib_pkgs, ) +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING if PY2: from io import BytesIO as StringIO else: from io import StringIO +if MYPY_CHECK_RUNNING: + from typing import ( # noqa: F401 + Optional, Tuple, Iterable, List, Match, Union, Any, Mapping, Text, + AnyStr, Container + ) + from pipenv.patched.notpip._vendor.pkg_resources import Distribution # noqa: F401 + from pipenv.patched.notpip._internal.models.link import Link # noqa: F401 + from pipenv.patched.notpip._internal.utils.ui import SpinnerInterface # noqa: F401 + + __all__ = ['rmtree', 'display_path', 'backup_dir', 'ask', 'splitext', 'format_size', 'is_installable_dir', @@ -49,19 +61,21 @@ __all__ = ['rmtree', 'display_path', 'backup_dir', 'renames', 'get_prog', 'unzip_file', 'untar_file', 'unpack_file', 'call_subprocess', 'captured_stdout', 'ensure_dir', - 'ARCHIVE_EXTENSIONS', 'SUPPORTED_EXTENSIONS', + 'ARCHIVE_EXTENSIONS', 'SUPPORTED_EXTENSIONS', 'WHEEL_EXTENSION', 'get_installed_version', 'remove_auth_from_url'] logger = std_logging.getLogger(__name__) +WHEEL_EXTENSION = '.whl' BZ2_EXTENSIONS = ('.tar.bz2', '.tbz') XZ_EXTENSIONS = ('.tar.xz', '.txz', '.tlz', '.tar.lz', '.tar.lzma') -ZIP_EXTENSIONS = ('.zip', '.whl') +ZIP_EXTENSIONS = ('.zip', WHEEL_EXTENSION) TAR_EXTENSIONS = ('.tar.gz', '.tgz', '.tar') ARCHIVE_EXTENSIONS = ( ZIP_EXTENSIONS + BZ2_EXTENSIONS + TAR_EXTENSIONS + XZ_EXTENSIONS) SUPPORTED_EXTENSIONS = ZIP_EXTENSIONS + TAR_EXTENSIONS + try: import bz2 # noqa SUPPORTED_EXTENSIONS += BZ2_EXTENSIONS @@ -76,14 +90,8 @@ except ImportError: logger.debug('lzma module is not available') -def import_or_raise(pkg_or_module_string, ExceptionType, *args, **kwargs): - try: - return __import__(pkg_or_module_string) - except ImportError: - raise ExceptionType(*args, **kwargs) - - def ensure_dir(path): + # type: (AnyStr) -> None """os.path.makedirs without EEXIST.""" try: os.makedirs(path) @@ -93,6 +101,7 @@ def ensure_dir(path): def get_prog(): + # type: () -> str try: prog = os.path.basename(sys.argv[0]) if prog in ('__main__.py', '-c'): @@ -107,8 +116,9 @@ def get_prog(): # Retry every half second for up to 3 seconds @retry(stop_max_delay=3000, wait_fixed=500) def rmtree(dir, ignore_errors=False): - shutil.rmtree(dir, ignore_errors=ignore_errors, - onerror=rmtree_errorhandler) + # type: (str, bool) -> None + from pipenv.vendor.vistir.path import rmtree as vistir_rmtree, handle_remove_readonly + vistir_rmtree(dir, onerror=handle_remove_readonly, ignore_errors=ignore_errors) def rmtree_errorhandler(func, path, exc_info): @@ -127,6 +137,7 @@ def rmtree_errorhandler(func, path, exc_info): def display_path(path): + # type: (Union[str, Text]) -> str """Gives the display value for a given path, making it relative to cwd if possible.""" path = os.path.normcase(os.path.abspath(path)) @@ -139,6 +150,7 @@ def display_path(path): def backup_dir(dir, ext='.bak'): + # type: (str, str) -> str """Figure out the name of a directory to back up the given dir to (adding .bak, .bak2, etc)""" n = 1 @@ -150,6 +162,7 @@ def backup_dir(dir, ext='.bak'): def ask_path_exists(message, options): + # type: (str, Iterable[str]) -> str for action in os.environ.get('PIP_EXISTS_ACTION', '').split(): if action in options: return action @@ -157,6 +170,7 @@ def ask_path_exists(message, options): def ask(message, options): + # type: (str, Iterable[str]) -> str """Ask the message interactively, with the given possible responses""" while 1: if os.environ.get('PIP_NO_INPUT'): @@ -176,6 +190,7 @@ def ask(message, options): def format_size(bytes): + # type: (float) -> str if bytes > 1000 * 1000: return '%.1fMB' % (bytes / 1000.0 / 1000) elif bytes > 10 * 1000: @@ -187,6 +202,7 @@ def format_size(bytes): def is_installable_dir(path): + # type: (str) -> bool """Is path is a directory containing setup.py or pyproject.toml? """ if not os.path.isdir(path): @@ -201,6 +217,7 @@ def is_installable_dir(path): def is_svn_page(html): + # type: (Union[str, Text]) -> Optional[Match[Union[str, Text]]] """ Returns true if the page appears to be the index page of an svn repository """ @@ -209,6 +226,7 @@ def is_svn_page(html): def file_contents(filename): + # type: (str) -> Text with open(filename, 'rb') as fp: return fp.read().decode('utf-8') @@ -223,6 +241,7 @@ def read_chunks(file, size=io.DEFAULT_BUFFER_SIZE): def split_leading_dir(path): + # type: (Union[str, Text]) -> List[Union[str, Text]] path = path.lstrip('/').lstrip('\\') if '/' in path and (('\\' in path and path.find('/') < path.find('\\')) or '\\' not in path): @@ -230,10 +249,11 @@ def split_leading_dir(path): elif '\\' in path: return path.split('\\', 1) else: - return path, '' + return [path, ''] def has_leading_dir(paths): + # type: (Iterable[Union[str, Text]]) -> bool """Returns true if all the paths have the same leading path name (i.e., everything is in one subdirectory in an archive)""" common_prefix = None @@ -249,6 +269,7 @@ def has_leading_dir(paths): def normalize_path(path, resolve_symlinks=True): + # type: (str, bool) -> str """ Convert a path to its canonical, case-normalized, absolute version. @@ -262,6 +283,7 @@ def normalize_path(path, resolve_symlinks=True): def splitext(path): + # type: (str) -> Tuple[str, str] """Like os.path.splitext, but take off .tar too""" base, ext = posixpath.splitext(path) if base.lower().endswith('.tar'): @@ -271,6 +293,7 @@ def splitext(path): def renames(old, new): + # type: (str, str) -> None """Like os.renames(), but handles renaming across devices.""" # Implementation borrowed from os.renames(). head, tail = os.path.split(new) @@ -288,6 +311,7 @@ def renames(old, new): def is_local(path): + # type: (str) -> bool """ Return True if path is within sys.prefix, if we're running in a virtualenv. @@ -300,6 +324,7 @@ def is_local(path): def dist_is_local(dist): + # type: (Distribution) -> bool """ Return True if given Distribution object is installed locally (i.e. within current virtualenv). @@ -311,6 +336,7 @@ def dist_is_local(dist): def dist_in_usersite(dist): + # type: (Distribution) -> bool """ Return True if given Distribution is installed in user site. """ @@ -319,6 +345,7 @@ def dist_in_usersite(dist): def dist_in_site_packages(dist): + # type: (Distribution) -> bool """ Return True if given Distribution is installed in sysconfig.get_python_lib(). @@ -329,7 +356,10 @@ def dist_in_site_packages(dist): def dist_is_editable(dist): - """Is distribution an editable install?""" + # type: (Distribution) -> bool + """ + Return True if given Distribution is an editable install. + """ for path_item in sys.path: egg_link = os.path.join(path_item, dist.project_name + '.egg-link') if os.path.isfile(egg_link): @@ -342,6 +372,7 @@ def get_installed_distributions(local_only=True, include_editables=True, editables_only=False, user_only=False): + # type: (bool, Container[str], bool, bool, bool) -> List[Distribution] """ Return a list of installed Distribution objects. @@ -385,7 +416,8 @@ def get_installed_distributions(local_only=True, def user_test(d): return True - return [d for d in pkg_resources.working_set + # because of pkg_resources vendoring, mypy cannot find stub in typeshed + return [d for d in pkg_resources.working_set # type: ignore if local_test(d) and d.key not in skip and editable_test(d) and @@ -395,6 +427,7 @@ def get_installed_distributions(local_only=True, def egg_link_path(dist): + # type: (Distribution) -> Optional[str] """ Return the path for the .egg-link file if it exists, otherwise, None. @@ -429,9 +462,11 @@ def egg_link_path(dist): egglink = os.path.join(site, dist.project_name) + '.egg-link' if os.path.isfile(egglink): return egglink + return None def dist_location(dist): + # type: (Distribution) -> str """ Get the site-packages location of this distribution. Generally this is dist.location, except in the case of develop-installed @@ -453,6 +488,7 @@ def current_umask(): def unzip_file(filename, location, flatten=True): + # type: (str, str, bool) -> None """ Unzip the file (with path `filename`) to the destination `location`. All files are written based on system defaults and umask (i.e. permissions are @@ -468,7 +504,6 @@ def unzip_file(filename, location, flatten=True): leading = has_leading_dir(zip.namelist()) and flatten for info in zip.infolist(): name = info.filename - data = zip.read(name) fn = name if leading: fn = split_leading_dir(name)[1] @@ -479,9 +514,12 @@ def unzip_file(filename, location, flatten=True): ensure_dir(fn) else: ensure_dir(dir) - fp = open(fn, 'wb') + # Don't use read() to avoid allocating an arbitrarily large + # chunk of memory for the file's content + fp = zip.open(name) try: - fp.write(data) + with open(fn, 'wb') as destfp: + shutil.copyfileobj(fp, destfp) finally: fp.close() mode = info.external_attr >> 16 @@ -496,6 +534,7 @@ def unzip_file(filename, location, flatten=True): def untar_file(filename, location): + # type: (str, str) -> None """ Untar the file (with path `filename`) to the destination `location`. All files are written based on system defaults and umask (i.e. permissions @@ -520,23 +559,21 @@ def untar_file(filename, location): mode = 'r:*' tar = tarfile.open(filename, mode) try: - # note: python<=2.5 doesn't seem to know about pax headers, filter them leading = has_leading_dir([ member.name for member in tar.getmembers() - if member.name != 'pax_global_header' ]) for member in tar.getmembers(): fn = member.name - if fn == 'pax_global_header': - continue if leading: - fn = split_leading_dir(fn)[1] + # https://github.com/python/mypy/issues/1174 + fn = split_leading_dir(fn)[1] # type: ignore path = os.path.join(location, fn) if member.isdir(): ensure_dir(path) elif member.issym(): try: - tar._extract_member(member, path) + # https://github.com/python/typeshed/issues/2673 + tar._extract_member(member, path) # type: ignore except Exception as exc: # Some corrupt tar files seem to produce this # (specifically bad symlinks) @@ -561,7 +598,8 @@ def untar_file(filename, location): shutil.copyfileobj(fp, destfp) fp.close() # Update the timestamp (useful for cython compiled files) - tar.utime(member, path) + # https://github.com/python/typeshed/issues/2673 + tar.utime(member, path) # type: ignore # member have any execute permissions for user/group/world? if member.mode & 0o111: # make dest file have execute for user/group/world @@ -571,7 +609,13 @@ def untar_file(filename, location): tar.close() -def unpack_file(filename, location, content_type, link): +def unpack_file( + filename, # type: str + location, # type: str + content_type, # type: Optional[str] + link # type: Optional[Link] +): + # type: (...) -> None filename = os.path.realpath(filename) if (content_type == 'application/zip' or filename.lower().endswith(ZIP_EXTENSIONS) or @@ -604,15 +648,27 @@ def unpack_file(filename, location, content_type, link): ) -def call_subprocess(cmd, show_stdout=True, cwd=None, - on_returncode='raise', - command_desc=None, - extra_environ=None, unset_environ=None, spinner=None): +def call_subprocess( + cmd, # type: List[str] + show_stdout=True, # type: bool + cwd=None, # type: Optional[str] + on_returncode='raise', # type: str + extra_ok_returncodes=None, # type: Optional[Iterable[int]] + command_desc=None, # type: Optional[str] + extra_environ=None, # type: Optional[Mapping[str, Any]] + unset_environ=None, # type: Optional[Iterable[str]] + spinner=None # type: Optional[SpinnerInterface] +): + # type: (...) -> Optional[Text] """ Args: + extra_ok_returncodes: an iterable of integer return codes that are + acceptable, in addition to 0. Defaults to None, which means []. unset_environ: an iterable of environment variable names to unset prior to calling subprocess.Popen(). """ + if extra_ok_returncodes is None: + extra_ok_returncodes = [] if unset_environ is None: unset_environ = [] # This function's handling of subprocess output is confusing and I @@ -689,7 +745,7 @@ def call_subprocess(cmd, show_stdout=True, cwd=None, spinner.finish("error") else: spinner.finish("done") - if proc.returncode: + if proc.returncode and proc.returncode not in extra_ok_returncodes: if on_returncode == 'raise': if (logger.getEffectiveLevel() > std_logging.DEBUG and not show_stdout): @@ -715,9 +771,11 @@ def call_subprocess(cmd, show_stdout=True, cwd=None, repr(on_returncode)) if not show_stdout: return ''.join(all_output) + return None def read_text_file(filename): + # type: (str) -> str """Return the contents of *filename*. Try to decode the file contents with utf-8, the preferred system encoding @@ -732,12 +790,13 @@ def read_text_file(filename): encodings = ['utf-8', locale.getpreferredencoding(False), 'latin1'] for enc in encodings: try: - data = data.decode(enc) + # https://github.com/python/mypy/issues/1174 + data = data.decode(enc) # type: ignore except UnicodeDecodeError: continue break - assert type(data) != bytes # Latin1 should have worked. + assert not isinstance(data, bytes) # Latin1 should have worked. return data @@ -805,6 +864,13 @@ def captured_stdout(): return captured_output('stdout') +def captured_stderr(): + """ + See captured_stdout(). + """ + return captured_output('stderr') + + class cached_property(object): """A property that is only computed once per instance and then replaces itself with an ordinary attribute. Deleting the attribute resets the @@ -856,13 +922,15 @@ def enum(*sequential, **named): return type('Enum', (), enums) -def make_vcs_requirement_url(repo_url, rev, egg_project_name, subdir=None): +def make_vcs_requirement_url(repo_url, rev, project_name, subdir=None): """ Return the URL for a VCS requirement. Args: repo_url: the remote VCS url, with any needed VCS prefix (e.g. "git+"). + project_name: the (unescaped) project name. """ + egg_project_name = pkg_resources.to_filename(project_name) req = '{}@{}#egg={}'.format(repo_url, rev, egg_project_name) if subdir: req += '&subdirectory={}'.format(subdir) @@ -887,22 +955,36 @@ def split_auth_from_netloc(netloc): # Split from the left because that's how urllib.parse.urlsplit() # behaves if more than one : is present (which again can be checked # using the password attribute of the return value) - user_pass = tuple(auth.split(':', 1)) + user_pass = auth.split(':', 1) else: user_pass = auth, None + user_pass = tuple( + None if x is None else urllib_unquote(x) for x in user_pass + ) + return netloc, user_pass -def remove_auth_from_url(url): - # Return a copy of url with 'username:password@' removed. - # username/pass params are passed to subversion through flags - # and are not recognized in the url. +def redact_netloc(netloc): + # type: (str) -> str + """ + Replace the password in a netloc with "****", if it exists. - # parsed url + For example, "user:pass@example.com" returns "user:****@example.com". + """ + netloc, (user, password) = split_auth_from_netloc(netloc) + if user is None: + return netloc + password = '' if password is None else ':****' + return '{user}{password}@{netloc}'.format(user=urllib_parse.quote(user), + password=password, + netloc=netloc) + + +def _transform_url(url, transform_netloc): purl = urllib_parse.urlsplit(url) - netloc, user_pass = split_auth_from_netloc(purl.netloc) - + netloc = transform_netloc(purl.netloc) # stripped url url_pieces = ( purl.scheme, netloc, purl.path, purl.query, purl.fragment @@ -911,6 +993,24 @@ def remove_auth_from_url(url): return surl +def _get_netloc(netloc): + return split_auth_from_netloc(netloc)[0] + + +def remove_auth_from_url(url): + # type: (str) -> str + # Return a copy of url with 'username:password@' removed. + # username/pass params are passed to subversion through flags + # and are not recognized in the url. + return _transform_url(url, _get_netloc) + + +def redact_password_from_url(url): + # type: (str) -> str + """Replace the password in a given url with ****.""" + return _transform_url(url, redact_netloc) + + def protect_pip_from_modification_on_windows(modifying_pip): """Protection of pip.exe from modification on Windows diff --git a/pipenv/patched/notpip/_internal/utils/outdated.py b/pipenv/patched/notpip/_internal/utils/outdated.py index f8b1fe04..83dc58cc 100644 --- a/pipenv/patched/notpip/_internal/utils/outdated.py +++ b/pipenv/patched/notpip/_internal/utils/outdated.py @@ -13,6 +13,13 @@ from pipenv.patched.notpip._internal.index import PackageFinder from pipenv.patched.notpip._internal.utils.compat import WINDOWS from pipenv.patched.notpip._internal.utils.filesystem import check_path_owner from pipenv.patched.notpip._internal.utils.misc import ensure_dir, get_installed_version +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + import optparse # noqa: F401 + from typing import Any, Dict # noqa: F401 + from pipenv.patched.notpip._internal.download import PipSession # noqa: F401 + SELFCHECK_DATE_FMT = "%Y-%m-%dT%H:%M:%SZ" @@ -22,7 +29,8 @@ logger = logging.getLogger(__name__) class SelfCheckState(object): def __init__(self, cache_dir): - self.state = {} + # type: (str) -> None + self.state = {} # type: Dict[str, Any] self.statefile_path = None # Try to load the existing state @@ -37,6 +45,7 @@ class SelfCheckState(object): pass def save(self, pypi_version, current_time): + # type: (str, datetime.datetime) -> None # If we do not have a path to cache in, don't bother saving. if not self.statefile_path: return @@ -68,6 +77,7 @@ class SelfCheckState(object): def was_installed_by_pip(pkg): + # type: (str) -> bool """Checks whether pkg was installed by pip This is used not to display the upgrade message when pip is in fact @@ -82,6 +92,7 @@ def was_installed_by_pip(pkg): def pip_version_check(session, options): + # type: (PipSession, optparse.Values) -> None """Check for an update for pip. Limit the frequency of checks to once per week. State is stored either in @@ -116,7 +127,6 @@ def pip_version_check(session, options): index_urls=[options.index_url] + options.extra_index_urls, allow_all_prereleases=False, # Explicitly set to False trusted_hosts=options.trusted_hosts, - process_dependency_links=options.process_dependency_links, session=session, ) all_candidates = finder.find_all_candidates("pip") diff --git a/pipenv/patched/notpip/_internal/utils/packaging.py b/pipenv/patched/notpip/_internal/utils/packaging.py index d1e8ecaa..dc944529 100644 --- a/pipenv/patched/notpip/_internal/utils/packaging.py +++ b/pipenv/patched/notpip/_internal/utils/packaging.py @@ -2,18 +2,26 @@ from __future__ import absolute_import import logging import sys -from email.parser import FeedParser # type: ignore +from email.parser import FeedParser from pipenv.patched.notpip._vendor import pkg_resources from pipenv.patched.notpip._vendor.packaging import specifiers, version from pipenv.patched.notpip._internal import exceptions from pipenv.patched.notpip._internal.utils.misc import display_path +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Optional # noqa: F401 + from email.message import Message # noqa: F401 + from pipenv.patched.notpip._vendor.pkg_resources import Distribution # noqa: F401 + logger = logging.getLogger(__name__) def check_requires_python(requires_python): + # type: (Optional[str]) -> bool """ Check if the python version in use match the `requires_python` specifier. @@ -34,6 +42,7 @@ def check_requires_python(requires_python): def get_metadata(dist): + # type: (Distribution) -> Message if (isinstance(dist, pkg_resources.DistInfoDistribution) and dist.has_metadata('METADATA')): metadata = dist.get_metadata('METADATA') @@ -48,7 +57,7 @@ def get_metadata(dist): return feed_parser.close() -def check_dist_requires_python(dist, absorb=True): +def check_dist_requires_python(dist, absorb=False): pkg_info_dict = get_metadata(dist) requires_python = pkg_info_dict.get('Requires-Python') if absorb: @@ -70,6 +79,7 @@ def check_dist_requires_python(dist, absorb=True): def get_installer(dist): + # type: (Distribution) -> str if dist.has_metadata('INSTALLER'): for line in dist.get_metadata_lines('INSTALLER'): if line.strip(): diff --git a/pipenv/patched/notpip/_internal/utils/temp_dir.py b/pipenv/patched/notpip/_internal/utils/temp_dir.py index 893dc975..d8121a59 100644 --- a/pipenv/patched/notpip/_internal/utils/temp_dir.py +++ b/pipenv/patched/notpip/_internal/utils/temp_dir.py @@ -1,5 +1,7 @@ from __future__ import absolute_import +import errno +import itertools import logging import os.path import tempfile @@ -57,7 +59,7 @@ class TempDirectory(object): self, self._cleanup, self.path, - warn_message=None + warn_message = None ) else: self._finalizer = None @@ -74,7 +76,7 @@ class TempDirectory(object): self.cleanup() def create(self): - """Create a temporary directory and store it's path in self.path + """Create a temporary directory and store its path in self.path """ if self.path is not None: logger.debug( @@ -112,3 +114,75 @@ class TempDirectory(object): pass else: self.path = None + + +class AdjacentTempDirectory(TempDirectory): + """Helper class that creates a temporary directory adjacent to a real one. + + Attributes: + original + The original directory to create a temp directory for. + path + After calling create() or entering, contains the full + path to the temporary directory. + delete + Whether the directory should be deleted when exiting + (when used as a contextmanager) + + """ + # The characters that may be used to name the temp directory + # We always prepend a ~ and then rotate through these until + # a usable name is found. + # pkg_resources raises a different error for .dist-info folder + # with leading '-' and invalid metadata + LEADING_CHARS = "-~.=%0123456789" + + def __init__(self, original, delete=None): + super(AdjacentTempDirectory, self).__init__(delete=delete) + self.original = original.rstrip('/\\') + + @classmethod + def _generate_names(cls, name): + """Generates a series of temporary names. + + The algorithm replaces the leading characters in the name + with ones that are valid filesystem characters, but are not + valid package names (for both Python and pip definitions of + package). + """ + for i in range(1, len(name)): + for candidate in itertools.combinations_with_replacement( + cls.LEADING_CHARS, i - 1): + new_name = '~' + ''.join(candidate) + name[i:] + if new_name != name: + yield new_name + + # If we make it this far, we will have to make a longer name + for i in range(len(cls.LEADING_CHARS)): + for candidate in itertools.combinations_with_replacement( + cls.LEADING_CHARS, i): + new_name = '~' + ''.join(candidate) + name + if new_name != name: + yield new_name + + def create(self): + root, name = os.path.split(self.original) + for candidate in self._generate_names(name): + path = os.path.join(root, candidate) + try: + os.mkdir(path) + except OSError as ex: + # Continue if the name exists already + if ex.errno != errno.EEXIST: + raise + else: + self.path = os.path.realpath(path) + break + + if not self.path: + # Final fallback on the default behavior. + self.path = os.path.realpath( + tempfile.mkdtemp(prefix="pip-{}-".format(self.kind)) + ) + self._register_finalizer() + logger.debug("Created temporary directory: {}".format(self.path)) diff --git a/pipenv/patched/notpip/_internal/utils/ui.py b/pipenv/patched/notpip/_internal/utils/ui.py index 6eebd17d..18119e0e 100644 --- a/pipenv/patched/notpip/_internal/utils/ui.py +++ b/pipenv/patched/notpip/_internal/utils/ui.py @@ -21,7 +21,7 @@ from pipenv.patched.notpip._internal.utils.misc import format_size from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Any # noqa: F401 + from typing import Any, Iterator, IO # noqa: F401 try: from pipenv.patched.notpip._vendor import colorama @@ -203,7 +203,7 @@ class BaseDownloadProgressBar(WindowsMixin, InterruptibleMixin, class DefaultDownloadProgressBar(BaseDownloadProgressBar, - _BaseBar): # type: ignore + _BaseBar): pass @@ -292,6 +292,7 @@ def DownloadProgressProvider(progress_bar, max=None): @contextlib.contextmanager def hidden_cursor(file): + # type: (IO) -> Iterator[None] # The Windows terminal does not support the hide/show cursor ANSI codes, # even via colorama. So don't even try. if WINDOWS: @@ -311,19 +312,32 @@ def hidden_cursor(file): class RateLimiter(object): def __init__(self, min_update_interval_seconds): + # type: (float) -> None self._min_update_interval_seconds = min_update_interval_seconds - self._last_update = 0 + self._last_update = 0 # type: float def ready(self): + # type: () -> bool now = time.time() delta = now - self._last_update return delta >= self._min_update_interval_seconds def reset(self): + # type: () -> None self._last_update = time.time() -class InteractiveSpinner(object): +class SpinnerInterface(object): + def spin(self): + # type: () -> None + raise NotImplementedError() + + def finish(self, final_status): + # type: (str) -> None + raise NotImplementedError() + + +class InteractiveSpinner(SpinnerInterface): def __init__(self, message, file=None, spin_chars="-\\|/", # Empirically, 8 updates/second looks nice min_update_interval_seconds=0.125): @@ -352,6 +366,7 @@ class InteractiveSpinner(object): self._rate_limiter.reset() def spin(self): + # type: () -> None if self._finished: return if not self._rate_limiter.ready(): @@ -359,6 +374,7 @@ class InteractiveSpinner(object): self._write(next(self._spin_cycle)) def finish(self, final_status): + # type: (str) -> None if self._finished: return self._write(final_status) @@ -371,8 +387,9 @@ class InteractiveSpinner(object): # We still print updates occasionally (once every 60 seconds by default) to # act as a keep-alive for systems like Travis-CI that take lack-of-output as # an indication that a task has frozen. -class NonInteractiveSpinner(object): +class NonInteractiveSpinner(SpinnerInterface): def __init__(self, message, min_update_interval_seconds=60): + # type: (str, float) -> None self._message = message self._finished = False self._rate_limiter = RateLimiter(min_update_interval_seconds) @@ -384,6 +401,7 @@ class NonInteractiveSpinner(object): logger.info("%s: %s", self._message, status) def spin(self): + # type: () -> None if self._finished: return if not self._rate_limiter.ready(): @@ -391,6 +409,7 @@ class NonInteractiveSpinner(object): self._update("still running...") def finish(self, final_status): + # type: (str) -> None if self._finished: return self._update("finished with status '%s'" % (final_status,)) @@ -399,13 +418,14 @@ class NonInteractiveSpinner(object): @contextlib.contextmanager def open_spinner(message): + # type: (str) -> Iterator[SpinnerInterface] # Interactive spinner goes directly to sys.stdout rather than being routed # through the logging system, but it acts like it has level INFO, # i.e. it's only displayed if we're at level INFO or better. # Non-interactive spinner goes through the logging system, so it is always # in sync with logging configuration. if sys.stdout.isatty() and logger.getEffectiveLevel() <= logging.INFO: - spinner = InteractiveSpinner(message) + spinner = InteractiveSpinner(message) # type: SpinnerInterface else: spinner = NonInteractiveSpinner(message) try: diff --git a/pipenv/patched/notpip/_internal/vcs/__init__.py b/pipenv/patched/notpip/_internal/vcs/__init__.py index 5aeac633..06d36148 100644 --- a/pipenv/patched/notpip/_internal/vcs/__init__.py +++ b/pipenv/patched/notpip/_internal/vcs/__init__.py @@ -16,15 +16,23 @@ from pipenv.patched.notpip._internal.utils.misc import ( from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Dict, Optional, Tuple # noqa: F401 - from pipenv.patched.notpip._internal.cli.base_command import Command # noqa: F401 + from typing import ( # noqa: F401 + Any, Dict, Iterable, List, Mapping, Optional, Text, Tuple, Type + ) + from pipenv.patched.notpip._internal.utils.ui import SpinnerInterface # noqa: F401 -__all__ = ['vcs', 'get_src_requirement'] + AuthInfo = Tuple[Optional[str], Optional[str]] + +__all__ = ['vcs'] logger = logging.getLogger(__name__) +class RemoteNotFoundError(Exception): + pass + + class RevOptions(object): """ @@ -35,6 +43,7 @@ class RevOptions(object): """ def __init__(self, vcs, rev=None, extra_args=None): + # type: (VersionControl, Optional[str], Optional[List[str]]) -> None """ Args: vcs: a VersionControl object. @@ -53,16 +62,18 @@ class RevOptions(object): @property def arg_rev(self): + # type: () -> Optional[str] if self.rev is None: return self.vcs.default_arg_rev return self.rev def to_args(self): + # type: () -> List[str] """ Return the VCS-specific command arguments. """ - args = [] + args = [] # type: List[str] rev = self.arg_rev if rev is not None: args += self.vcs.get_base_rev_args(rev) @@ -71,12 +82,14 @@ class RevOptions(object): return args def to_display(self): + # type: () -> str if not self.rev: return '' return ' (to revision {})'.format(self.rev) def make_new(self, rev): + # type: (str) -> RevOptions """ Make a copy of the current instance, but with a new rev. @@ -87,10 +100,11 @@ class RevOptions(object): class VcsSupport(object): - _registry = {} # type: Dict[str, Command] + _registry = {} # type: Dict[str, Type[VersionControl]] schemes = ['ssh', 'git', 'hg', 'bzr', 'sftp', 'svn'] def __init__(self): + # type: () -> None # Register more schemes with urlparse for various version control # systems urllib_parse.uses_netloc.extend(self.schemes) @@ -104,20 +118,24 @@ class VcsSupport(object): @property def backends(self): + # type: () -> List[Type[VersionControl]] return list(self._registry.values()) @property def dirnames(self): + # type: () -> List[str] return [backend.dirname for backend in self.backends] @property def all_schemes(self): - schemes = [] + # type: () -> List[str] + schemes = [] # type: List[str] for backend in self.backends: schemes.extend(backend.schemes) return schemes def register(self, cls): + # type: (Type[VersionControl]) -> None if not hasattr(cls, 'name'): logger.warning('Cannot register VCS %s', cls.__name__) return @@ -126,6 +144,7 @@ class VcsSupport(object): logger.debug('Registered VCS backend: %s', cls.name) def unregister(self, cls=None, name=None): + # type: (Optional[Type[VersionControl]], Optional[str]) -> None if name in self._registry: del self._registry[name] elif cls in self._registry.values(): @@ -133,27 +152,24 @@ class VcsSupport(object): else: logger.warning('Cannot unregister because no class or name given') - def get_backend_name(self, location): + def get_backend_type(self, location): + # type: (str) -> Optional[Type[VersionControl]] """ - Return the name of the version control backend if found at given - location, e.g. vcs.get_backend_name('/path/to/vcs/checkout') + Return the type of the version control backend if found at given + location, e.g. vcs.get_backend_type('/path/to/vcs/checkout') """ for vc_type in self._registry.values(): if vc_type.controls_location(location): logger.debug('Determine that %s uses VCS: %s', location, vc_type.name) - return vc_type.name + return vc_type return None def get_backend(self, name): + # type: (str) -> Optional[Type[VersionControl]] name = name.lower() if name in self._registry: return self._registry[name] - - def get_backend_from_location(self, location): - vc_type = self.get_backend_name(location) - if vc_type: - return self.get_backend(vc_type) return None @@ -163,6 +179,7 @@ vcs = VcsSupport() class VersionControl(object): name = '' dirname = '' + repo_name = '' # List of supported schemes for this Version Control schemes = () # type: Tuple[str, ...] # Iterable of environment variable names to pass to call_subprocess(). @@ -183,6 +200,7 @@ class VersionControl(object): raise NotImplementedError def make_rev_options(self, rev=None, extra_args=None): + # type: (Optional[str], Optional[List[str]]) -> RevOptions """ Return a RevOptions object. @@ -192,13 +210,15 @@ class VersionControl(object): """ return RevOptions(self, rev, extra_args=extra_args) - def _is_local_repository(self, repo): + @classmethod + def _is_local_repository(cls, repo): + # type: (str) -> bool """ posix absolute paths start with os.path.sep, win32 ones start with drive (like c:\\folder) """ drive, tail = os.path.splitdrive(repo) - return repo.startswith(os.path.sep) or drive + return repo.startswith(os.path.sep) or bool(drive) def export(self, location): """ @@ -226,6 +246,7 @@ class VersionControl(object): return netloc, (None, None) def get_url_rev_and_auth(self, url): + # type: (str) -> Tuple[str, Optional[str], AuthInfo] """ Parse the repository URL to use, and return the URL, revision, and auth info to use. @@ -255,6 +276,7 @@ class VersionControl(object): return [] def get_url_rev_options(self, url): + # type: (str) -> Tuple[str, RevOptions] """ Return the URL and RevOptions object to use in obtain() and in some cases export(), as a tuple (url, rev_options). @@ -267,6 +289,7 @@ class VersionControl(object): return url, rev_options def normalize_url(self, url): + # type: (str) -> str """ Normalize a URL for comparison by unquoting it and removing any trailing slash. @@ -274,6 +297,7 @@ class VersionControl(object): return urllib_parse.unquote(url).rstrip('/') def compare_urls(self, url1, url2): + # type: (str, str) -> bool """ Compare two repo URLs for identity, ignoring incidental differences. """ @@ -319,6 +343,7 @@ class VersionControl(object): raise NotImplementedError def obtain(self, dest): + # type: (str) -> None """ Install or update in editable mode the package represented by this VersionControl object. @@ -334,7 +359,7 @@ class VersionControl(object): rev_display = rev_options.to_display() if self.is_repository_directory(dest): - existing_url = self.get_url(dest) + existing_url = self.get_remote_url(dest) if self.compare_urls(existing_url, url): logger.debug( '%s in %s exists, and has correct URL (%s)', @@ -370,7 +395,9 @@ class VersionControl(object): self.name, self.repo_name, ) - prompt = ('(i)gnore, (w)ipe, (b)ackup ', ('i', 'w', 'b')) + # https://github.com/python/mypy/issues/1174 + prompt = ('(i)gnore, (w)ipe, (b)ackup ', # type: ignore + ('i', 'w', 'b')) logger.warning( 'The plan is to install the %s repository %s', @@ -409,6 +436,7 @@ class VersionControl(object): self.switch(dest, url, rev_options) def unpack(self, location): + # type: (str) -> None """ Clean up current location and download the url repository (and vcs infos) into location @@ -417,7 +445,8 @@ class VersionControl(object): rmtree(location) self.obtain(location) - def get_src_requirement(self, dist, location): + @classmethod + def get_src_requirement(cls, location, project_name): """ Return a string representing the requirement needed to redownload the files currently present in location, something @@ -426,33 +455,49 @@ class VersionControl(object): """ raise NotImplementedError - def get_url(self, location): + @classmethod + def get_remote_url(cls, location): """ Return the url used at location + + Raises RemoteNotFoundError if the repository does not have a remote + url configured. """ raise NotImplementedError - def get_revision(self, location): + @classmethod + def get_revision(cls, location): """ Return the current commit id of the files at the given location. """ raise NotImplementedError - def run_command(self, cmd, show_stdout=True, cwd=None, - on_returncode='raise', - command_desc=None, - extra_environ=None, spinner=None): + @classmethod + def run_command( + cls, + cmd, # type: List[str] + show_stdout=True, # type: bool + cwd=None, # type: Optional[str] + on_returncode='raise', # type: str + extra_ok_returncodes=None, # type: Optional[Iterable[int]] + command_desc=None, # type: Optional[str] + extra_environ=None, # type: Optional[Mapping[str, Any]] + spinner=None # type: Optional[SpinnerInterface] + ): + # type: (...) -> Optional[Text] """ Run a VCS subcommand This is simply a wrapper around call_subprocess that adds the VCS command name, and checks that the VCS is available """ - cmd = [self.name] + cmd + cmd = [cls.name] + cmd try: return call_subprocess(cmd, show_stdout, cwd, - on_returncode, - command_desc, extra_environ, - unset_environ=self.unset_environ, + on_returncode=on_returncode, + extra_ok_returncodes=extra_ok_returncodes, + command_desc=command_desc, + extra_environ=extra_environ, + unset_environ=cls.unset_environ, spinner=spinner) except OSError as e: # errno.ENOENT = no such file or directory @@ -461,12 +506,13 @@ class VersionControl(object): raise BadCommand( 'Cannot find command %r - do you have ' '%r installed and in your ' - 'PATH?' % (self.name, self.name)) + 'PATH?' % (cls.name, cls.name)) else: raise # re-raise exception if a different error occurred @classmethod def is_repository_directory(cls, path): + # type: (str) -> bool """ Return whether a directory path is a repository directory. """ @@ -476,6 +522,7 @@ class VersionControl(object): @classmethod def controls_location(cls, location): + # type: (str) -> bool """ Check if a location is controlled by the vcs. It is meant to be overridden to implement smarter detection @@ -485,25 +532,3 @@ class VersionControl(object): the Git override checks that Git is actually available. """ return cls.is_repository_directory(location) - - -def get_src_requirement(dist, location): - version_control = vcs.get_backend_from_location(location) - if version_control: - try: - return version_control().get_src_requirement(dist, - location) - except BadCommand: - logger.warning( - 'cannot determine version of editable source in %s ' - '(%s command not found in path)', - location, - version_control.name, - ) - return dist.as_requirement() - logger.warning( - 'cannot determine version of editable source in %s (is not SVN ' - 'checkout, Git clone, Mercurial clone or Bazaar branch)', - location, - ) - return dist.as_requirement() diff --git a/pipenv/patched/notpip/_internal/vcs/bazaar.py b/pipenv/patched/notpip/_internal/vcs/bazaar.py index 890448ed..256eb9e2 100644 --- a/pipenv/patched/notpip/_internal/vcs/bazaar.py +++ b/pipenv/patched/notpip/_internal/vcs/bazaar.py @@ -75,34 +75,36 @@ class Bazaar(VersionControl): url = 'bzr+' + url return url, rev, user_pass - def get_url(self, location): - urls = self.run_command(['info'], show_stdout=False, cwd=location) + @classmethod + def get_remote_url(cls, location): + urls = cls.run_command(['info'], show_stdout=False, cwd=location) for line in urls.splitlines(): line = line.strip() for x in ('checkout of branch: ', 'parent branch: '): if line.startswith(x): repo = line.split(x)[1] - if self._is_local_repository(repo): + if cls._is_local_repository(repo): return path_to_url(repo) return repo return None - def get_revision(self, location): - revision = self.run_command( + @classmethod + def get_revision(cls, location): + revision = cls.run_command( ['revno'], show_stdout=False, cwd=location, ) return revision.splitlines()[-1] - def get_src_requirement(self, dist, location): - repo = self.get_url(location) + @classmethod + def get_src_requirement(cls, location, project_name): + repo = cls.get_remote_url(location) if not repo: return None if not repo.lower().startswith('bzr:'): repo = 'bzr+' + repo - current_rev = self.get_revision(location) - egg_project_name = dist.egg_name().split('-', 1)[0] - return make_vcs_requirement_url(repo, current_rev, egg_project_name) + current_rev = cls.get_revision(location) + return make_vcs_requirement_url(repo, current_rev, project_name) def is_commit_id_equal(self, dest, name): """Always assume the versions don't match""" diff --git a/pipenv/patched/notpip/_internal/vcs/git.py b/pipenv/patched/notpip/_internal/vcs/git.py index 3db56144..310eb9fb 100644 --- a/pipenv/patched/notpip/_internal/vcs/git.py +++ b/pipenv/patched/notpip/_internal/vcs/git.py @@ -10,9 +10,11 @@ from pipenv.patched.notpip._vendor.six.moves.urllib import request as urllib_req from pipenv.patched.notpip._internal.exceptions import BadCommand from pipenv.patched.notpip._internal.utils.compat import samefile -from pipenv.patched.notpip._internal.utils.misc import display_path, make_vcs_requirement_url +from pipenv.patched.notpip._internal.utils.misc import ( + display_path, make_vcs_requirement_url, redact_password_from_url, +) from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory -from pipenv.patched.notpip._internal.vcs import VersionControl, vcs +from pipenv.patched.notpip._internal.vcs import RemoteNotFoundError, VersionControl, vcs urlsplit = urllib_parse.urlsplit urlunsplit = urllib_parse.urlunsplit @@ -77,19 +79,25 @@ class Git(VersionControl): version = '.'.join(version.split('.')[:3]) return parse_version(version) - def get_branch(self, location): + def get_current_branch(self, location): """ Return the current branch, or None if HEAD isn't at a branch (e.g. detached HEAD). """ - args = ['rev-parse', '--abbrev-ref', 'HEAD'] - output = self.run_command(args, show_stdout=False, cwd=location) - branch = output.strip() + # git-symbolic-ref exits with empty stdout if "HEAD" is a detached + # HEAD rather than a symbolic ref. In addition, the -q causes the + # command to exit with status code 1 instead of 128 in this case + # and to suppress the message to stderr. + args = ['symbolic-ref', '-q', 'HEAD'] + output = self.run_command( + args, extra_ok_returncodes=(1, ), show_stdout=False, cwd=location, + ) + ref = output.strip() - if branch == 'HEAD': - return None + if ref.startswith('refs/heads/'): + return ref[len('refs/heads/'):] - return branch + return None def export(self, location): """Export the Git repository at the url to the destination location""" @@ -193,7 +201,8 @@ class Git(VersionControl): def fetch_new(self, dest, url, rev_options): rev_display = rev_options.to_display() logger.info( - 'Cloning %s%s to %s', url, rev_display, display_path(dest), + 'Cloning %s%s to %s', redact_password_from_url(url), + rev_display, display_path(dest), ) self.run_command(['clone', '-q', url, dest]) @@ -207,7 +216,7 @@ class Git(VersionControl): if not self.is_commit_id_equal(dest, rev_options.rev): cmd_args = ['checkout', '-q'] + rev_options.to_args() self.run_command(cmd_args, cwd=dest) - elif self.get_branch(dest) != branch_name: + elif self.get_current_branch(dest) != branch_name: # Then a specific branch was requested, and that branch # is not yet checked out. track_branch = 'origin/{}'.format(branch_name) @@ -240,14 +249,26 @@ class Git(VersionControl): #: update submodules self.update_submodules(dest) - def get_url(self, location): - """Return URL of the first remote encountered.""" - remotes = self.run_command( + @classmethod + def get_remote_url(cls, location): + """ + Return URL of the first remote encountered. + + Raises RemoteNotFoundError if the repository does not have a remote + url configured. + """ + # We need to pass 1 for extra_ok_returncodes since the command + # exits with return code 1 if there are no matching lines. + stdout = cls.run_command( ['config', '--get-regexp', r'remote\..*\.url'], - show_stdout=False, cwd=location, + extra_ok_returncodes=(1, ), show_stdout=False, cwd=location, ) - remotes = remotes.splitlines() - found_remote = remotes[0] + remotes = stdout.splitlines() + try: + found_remote = remotes[0] + except IndexError: + raise RemoteNotFoundError + for remote in remotes: if remote.startswith('remote.origin.url '): found_remote = remote @@ -255,19 +276,21 @@ class Git(VersionControl): url = found_remote.split(' ')[1] return url.strip() - def get_revision(self, location, rev=None): + @classmethod + def get_revision(cls, location, rev=None): if rev is None: rev = 'HEAD' - current_rev = self.run_command( + current_rev = cls.run_command( ['rev-parse', rev], show_stdout=False, cwd=location, ) return current_rev.strip() - def _get_subdirectory(self, location): + @classmethod + def _get_subdirectory(cls, location): """Return the relative path of setup.py to the git repo root.""" # find the repo root - git_dir = self.run_command(['rev-parse', '--git-dir'], - show_stdout=False, cwd=location).strip() + git_dir = cls.run_command(['rev-parse', '--git-dir'], + show_stdout=False, cwd=location).strip() if not os.path.isabs(git_dir): git_dir = os.path.join(location, git_dir) root_dir = os.path.join(git_dir, '..') @@ -290,14 +313,14 @@ class Git(VersionControl): return None return os.path.relpath(location, root_dir) - def get_src_requirement(self, dist, location): - repo = self.get_url(location) + @classmethod + def get_src_requirement(cls, location, project_name): + repo = cls.get_remote_url(location) if not repo.lower().startswith('git:'): repo = 'git+' + repo - current_rev = self.get_revision(location) - egg_project_name = dist.egg_name().split('-', 1)[0] - subdir = self._get_subdirectory(location) - req = make_vcs_requirement_url(repo, current_rev, egg_project_name, + current_rev = cls.get_revision(location) + subdir = cls._get_subdirectory(location) + req = make_vcs_requirement_url(repo, current_rev, project_name, subdir=subdir) return req @@ -332,10 +355,10 @@ class Git(VersionControl): if super(Git, cls).controls_location(location): return True try: - r = cls().run_command(['rev-parse'], - cwd=location, - show_stdout=False, - on_returncode='ignore') + r = cls.run_command(['rev-parse'], + cwd=location, + show_stdout=False, + on_returncode='ignore') return not r except BadCommand: logger.debug("could not determine if %s is under git control " diff --git a/pipenv/patched/notpip/_internal/vcs/mercurial.py b/pipenv/patched/notpip/_internal/vcs/mercurial.py index d76d47f3..37a3f7c2 100644 --- a/pipenv/patched/notpip/_internal/vcs/mercurial.py +++ b/pipenv/patched/notpip/_internal/vcs/mercurial.py @@ -64,34 +64,36 @@ class Mercurial(VersionControl): cmd_args = ['update', '-q'] + rev_options.to_args() self.run_command(cmd_args, cwd=dest) - def get_url(self, location): - url = self.run_command( + @classmethod + def get_remote_url(cls, location): + url = cls.run_command( ['showconfig', 'paths.default'], show_stdout=False, cwd=location).strip() - if self._is_local_repository(url): + if cls._is_local_repository(url): url = path_to_url(url) return url.strip() - def get_revision(self, location): - current_revision = self.run_command( + @classmethod + def get_revision(cls, location): + current_revision = cls.run_command( ['parents', '--template={rev}'], show_stdout=False, cwd=location).strip() return current_revision - def get_revision_hash(self, location): - current_rev_hash = self.run_command( + @classmethod + def get_revision_hash(cls, location): + current_rev_hash = cls.run_command( ['parents', '--template={node}'], show_stdout=False, cwd=location).strip() return current_rev_hash - def get_src_requirement(self, dist, location): - repo = self.get_url(location) + @classmethod + def get_src_requirement(cls, location, project_name): + repo = cls.get_remote_url(location) if not repo.lower().startswith('hg:'): repo = 'hg+' + repo - current_rev_hash = self.get_revision_hash(location) - egg_project_name = dist.egg_name().split('-', 1)[0] - return make_vcs_requirement_url(repo, current_rev_hash, - egg_project_name) + current_rev_hash = cls.get_revision_hash(location) + return make_vcs_requirement_url(repo, current_rev_hash, project_name) def is_commit_id_equal(self, dest, name): """Always assume the versions don't match""" diff --git a/pipenv/patched/notpip/_internal/vcs/subversion.py b/pipenv/patched/notpip/_internal/vcs/subversion.py index f3c3db4d..e62d3def 100644 --- a/pipenv/patched/notpip/_internal/vcs/subversion.py +++ b/pipenv/patched/notpip/_internal/vcs/subversion.py @@ -4,7 +4,6 @@ import logging import os import re -from pipenv.patched.notpip._internal.models.link import Link from pipenv.patched.notpip._internal.utils.logging import indent_log from pipenv.patched.notpip._internal.utils.misc import ( display_path, make_vcs_requirement_url, rmtree, split_auth_from_netloc, @@ -61,21 +60,8 @@ class Subversion(VersionControl): cmd_args = ['update'] + rev_options.to_args() + [dest] self.run_command(cmd_args) - def get_location(self, dist, dependency_links): - for url in dependency_links: - egg_fragment = Link(url).egg_fragment - if not egg_fragment: - continue - if '-' in egg_fragment: - # FIXME: will this work when a package has - in the name? - key = '-'.join(egg_fragment.split('-')[:-1]).lower() - else: - key = egg_fragment - if key == dist.key: - return url.split('#', 1)[0] - return None - - def get_revision(self, location): + @classmethod + def get_revision(cls, location): """ Return the maximum revision for all files under a given location """ @@ -83,16 +69,16 @@ class Subversion(VersionControl): revision = 0 for base, dirs, files in os.walk(location): - if self.dirname not in dirs: + if cls.dirname not in dirs: dirs[:] = [] continue # no sense walking uncontrolled subdirs - dirs.remove(self.dirname) - entries_fn = os.path.join(base, self.dirname, 'entries') + dirs.remove(cls.dirname) + entries_fn = os.path.join(base, cls.dirname, 'entries') if not os.path.exists(entries_fn): # FIXME: should we warn? continue - dirurl, localrev = self._get_svn_url_rev(base) + dirurl, localrev = cls._get_svn_url_rev(base) if base == location: base = dirurl + '/' # save the root url @@ -131,7 +117,8 @@ class Subversion(VersionControl): return extra_args - def get_url(self, location): + @classmethod + def get_remote_url(cls, location): # In cases where the source is in a subdirectory, not alongside # setup.py we have to look up in the location until we find a real # setup.py @@ -149,12 +136,13 @@ class Subversion(VersionControl): ) return None - return self._get_svn_url_rev(location)[0] + return cls._get_svn_url_rev(location)[0] - def _get_svn_url_rev(self, location): + @classmethod + def _get_svn_url_rev(cls, location): from pipenv.patched.notpip._internal.exceptions import InstallationError - entries_path = os.path.join(location, self.dirname, 'entries') + entries_path = os.path.join(location, cls.dirname, 'entries') if os.path.exists(entries_path): with open(entries_path) as f: data = f.read() @@ -177,7 +165,7 @@ class Subversion(VersionControl): else: try: # subversion >= 1.7 - xml = self.run_command( + xml = cls.run_command( ['info', '--xml', location], show_stdout=False, ) @@ -195,15 +183,14 @@ class Subversion(VersionControl): return url, rev - def get_src_requirement(self, dist, location): - repo = self.get_url(location) + @classmethod + def get_src_requirement(cls, location, project_name): + repo = cls.get_remote_url(location) if repo is None: return None repo = 'svn+' + repo - rev = self.get_revision(location) - # FIXME: why not project name? - egg_project_name = dist.egg_name().split('-', 1)[0] - return make_vcs_requirement_url(repo, rev, egg_project_name) + rev = cls.get_revision(location) + return make_vcs_requirement_url(repo, rev, project_name) def is_commit_id_equal(self, dest, name): """Always assume the versions don't match""" diff --git a/pipenv/patched/notpip/_internal/wheel.py b/pipenv/patched/notpip/_internal/wheel.py index 6df5a3a3..06c880e8 100644 --- a/pipenv/patched/notpip/_internal/wheel.py +++ b/pipenv/patched/notpip/_internal/wheel.py @@ -30,6 +30,7 @@ from pipenv.patched.notpip._internal.exceptions import ( from pipenv.patched.notpip._internal.locations import ( PIP_DELETE_MARKER_FILENAME, distutils_scheme, ) +from pipenv.patched.notpip._internal.models.link import Link from pipenv.patched.notpip._internal.utils.logging import indent_log from pipenv.patched.notpip._internal.utils.misc import ( call_subprocess, captured_stdout, ensure_dir, read_chunks, @@ -40,9 +41,22 @@ from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING from pipenv.patched.notpip._internal.utils.ui import open_spinner if MYPY_CHECK_RUNNING: - from typing import Dict, List, Optional # noqa: F401 + from typing import ( # noqa: F401 + Dict, List, Optional, Sequence, Mapping, Tuple, IO, Text, Any, + Union, Iterable + ) + from pipenv.patched.notpip._vendor.packaging.requirements import Requirement # noqa: F401 + from pipenv.patched.notpip._internal.req.req_install import InstallRequirement # noqa: F401 + from pipenv.patched.notpip._internal.download import PipSession # noqa: F401 + from pipenv.patched.notpip._internal.index import FormatControl, PackageFinder # noqa: F401 + from pipenv.patched.notpip._internal.operations.prepare import ( # noqa: F401 + RequirementPreparer + ) + from pipenv.patched.notpip._internal.cache import WheelCache # noqa: F401 + from pipenv.patched.notpip._internal.pep425tags import Pep425Tag # noqa: F401 + + InstalledCSVRow = Tuple[str, ...] -wheel_ext = '.whl' VERSION_COMPATIBLE = (1, 0) @@ -50,7 +64,12 @@ VERSION_COMPATIBLE = (1, 0) logger = logging.getLogger(__name__) +def normpath(src, p): + return os.path.relpath(src, p).replace(os.path.sep, '/') + + def rehash(path, blocksize=1 << 20): + # type: (str, int) -> Tuple[str, str] """Return (hash, length) for path using hashlib.sha256()""" h = hashlib.sha256() length = 0 @@ -61,20 +80,32 @@ def rehash(path, blocksize=1 << 20): digest = 'sha256=' + urlsafe_b64encode( h.digest() ).decode('latin1').rstrip('=') - return (digest, length) + # unicode/str python2 issues + return (digest, str(length)) # type: ignore def open_for_csv(name, mode): + # type: (str, Text) -> IO if sys.version_info[0] < 3: - nl = {} + nl = {} # type: Dict[str, Any] bin = 'b' else: - nl = {'newline': ''} + nl = {'newline': ''} # type: Dict[str, Any] bin = '' return open(name, mode + bin, **nl) +def replace_python_tag(wheelname, new_tag): + # type: (str, str) -> str + """Replace the Python tag in a wheel file name with a new value. + """ + parts = wheelname.split('-') + parts[-3] = new_tag + return '-'.join(parts) + + def fix_script(path): + # type: (str) -> Optional[bool] """Replace #!python with #!/path/to/python Return True if file was changed.""" # XXX RECORD hashes will need to be updated @@ -90,6 +121,7 @@ def fix_script(path): script.write(firstline) script.write(rest) return True + return None dist_info_re = re.compile(r"""^(?P(?P.+?)(-(?P.+?))?) @@ -97,6 +129,7 @@ dist_info_re = re.compile(r"""^(?P(?P.+?)(-(?P.+?))?) def root_is_purelib(name, wheeldir): + # type: (str, str) -> bool """ Return True if the extracted wheel in wheeldir should go into purelib. """ @@ -113,6 +146,7 @@ def root_is_purelib(name, wheeldir): def get_entrypoints(filename): + # type: (str) -> Tuple[Dict[str, str], Dict[str, str]] if not os.path.exists(filename): return {}, {} @@ -144,7 +178,7 @@ def get_entrypoints(filename): def message_about_scripts_not_on_PATH(scripts): - # type: (List[str]) -> Optional[str] + # type: (Sequence[str]) -> Optional[str] """Determine if any scripts are not on PATH and format a warning. Returns a warning message if one or more scripts are not on PATH, @@ -205,10 +239,81 @@ def message_about_scripts_not_on_PATH(scripts): return "\n".join(msg_lines) -def move_wheel_files(name, req, wheeldir, user=False, home=None, root=None, - pycompile=True, scheme=None, isolated=False, prefix=None, - warn_script_location=True): +def sorted_outrows(outrows): + # type: (Iterable[InstalledCSVRow]) -> List[InstalledCSVRow] + """ + Return the given rows of a RECORD file in sorted order. + + Each row is a 3-tuple (path, hash, size) and corresponds to a record of + a RECORD file (see PEP 376 and PEP 427 for details). For the rows + passed to this function, the size can be an integer as an int or string, + or the empty string. + """ + # Normally, there should only be one row per path, in which case the + # second and third elements don't come into play when sorting. + # However, in cases in the wild where a path might happen to occur twice, + # we don't want the sort operation to trigger an error (but still want + # determinism). Since the third element can be an int or string, we + # coerce each element to a string to avoid a TypeError in this case. + # For additional background, see-- + # https://github.com/pypa/pip/issues/5868 + return sorted(outrows, key=lambda row: tuple(str(x) for x in row)) + + +def get_csv_rows_for_installed( + old_csv_rows, # type: Iterable[List[str]] + installed, # type: Dict[str, str] + changed, # type: set + generated, # type: List[str] + lib_dir, # type: str +): + # type: (...) -> List[InstalledCSVRow] + """ + :param installed: A map from archive RECORD path to installation RECORD + path. + """ + installed_rows = [] # type: List[InstalledCSVRow] + for row in old_csv_rows: + if len(row) > 3: + logger.warning( + 'RECORD line has more than three elements: {}'.format(row) + ) + # Make a copy because we are mutating the row. + row = list(row) + old_path = row[0] + new_path = installed.pop(old_path, old_path) + row[0] = new_path + if new_path in changed: + digest, length = rehash(new_path) + row[1] = digest + row[2] = length + installed_rows.append(tuple(row)) + for f in generated: + digest, length = rehash(f) + installed_rows.append((normpath(f, lib_dir), digest, str(length))) + for f in installed: + installed_rows.append((installed[f], '', '')) + return installed_rows + + +def move_wheel_files( + name, # type: str + req, # type: Requirement + wheeldir, # type: str + user=False, # type: bool + home=None, # type: Optional[str] + root=None, # type: Optional[str] + pycompile=True, # type: bool + scheme=None, # type: Optional[Mapping[str, str]] + isolated=False, # type: bool + prefix=None, # type: Optional[str] + warn_script_location=True # type: bool +): + # type: (...) -> None """Install a wheel""" + # TODO: Investigate and break this up. + # TODO: Look into moving this into a dedicated class for representing an + # installation. if not scheme: scheme = distutils_scheme( @@ -221,7 +326,7 @@ def move_wheel_files(name, req, wheeldir, user=False, home=None, root=None, else: lib_dir = scheme['platlib'] - info_dir = [] + info_dir = [] # type: List[str] data_dirs = [] source = wheeldir.rstrip(os.path.sep) + os.path.sep @@ -229,9 +334,9 @@ def move_wheel_files(name, req, wheeldir, user=False, home=None, root=None, # installed = files copied from the wheel to the destination # changed = files changed while installing (scripts #! line typically) # generated = files newly generated during the install (script wrappers) - installed = {} + installed = {} # type: Dict[str, str] changed = set() - generated = [] + generated = [] # type: List[str] # Compile all of the pyc files that we're going to be installing if pycompile: @@ -241,9 +346,6 @@ def move_wheel_files(name, req, wheeldir, user=False, home=None, root=None, compileall.compile_dir(source, force=True, quiet=True) logger.debug(stdout.getvalue()) - def normpath(src, p): - return os.path.relpath(src, p).replace(os.path.sep, '/') - def record_installed(srcfile, destfile, modified=False): """Map archive RECORD paths to installation RECORD paths.""" oldpath = normpath(srcfile, wheeldir) @@ -389,8 +491,9 @@ def move_wheel_files(name, req, wheeldir, user=False, home=None, root=None, "import_name": entry.suffix.split(".")[0], "func": entry.suffix, } - - maker._get_script_text = _get_script_text + # ignore type, because mypy disallows assigning to a method, + # see https://github.com/python/mypy/issues/2427 + maker._get_script_text = _get_script_text # type: ignore maker.script_template = r"""# -*- coding: utf-8 -*- import re import sys @@ -500,28 +603,23 @@ if __name__ == '__main__': with open_for_csv(record, 'r') as record_in: with open_for_csv(temp_record, 'w+') as record_out: reader = csv.reader(record_in) + outrows = get_csv_rows_for_installed( + reader, installed=installed, changed=changed, + generated=generated, lib_dir=lib_dir, + ) writer = csv.writer(record_out) - outrows = [] - for row in reader: - row[0] = installed.pop(row[0], row[0]) - if row[0] in changed: - row[1], row[2] = rehash(row[0]) - outrows.append(tuple(row)) - for f in generated: - digest, length = rehash(f) - outrows.append((normpath(f, lib_dir), digest, length)) - for f in installed: - outrows.append((installed[f], '', '')) - for row in sorted(outrows): + # Sort to simplify testing. + for row in sorted_outrows(outrows): writer.writerow(row) shutil.move(temp_record, record) def wheel_version(source_dir): + # type: (Optional[str]) -> Optional[Tuple[int, ...]] """ Return the Wheel-Version of an extracted wheel, if possible. - Otherwise, return False if we couldn't parse / extract it. + Otherwise, return None if we couldn't parse / extract it. """ try: dist = [d for d in pkg_resources.find_on_path(None, source_dir)][0] @@ -533,10 +631,11 @@ def wheel_version(source_dir): version = tuple(map(int, version.split('.'))) return version except Exception: - return False + return None def check_compatibility(version, name): + # type: (Optional[Tuple[int, ...]], str) -> None """ Raises errors or warns if called with an incompatible Wheel-Version. @@ -568,7 +667,8 @@ def check_compatibility(version, name): class Wheel(object): """A wheel file""" - # TODO: maybe move the install code into this class + # TODO: Maybe move the class into the models sub-package + # TODO: Maybe move the install code into this class wheel_file_re = re.compile( r"""^(?P(?P.+?)-(?P.*?)) @@ -578,6 +678,7 @@ class Wheel(object): ) def __init__(self, filename): + # type: (str) -> None """ :raises InvalidWheelFilename: when the filename is invalid for a wheel """ @@ -603,6 +704,7 @@ class Wheel(object): } def support_index_min(self, tags=None): + # type: (Optional[List[Pep425Tag]]) -> Optional[int] """ Return the lowest index that one of the wheel's file_tag combinations achieves in the supported_tags list e.g. if there are 8 supported tags, @@ -615,17 +717,146 @@ class Wheel(object): return min(indexes) if indexes else None def supported(self, tags=None): + # type: (Optional[List[Pep425Tag]]) -> bool """Is this wheel supported on this system?""" if tags is None: # for mock tags = pep425tags.get_supported() return bool(set(tags).intersection(self.file_tags)) +def _contains_egg_info( + s, _egg_info_re=re.compile(r'([a-z0-9_.]+)-([a-z0-9_.!+-]+)', re.I)): + """Determine whether the string looks like an egg_info. + + :param s: The string to parse. E.g. foo-2.1 + """ + return bool(_egg_info_re.search(s)) + + +def should_use_ephemeral_cache( + req, # type: InstallRequirement + format_control, # type: FormatControl + autobuilding, # type: bool + cache_available # type: bool +): + # type: (...) -> Optional[bool] + """ + Return whether to build an InstallRequirement object using the + ephemeral cache. + + :param cache_available: whether a cache directory is available for the + autobuilding=True case. + + :return: True or False to build the requirement with ephem_cache=True + or False, respectively; or None not to build the requirement. + """ + if req.constraint: + return None + if req.is_wheel: + if not autobuilding: + logger.info( + 'Skipping %s, due to already being wheel.', req.name, + ) + return None + if not autobuilding: + return False + + if req.editable or not req.source_dir: + return None + + if req.link and not req.link.is_artifact: + # VCS checkout. Build wheel just for this run. + return True + + if "binary" not in format_control.get_allowed_formats( + canonicalize_name(req.name)): + logger.info( + "Skipping bdist_wheel for %s, due to binaries " + "being disabled for it.", req.name, + ) + return None + + link = req.link + base, ext = link.splitext() + if cache_available and _contains_egg_info(base): + return False + + # Otherwise, build the wheel just for this run using the ephemeral + # cache since we are either in the case of e.g. a local directory, or + # no cache directory is available to use. + return True + + +def format_command( + command_args, # type: List[str] + command_output, # type: str +): + # type: (...) -> str + """ + Format command information for logging. + """ + text = 'Command arguments: {}\n'.format(command_args) + + if not command_output: + text += 'Command output: None' + elif logger.getEffectiveLevel() > logging.DEBUG: + text += 'Command output: [use --verbose to show]' + else: + if not command_output.endswith('\n'): + command_output += '\n' + text += ( + 'Command output:\n{}' + '-----------------------------------------' + ).format(command_output) + + return text + + +def get_legacy_build_wheel_path( + names, # type: List[str] + temp_dir, # type: str + req, # type: InstallRequirement + command_args, # type: List[str] + command_output, # type: str +): + # type: (...) -> Optional[str] + """ + Return the path to the wheel in the temporary build directory. + """ + # Sort for determinism. + names = sorted(names) + if not names: + msg = ( + 'Legacy build of wheel for {!r} created no files.\n' + ).format(req.name) + msg += format_command(command_args, command_output) + logger.warning(msg) + return None + + if len(names) > 1: + msg = ( + 'Legacy build of wheel for {!r} created more than one file.\n' + 'Filenames (choosing first): {}\n' + ).format(req.name, names) + msg += format_command(command_args, command_output) + logger.warning(msg) + + return os.path.join(temp_dir, names[0]) + + class WheelBuilder(object): """Build wheels from a RequirementSet.""" - def __init__(self, finder, preparer, wheel_cache, - build_options=None, global_options=None, no_clean=False): + def __init__( + self, + finder, # type: PackageFinder + preparer, # type: RequirementPreparer + wheel_cache, # type: WheelCache + build_options=None, # type: Optional[List[str]] + global_options=None, # type: Optional[List[str]] + no_clean=False # type: bool + ): + # type: (...) -> None self.finder = finder self.preparer = preparer self.wheel_cache = wheel_cache @@ -648,15 +879,18 @@ class WheelBuilder(object): def _build_one_inside_env(self, req, output_dir, python_tag=None): with TempDirectory(kind="wheel") as temp_dir: - if self.__build_one(req, temp_dir.path, python_tag=python_tag): + if req.use_pep517: + builder = self._build_one_pep517 + else: + builder = self._build_one_legacy + wheel_path = builder(req, temp_dir.path, python_tag=python_tag) + if wheel_path is not None: + wheel_name = os.path.basename(wheel_path) + dest_path = os.path.join(output_dir, wheel_name) try: - wheel_name = os.listdir(temp_dir.path)[0] - wheel_path = os.path.join(output_dir, wheel_name) - shutil.move( - os.path.join(temp_dir.path, wheel_name), wheel_path - ) + shutil.move(wheel_path, dest_path) logger.info('Stored in directory: %s', output_dir) - return wheel_path + return dest_path except Exception: pass # Ignore return, we can't do anything else useful. @@ -674,10 +908,43 @@ class WheelBuilder(object): SETUPTOOLS_SHIM % req.setup_py ] + list(self.global_options) - def __build_one(self, req, tempd, python_tag=None): + def _build_one_pep517(self, req, tempd, python_tag=None): + """Build one InstallRequirement using the PEP 517 build process. + + Returns path to wheel if successfully built. Otherwise, returns None. + """ + assert req.metadata_directory is not None + try: + req.spin_message = 'Building wheel for %s (PEP 517)' % (req.name,) + logger.debug('Destination directory: %s', tempd) + wheel_name = req.pep517_backend.build_wheel( + tempd, + metadata_directory=req.metadata_directory + ) + if python_tag: + # General PEP 517 backends don't necessarily support + # a "--python-tag" option, so we rename the wheel + # file directly. + new_name = replace_python_tag(wheel_name, python_tag) + os.rename( + os.path.join(tempd, wheel_name), + os.path.join(tempd, new_name) + ) + # Reassign to simplify the return at the end of function + wheel_name = new_name + except Exception: + logger.error('Failed building wheel for %s', req.name) + return None + return os.path.join(tempd, wheel_name) + + def _build_one_legacy(self, req, tempd, python_tag=None): + """Build one InstallRequirement using the "legacy" build process. + + Returns path to wheel if successfully built. Otherwise, returns None. + """ base_args = self._base_setup_args(req) - spin_message = 'Running setup.py bdist_wheel for %s' % (req.name,) + spin_message = 'Building wheel for %s (setup.py)' % (req.name,) with open_spinner(spin_message) as spinner: logger.debug('Destination directory: %s', tempd) wheel_args = base_args + ['bdist_wheel', '-d', tempd] \ @@ -687,13 +954,21 @@ class WheelBuilder(object): wheel_args += ["--python-tag", python_tag] try: - call_subprocess(wheel_args, cwd=req.setup_py_dir, - show_stdout=False, spinner=spinner) - return True + output = call_subprocess(wheel_args, cwd=req.setup_py_dir, + show_stdout=False, spinner=spinner) except Exception: spinner.finish("error") logger.error('Failed building wheel for %s', req.name) - return False + return None + names = os.listdir(tempd) + wheel_path = get_legacy_build_wheel_path( + names=names, + temp_dir=tempd, + req=req, + command_args=wheel_args, + command_output=output, + ) + return wheel_path def _clean_one(self, req): base_args = self._base_setup_args(req) @@ -707,57 +982,46 @@ class WheelBuilder(object): logger.error('Failed cleaning build dir for %s', req.name) return False - def build(self, requirements, session, autobuilding=False): + def build( + self, + requirements, # type: Iterable[InstallRequirement] + session, # type: PipSession + autobuilding=False # type: bool + ): + # type: (...) -> List[InstallRequirement] """Build wheels. :param unpack: If True, replace the sdist we built from with the newly built wheel, in preparation for installation. :return: True if all the wheels built correctly. """ - from pipenv.patched.notpip._internal import index - from pipenv.patched.notpip._internal.models.link import Link - - building_is_possible = self._wheel_dir or ( - autobuilding and self.wheel_cache.cache_dir - ) - assert building_is_possible - buildset = [] format_control = self.finder.format_control + # Whether a cache directory is available for autobuilding=True. + cache_available = bool(self._wheel_dir or self.wheel_cache.cache_dir) + for req in requirements: - if req.constraint: + ephem_cache = should_use_ephemeral_cache( + req, format_control=format_control, autobuilding=autobuilding, + cache_available=cache_available, + ) + if ephem_cache is None: continue - if req.is_wheel: - if not autobuilding: - logger.info( - 'Skipping %s, due to already being wheel.', req.name, - ) - elif autobuilding and req.editable: - pass - elif autobuilding and not req.source_dir: - pass - elif autobuilding and req.link and not req.link.is_artifact: - # VCS checkout. Build wheel just for this run. - buildset.append((req, True)) - else: - ephem_cache = False - if autobuilding: - link = req.link - base, ext = link.splitext() - if index.egg_info_matches(base, None, link) is None: - # E.g. local directory. Build wheel just for this run. - ephem_cache = True - if "binary" not in format_control.get_allowed_formats( - canonicalize_name(req.name)): - logger.info( - "Skipping bdist_wheel for %s, due to binaries " - "being disabled for it.", req.name, - ) - continue - buildset.append((req, ephem_cache)) + + buildset.append((req, ephem_cache)) if not buildset: - return True + return [] + + # Is any wheel build not using the ephemeral cache? + if any(not ephem_cache for _, ephem_cache in buildset): + have_directory_for_build = self._wheel_dir or ( + autobuilding and self.wheel_cache.cache_dir + ) + assert have_directory_for_build + + # TODO by @pradyunsg + # Should break up this method into 2 separate methods. # Build the wheels. logger.info( @@ -829,5 +1093,5 @@ class WheelBuilder(object): 'Failed to build %s', ' '.join([req.name for req in build_failure]), ) - # Return True if all builds were successful - return len(build_failure) == 0 + # Return a list of requirements that failed to build + return build_failure diff --git a/pipenv/patched/notpip/_vendor/__init__.py b/pipenv/patched/notpip/_vendor/__init__.py index b6294b21..1256c039 100644 --- a/pipenv/patched/notpip/_vendor/__init__.py +++ b/pipenv/patched/notpip/_vendor/__init__.py @@ -74,6 +74,7 @@ if DEBUNDLED: vendored("packaging") vendored("packaging.version") vendored("packaging.specifiers") + vendored("pep517") vendored("pkg_resources") vendored("progress") vendored("pytoml") diff --git a/pipenv/patched/notpip/_vendor/certifi/__init__.py b/pipenv/patched/notpip/_vendor/certifi/__init__.py index aa329fbb..ef71f3af 100644 --- a/pipenv/patched/notpip/_vendor/certifi/__init__.py +++ b/pipenv/patched/notpip/_vendor/certifi/__init__.py @@ -1,3 +1,3 @@ -from .core import where, old_where +from .core import where -__version__ = "2018.08.24" +__version__ = "2018.11.29" diff --git a/pipenv/patched/notpip/_vendor/certifi/cacert.pem b/pipenv/patched/notpip/_vendor/certifi/cacert.pem index 85de024e..db68797e 100644 --- a/pipenv/patched/notpip/_vendor/certifi/cacert.pem +++ b/pipenv/patched/notpip/_vendor/certifi/cacert.pem @@ -326,36 +326,6 @@ OCiNUW7dFGdTbHFcJoRNdVq2fmBWqU2t+5sel/MN2dKXVHfaPRK34B7vCAas+YWH QMAJKOSLakhT2+zNVVXxxvjpoixMptEmX36vWkzaH6byHCx+rgIW0lbQL1dTR+iS -----END CERTIFICATE----- -# Issuer: CN=Visa eCommerce Root O=VISA OU=Visa International Service Association -# Subject: CN=Visa eCommerce Root O=VISA OU=Visa International Service Association -# Label: "Visa eCommerce Root" -# Serial: 25952180776285836048024890241505565794 -# MD5 Fingerprint: fc:11:b8:d8:08:93:30:00:6d:23:f9:7e:eb:52:1e:02 -# SHA1 Fingerprint: 70:17:9b:86:8c:00:a4:fa:60:91:52:22:3f:9f:3e:32:bd:e0:05:62 -# SHA256 Fingerprint: 69:fa:c9:bd:55:fb:0a:c7:8d:53:bb:ee:5c:f1:d5:97:98:9f:d0:aa:ab:20:a2:51:51:bd:f1:73:3e:e7:d1:22 ------BEGIN CERTIFICATE----- -MIIDojCCAoqgAwIBAgIQE4Y1TR0/BvLB+WUF1ZAcYjANBgkqhkiG9w0BAQUFADBr -MQswCQYDVQQGEwJVUzENMAsGA1UEChMEVklTQTEvMC0GA1UECxMmVmlzYSBJbnRl -cm5hdGlvbmFsIFNlcnZpY2UgQXNzb2NpYXRpb24xHDAaBgNVBAMTE1Zpc2EgZUNv -bW1lcmNlIFJvb3QwHhcNMDIwNjI2MDIxODM2WhcNMjIwNjI0MDAxNjEyWjBrMQsw -CQYDVQQGEwJVUzENMAsGA1UEChMEVklTQTEvMC0GA1UECxMmVmlzYSBJbnRlcm5h -dGlvbmFsIFNlcnZpY2UgQXNzb2NpYXRpb24xHDAaBgNVBAMTE1Zpc2EgZUNvbW1l -cmNlIFJvb3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCvV95WHm6h -2mCxlCfLF9sHP4CFT8icttD0b0/Pmdjh28JIXDqsOTPHH2qLJj0rNfVIsZHBAk4E -lpF7sDPwsRROEW+1QK8bRaVK7362rPKgH1g/EkZgPI2h4H3PVz4zHvtH8aoVlwdV -ZqW1LS7YgFmypw23RuwhY/81q6UCzyr0TP579ZRdhE2o8mCP2w4lPJ9zcc+U30rq -299yOIzzlr3xF7zSujtFWsan9sYXiwGd/BmoKoMWuDpI/k4+oKsGGelT84ATB+0t -vz8KPFUgOSwsAGl0lUq8ILKpeeUYiZGo3BxN77t+Nwtd/jmliFKMAGzsGHxBvfaL -dXe6YJ2E5/4tAgMBAAGjQjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQD -AgEGMB0GA1UdDgQWBBQVOIMPPyw/cDMezUb+B4wg4NfDtzANBgkqhkiG9w0BAQUF -AAOCAQEAX/FBfXxcCLkr4NWSR/pnXKUTwwMhmytMiUbPWU3J/qVAtmPN3XEolWcR -zCSs00Rsca4BIGsDoo8Ytyk6feUWYFN4PMCvFYP3j1IzJL1kk5fui/fbGKhtcbP3 -LBfQdCVp9/5rPJS+TUtBjE7ic9DjkCJzQ83z7+pzzkWKsKZJ/0x9nXGIxHYdkFsd -7v3M9+79YKWxehZx0RbQfBI8bGmX265fOZpwLwU8GUYEmSA20GBuYQa7FkKMcPcw -++DbZqMAAb3mLNqRX6BGi01qnD093QVG/na/oAo85ADmJ7f/hC3euiInlhBx6yLt -398znM/jra6O1I7mT1GvFpLgXPYHDw== ------END CERTIFICATE----- - # Issuer: CN=AAA Certificate Services O=Comodo CA Limited # Subject: CN=AAA Certificate Services O=Comodo CA Limited # Label: "Comodo AAA Services root" @@ -4298,3 +4268,245 @@ rYy0UGYwEAYJKwYBBAGCNxUBBAMCAQAwCgYIKoZIzj0EAwMDaAAwZQIwJsdpW9zV 57LnyAyMjMPdeYwbY9XJUpROTYJKcx6ygISpJcBMWm1JKWB4E+J+SOtkAjEA2zQg Mgj/mkkCtojeFK9dbJlxjRo/i9fgojaGHAeCOnZT/cKi7e97sIBPWA9LUzm9 -----END CERTIFICATE----- + +# Issuer: CN=GTS Root R1 O=Google Trust Services LLC +# Subject: CN=GTS Root R1 O=Google Trust Services LLC +# Label: "GTS Root R1" +# Serial: 146587175971765017618439757810265552097 +# MD5 Fingerprint: 82:1a:ef:d4:d2:4a:f2:9f:e2:3d:97:06:14:70:72:85 +# SHA1 Fingerprint: e1:c9:50:e6:ef:22:f8:4c:56:45:72:8b:92:20:60:d7:d5:a7:a3:e8 +# SHA256 Fingerprint: 2a:57:54:71:e3:13:40:bc:21:58:1c:bd:2c:f1:3e:15:84:63:20:3e:ce:94:bc:f9:d3:cc:19:6b:f0:9a:54:72 +-----BEGIN CERTIFICATE----- +MIIFWjCCA0KgAwIBAgIQbkepxUtHDA3sM9CJuRz04TANBgkqhkiG9w0BAQwFADBH +MQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExM +QzEUMBIGA1UEAxMLR1RTIFJvb3QgUjEwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIy +MDAwMDAwWjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNl +cnZpY2VzIExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjEwggIiMA0GCSqGSIb3DQEB +AQUAA4ICDwAwggIKAoICAQC2EQKLHuOhd5s73L+UPreVp0A8of2C+X0yBoJx9vaM +f/vo27xqLpeXo4xL+Sv2sfnOhB2x+cWX3u+58qPpvBKJXqeqUqv4IyfLpLGcY9vX +mX7wCl7raKb0xlpHDU0QM+NOsROjyBhsS+z8CZDfnWQpJSMHobTSPS5g4M/SCYe7 +zUjwTcLCeoiKu7rPWRnWr4+wB7CeMfGCwcDfLqZtbBkOtdh+JhpFAz2weaSUKK0P +fyblqAj+lug8aJRT7oM6iCsVlgmy4HqMLnXWnOunVmSPlk9orj2XwoSPwLxAwAtc +vfaHszVsrBhQf4TgTM2S0yDpM7xSma8ytSmzJSq0SPly4cpk9+aCEI3oncKKiPo4 +Zor8Y/kB+Xj9e1x3+naH+uzfsQ55lVe0vSbv1gHR6xYKu44LtcXFilWr06zqkUsp +zBmkMiVOKvFlRNACzqrOSbTqn3yDsEB750Orp2yjj32JgfpMpf/VjsPOS+C12LOO +Rc92wO1AK/1TD7Cn1TsNsYqiA94xrcx36m97PtbfkSIS5r762DL8EGMUUXLeXdYW +k70paDPvOmbsB4om3xPXV2V4J95eSRQAogB/mqghtqmxlbCluQ0WEdrHbEg8QOB+ +DVrNVjzRlwW5y0vtOUucxD/SVRNuJLDWcfr0wbrM7Rv1/oFB2ACYPTrIrnqYNxgF +lQIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNV +HQ4EFgQU5K8rJnEaK0gnhS9SZizv8IkTcT4wDQYJKoZIhvcNAQEMBQADggIBADiW +Cu49tJYeX++dnAsznyvgyv3SjgofQXSlfKqE1OXyHuY3UjKcC9FhHb8owbZEKTV1 +d5iyfNm9dKyKaOOpMQkpAWBz40d8U6iQSifvS9efk+eCNs6aaAyC58/UEBZvXw6Z +XPYfcX3v73svfuo21pdwCxXu11xWajOl40k4DLh9+42FpLFZXvRq4d2h9mREruZR +gyFmxhE+885H7pwoHyXa/6xmld01D1zvICxi/ZG6qcz8WpyTgYMpl0p8WnK0OdC3 +d8t5/Wk6kjftbjhlRn7pYL15iJdfOBL07q9bgsiG1eGZbYwE8na6SfZu6W0eX6Dv +J4J2QPim01hcDyxC2kLGe4g0x8HYRZvBPsVhHdljUEn2NIVq4BjFbkerQUIpm/Zg +DdIx02OYI5NaAIFItO/Nis3Jz5nu2Z6qNuFoS3FJFDYoOj0dzpqPJeaAcWErtXvM ++SUWgeExX6GjfhaknBZqlxi9dnKlC54dNuYvoS++cJEPqOba+MSSQGwlfnuzCdyy +F62ARPBopY+Udf90WuioAnwMCeKpSwughQtiue+hMZL77/ZRBIls6Kl0obsXs7X9 +SQ98POyDGCBDTtWTurQ0sR8WNh8M5mQ5Fkzc4P4dyKliPUDqysU0ArSuiYgzNdws +E3PYJ/HQcu51OyLemGhmW/HGY0dVHLqlCFF1pkgl +-----END CERTIFICATE----- + +# Issuer: CN=GTS Root R2 O=Google Trust Services LLC +# Subject: CN=GTS Root R2 O=Google Trust Services LLC +# Label: "GTS Root R2" +# Serial: 146587176055767053814479386953112547951 +# MD5 Fingerprint: 44:ed:9a:0e:a4:09:3b:00:f2:ae:4c:a3:c6:61:b0:8b +# SHA1 Fingerprint: d2:73:96:2a:2a:5e:39:9f:73:3f:e1:c7:1e:64:3f:03:38:34:fc:4d +# SHA256 Fingerprint: c4:5d:7b:b0:8e:6d:67:e6:2e:42:35:11:0b:56:4e:5f:78:fd:92:ef:05:8c:84:0a:ea:4e:64:55:d7:58:5c:60 +-----BEGIN CERTIFICATE----- +MIIFWjCCA0KgAwIBAgIQbkepxlqz5yDFMJo/aFLybzANBgkqhkiG9w0BAQwFADBH +MQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExM +QzEUMBIGA1UEAxMLR1RTIFJvb3QgUjIwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIy +MDAwMDAwWjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNl +cnZpY2VzIExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjIwggIiMA0GCSqGSIb3DQEB +AQUAA4ICDwAwggIKAoICAQDO3v2m++zsFDQ8BwZabFn3GTXd98GdVarTzTukk3Lv +CvptnfbwhYBboUhSnznFt+4orO/LdmgUud+tAWyZH8QiHZ/+cnfgLFuv5AS/T3Kg +GjSY6Dlo7JUle3ah5mm5hRm9iYz+re026nO8/4Piy33B0s5Ks40FnotJk9/BW9Bu +XvAuMC6C/Pq8tBcKSOWIm8Wba96wyrQD8Nr0kLhlZPdcTK3ofmZemde4wj7I0BOd +re7kRXuJVfeKH2JShBKzwkCX44ofR5GmdFrS+LFjKBC4swm4VndAoiaYecb+3yXu +PuWgf9RhD1FLPD+M2uFwdNjCaKH5wQzpoeJ/u1U8dgbuak7MkogwTZq9TwtImoS1 +mKPV+3PBV2HdKFZ1E66HjucMUQkQdYhMvI35ezzUIkgfKtzra7tEscszcTJGr61K +8YzodDqs5xoic4DSMPclQsciOzsSrZYuxsN2B6ogtzVJV+mSSeh2FnIxZyuWfoqj +x5RWIr9qS34BIbIjMt/kmkRtWVtd9QCgHJvGeJeNkP+byKq0rxFROV7Z+2et1VsR +nTKaG73VululycslaVNVJ1zgyjbLiGH7HrfQy+4W+9OmTN6SpdTi3/UGVN4unUu0 +kzCqgc7dGtxRcw1PcOnlthYhGXmy5okLdWTK1au8CcEYof/UVKGFPP0UJAOyh9Ok +twIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNV +HQ4EFgQUu//KjiOfT5nK2+JopqUVJxce2Q4wDQYJKoZIhvcNAQEMBQADggIBALZp +8KZ3/p7uC4Gt4cCpx/k1HUCCq+YEtN/L9x0Pg/B+E02NjO7jMyLDOfxA325BS0JT +vhaI8dI4XsRomRyYUpOM52jtG2pzegVATX9lO9ZY8c6DR2Dj/5epnGB3GFW1fgiT +z9D2PGcDFWEJ+YF59exTpJ/JjwGLc8R3dtyDovUMSRqodt6Sm2T4syzFJ9MHwAiA +pJiS4wGWAqoC7o87xdFtCjMwc3i5T1QWvwsHoaRc5svJXISPD+AVdyx+Jn7axEvb +pxZ3B7DNdehyQtaVhJ2Gg/LkkM0JR9SLA3DaWsYDQvTtN6LwG1BUSw7YhN4ZKJmB +R64JGz9I0cNv4rBgF/XuIwKl2gBbbZCr7qLpGzvpx0QnRY5rn/WkhLx3+WuXrD5R +RaIRpsyF7gpo8j5QOHokYh4XIDdtak23CZvJ/KRY9bb7nE4Yu5UC56GtmwfuNmsk +0jmGwZODUNKBRqhfYlcsu2xkiAhu7xNUX90txGdj08+JN7+dIPT7eoOboB6BAFDC +5AwiWVIQ7UNWhwD4FFKnHYuTjKJNRn8nxnGbJN7k2oaLDX5rIMHAnuFl2GqjpuiF +izoHCBy69Y9Vmhh1fuXsgWbRIXOhNUQLgD1bnF5vKheW0YMjiGZt5obicDIvUiLn +yOd/xCxgXS/Dr55FBcOEArf9LAhST4Ldo/DUhgkC +-----END CERTIFICATE----- + +# Issuer: CN=GTS Root R3 O=Google Trust Services LLC +# Subject: CN=GTS Root R3 O=Google Trust Services LLC +# Label: "GTS Root R3" +# Serial: 146587176140553309517047991083707763997 +# MD5 Fingerprint: 1a:79:5b:6b:04:52:9c:5d:c7:74:33:1b:25:9a:f9:25 +# SHA1 Fingerprint: 30:d4:24:6f:07:ff:db:91:89:8a:0b:e9:49:66:11:eb:8c:5e:46:e5 +# SHA256 Fingerprint: 15:d5:b8:77:46:19:ea:7d:54:ce:1c:a6:d0:b0:c4:03:e0:37:a9:17:f1:31:e8:a0:4e:1e:6b:7a:71:ba:bc:e5 +-----BEGIN CERTIFICATE----- +MIICDDCCAZGgAwIBAgIQbkepx2ypcyRAiQ8DVd2NHTAKBggqhkjOPQQDAzBHMQsw +CQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzEU +MBIGA1UEAxMLR1RTIFJvb3QgUjMwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIyMDAw +MDAwWjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZp +Y2VzIExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjMwdjAQBgcqhkjOPQIBBgUrgQQA +IgNiAAQfTzOHMymKoYTey8chWEGJ6ladK0uFxh1MJ7x/JlFyb+Kf1qPKzEUURout +736GjOyxfi//qXGdGIRFBEFVbivqJn+7kAHjSxm65FSWRQmx1WyRRK2EE46ajA2A +DDL24CejQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1Ud +DgQWBBTB8Sa6oC2uhYHP0/EqEr24Cmf9vDAKBggqhkjOPQQDAwNpADBmAjEAgFuk +fCPAlaUs3L6JbyO5o91lAFJekazInXJ0glMLfalAvWhgxeG4VDvBNhcl2MG9AjEA +njWSdIUlUfUk7GRSJFClH9voy8l27OyCbvWFGFPouOOaKaqW04MjyaR7YbPMAuhd +-----END CERTIFICATE----- + +# Issuer: CN=GTS Root R4 O=Google Trust Services LLC +# Subject: CN=GTS Root R4 O=Google Trust Services LLC +# Label: "GTS Root R4" +# Serial: 146587176229350439916519468929765261721 +# MD5 Fingerprint: 5d:b6:6a:c4:60:17:24:6a:1a:99:a8:4b:ee:5e:b4:26 +# SHA1 Fingerprint: 2a:1d:60:27:d9:4a:b1:0a:1c:4d:91:5c:cd:33:a0:cb:3e:2d:54:cb +# SHA256 Fingerprint: 71:cc:a5:39:1f:9e:79:4b:04:80:25:30:b3:63:e1:21:da:8a:30:43:bb:26:66:2f:ea:4d:ca:7f:c9:51:a4:bd +-----BEGIN CERTIFICATE----- +MIICCjCCAZGgAwIBAgIQbkepyIuUtui7OyrYorLBmTAKBggqhkjOPQQDAzBHMQsw +CQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzEU +MBIGA1UEAxMLR1RTIFJvb3QgUjQwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIyMDAw +MDAwWjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZp +Y2VzIExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjQwdjAQBgcqhkjOPQIBBgUrgQQA +IgNiAATzdHOnaItgrkO4NcWBMHtLSZ37wWHO5t5GvWvVYRg1rkDdc/eJkTBa6zzu +hXyiQHY7qca4R9gq55KRanPpsXI5nymfopjTX15YhmUPoYRlBtHci8nHc8iMai/l +xKvRHYqjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1Ud +DgQWBBSATNbrdP9JNqPV2Py1PsVq8JQdjDAKBggqhkjOPQQDAwNnADBkAjBqUFJ0 +CMRw3J5QdCHojXohw0+WbhXRIjVhLfoIN+4Zba3bssx9BzT1YBkstTTZbyACMANx +sbqjYAuG7ZoIapVon+Kz4ZNkfF6Tpt95LY2F45TPI11xzPKwTdb+mciUqXWi4w== +-----END CERTIFICATE----- + +# Issuer: CN=UCA Global G2 Root O=UniTrust +# Subject: CN=UCA Global G2 Root O=UniTrust +# Label: "UCA Global G2 Root" +# Serial: 124779693093741543919145257850076631279 +# MD5 Fingerprint: 80:fe:f0:c4:4a:f0:5c:62:32:9f:1c:ba:78:a9:50:f8 +# SHA1 Fingerprint: 28:f9:78:16:19:7a:ff:18:25:18:aa:44:fe:c1:a0:ce:5c:b6:4c:8a +# SHA256 Fingerprint: 9b:ea:11:c9:76:fe:01:47:64:c1:be:56:a6:f9:14:b5:a5:60:31:7a:bd:99:88:39:33:82:e5:16:1a:a0:49:3c +-----BEGIN CERTIFICATE----- +MIIFRjCCAy6gAwIBAgIQXd+x2lqj7V2+WmUgZQOQ7zANBgkqhkiG9w0BAQsFADA9 +MQswCQYDVQQGEwJDTjERMA8GA1UECgwIVW5pVHJ1c3QxGzAZBgNVBAMMElVDQSBH +bG9iYWwgRzIgUm9vdDAeFw0xNjAzMTEwMDAwMDBaFw00MDEyMzEwMDAwMDBaMD0x +CzAJBgNVBAYTAkNOMREwDwYDVQQKDAhVbmlUcnVzdDEbMBkGA1UEAwwSVUNBIEds +b2JhbCBHMiBSb290MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAxeYr +b3zvJgUno4Ek2m/LAfmZmqkywiKHYUGRO8vDaBsGxUypK8FnFyIdK+35KYmToni9 +kmugow2ifsqTs6bRjDXVdfkX9s9FxeV67HeToI8jrg4aA3++1NDtLnurRiNb/yzm +VHqUwCoV8MmNsHo7JOHXaOIxPAYzRrZUEaalLyJUKlgNAQLx+hVRZ2zA+te2G3/R +VogvGjqNO7uCEeBHANBSh6v7hn4PJGtAnTRnvI3HLYZveT6OqTwXS3+wmeOwcWDc +C/Vkw85DvG1xudLeJ1uK6NjGruFZfc8oLTW4lVYa8bJYS7cSN8h8s+1LgOGN+jIj +tm+3SJUIsUROhYw6AlQgL9+/V087OpAh18EmNVQg7Mc/R+zvWr9LesGtOxdQXGLY +D0tK3Cv6brxzks3sx1DoQZbXqX5t2Okdj4q1uViSukqSKwxW/YDrCPBeKW4bHAyv +j5OJrdu9o54hyokZ7N+1wxrrFv54NkzWbtA+FxyQF2smuvt6L78RHBgOLXMDj6Dl +NaBa4kx1HXHhOThTeEDMg5PXCp6dW4+K5OXgSORIskfNTip1KnvyIvbJvgmRlld6 +iIis7nCs+dwp4wwcOxJORNanTrAmyPPZGpeRaOrvjUYG0lZFWJo8DA+DuAUlwznP +O6Q0ibd5Ei9Hxeepl2n8pndntd978XplFeRhVmUCAwEAAaNCMEAwDgYDVR0PAQH/ +BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFIHEjMz15DD/pQwIX4wV +ZyF0Ad/fMA0GCSqGSIb3DQEBCwUAA4ICAQATZSL1jiutROTL/7lo5sOASD0Ee/oj +L3rtNtqyzm325p7lX1iPyzcyochltq44PTUbPrw7tgTQvPlJ9Zv3hcU2tsu8+Mg5 +1eRfB70VVJd0ysrtT7q6ZHafgbiERUlMjW+i67HM0cOU2kTC5uLqGOiiHycFutfl +1qnN3e92mI0ADs0b+gO3joBYDic/UvuUospeZcnWhNq5NXHzJsBPd+aBJ9J3O5oU +b3n09tDh05S60FdRvScFDcH9yBIw7m+NESsIndTUv4BFFJqIRNow6rSn4+7vW4LV +PtateJLbXDzz2K36uGt/xDYotgIVilQsnLAXc47QN6MUPJiVAAwpBVueSUmxX8fj +y88nZY41F7dXyDDZQVu5FLbowg+UMaeUmMxq67XhJ/UQqAHojhJi6IjMtX9Gl8Cb +EGY4GjZGXyJoPd/JxhMnq1MGrKI8hgZlb7F+sSlEmqO6SWkoaY/X5V+tBIZkbxqg +DMUIYs6Ao9Dz7GjevjPHF1t/gMRMTLGmhIrDO7gJzRSBuhjjVFc2/tsvfEehOjPI ++Vg7RE+xygKJBJYoaMVLuCaJu9YzL1DV/pqJuhgyklTGW+Cd+V7lDSKb9triyCGy +YiGqhkCyLmTTX8jjfhFnRR8F/uOi77Oos/N9j/gMHyIfLXC0uAE0djAA5SN4p1bX +UB+K+wb1whnw0A== +-----END CERTIFICATE----- + +# Issuer: CN=UCA Extended Validation Root O=UniTrust +# Subject: CN=UCA Extended Validation Root O=UniTrust +# Label: "UCA Extended Validation Root" +# Serial: 106100277556486529736699587978573607008 +# MD5 Fingerprint: a1:f3:5f:43:c6:34:9b:da:bf:8c:7e:05:53:ad:96:e2 +# SHA1 Fingerprint: a3:a1:b0:6f:24:61:23:4a:e3:36:a5:c2:37:fc:a6:ff:dd:f0:d7:3a +# SHA256 Fingerprint: d4:3a:f9:b3:54:73:75:5c:96:84:fc:06:d7:d8:cb:70:ee:5c:28:e7:73:fb:29:4e:b4:1e:e7:17:22:92:4d:24 +-----BEGIN CERTIFICATE----- +MIIFWjCCA0KgAwIBAgIQT9Irj/VkyDOeTzRYZiNwYDANBgkqhkiG9w0BAQsFADBH +MQswCQYDVQQGEwJDTjERMA8GA1UECgwIVW5pVHJ1c3QxJTAjBgNVBAMMHFVDQSBF +eHRlbmRlZCBWYWxpZGF0aW9uIFJvb3QwHhcNMTUwMzEzMDAwMDAwWhcNMzgxMjMx +MDAwMDAwWjBHMQswCQYDVQQGEwJDTjERMA8GA1UECgwIVW5pVHJ1c3QxJTAjBgNV +BAMMHFVDQSBFeHRlbmRlZCBWYWxpZGF0aW9uIFJvb3QwggIiMA0GCSqGSIb3DQEB +AQUAA4ICDwAwggIKAoICAQCpCQcoEwKwmeBkqh5DFnpzsZGgdT6o+uM4AHrsiWog +D4vFsJszA1qGxliG1cGFu0/GnEBNyr7uaZa4rYEwmnySBesFK5pI0Lh2PpbIILvS +sPGP2KxFRv+qZ2C0d35qHzwaUnoEPQc8hQ2E0B92CvdqFN9y4zR8V05WAT558aop +O2z6+I9tTcg1367r3CTueUWnhbYFiN6IXSV8l2RnCdm/WhUFhvMJHuxYMjMR83dk +sHYf5BA1FxvyDrFspCqjc/wJHx4yGVMR59mzLC52LqGj3n5qiAno8geK+LLNEOfi +c0CTuwjRP+H8C5SzJe98ptfRr5//lpr1kXuYC3fUfugH0mK1lTnj8/FtDw5lhIpj +VMWAtuCeS31HJqcBCF3RiJ7XwzJE+oJKCmhUfzhTA8ykADNkUVkLo4KRel7sFsLz +KuZi2irbWWIQJUoqgQtHB0MGcIfS+pMRKXpITeuUx3BNr2fVUbGAIAEBtHoIppB/ +TuDvB0GHr2qlXov7z1CymlSvw4m6WC31MJixNnI5fkkE/SmnTHnkBVfblLkWU41G +sx2VYVdWf6/wFlthWG82UBEL2KwrlRYaDh8IzTY0ZRBiZtWAXxQgXy0MoHgKaNYs +1+lvK9JKBZP8nm9rZ/+I8U6laUpSNwXqxhaN0sSZ0YIrO7o1dfdRUVjzyAfd5LQD +fwIDAQABo0IwQDAdBgNVHQ4EFgQU2XQ65DA9DfcS3H5aBZ8eNJr34RQwDwYDVR0T +AQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAYYwDQYJKoZIhvcNAQELBQADggIBADaN +l8xCFWQpN5smLNb7rhVpLGsaGvdftvkHTFnq88nIua7Mui563MD1sC3AO6+fcAUR +ap8lTwEpcOPlDOHqWnzcSbvBHiqB9RZLcpHIojG5qtr8nR/zXUACE/xOHAbKsxSQ +VBcZEhrxH9cMaVr2cXj0lH2RC47skFSOvG+hTKv8dGT9cZr4QQehzZHkPJrgmzI5 +c6sq1WnIeJEmMX3ixzDx/BR4dxIOE/TdFpS/S2d7cFOFyrC78zhNLJA5wA3CXWvp +4uXViI3WLL+rG761KIcSF3Ru/H38j9CHJrAb+7lsq+KePRXBOy5nAliRn+/4Qh8s +t2j1da3Ptfb/EX3C8CSlrdP6oDyp+l3cpaDvRKS+1ujl5BOWF3sGPjLtx7dCvHaj +2GU4Kzg1USEODm8uNBNA4StnDG1KQTAYI1oyVZnJF+A83vbsea0rWBmirSwiGpWO +vpaQXUJXxPkUAzUrHC1RVwinOt4/5Mi0A3PCwSaAuwtCH60NryZy2sy+s6ODWA2C +xR9GUeOcGMyNm43sSet1UNWMKFnKdDTajAshqx7qG+XH/RU+wBeq+yNuJkbL+vmx +cmtpzyKEC2IPrNkZAJSidjzULZrtBJ4tBmIQN1IchXIbJ+XMxjHsN+xjWZsLHXbM +fjKaiJUINlK73nZfdklJrX+9ZSCyycErdhh2n1ax +-----END CERTIFICATE----- + +# Issuer: CN=Certigna Root CA O=Dhimyotis OU=0002 48146308100036 +# Subject: CN=Certigna Root CA O=Dhimyotis OU=0002 48146308100036 +# Label: "Certigna Root CA" +# Serial: 269714418870597844693661054334862075617 +# MD5 Fingerprint: 0e:5c:30:62:27:eb:5b:bc:d7:ae:62:ba:e9:d5:df:77 +# SHA1 Fingerprint: 2d:0d:52:14:ff:9e:ad:99:24:01:74:20:47:6e:6c:85:27:27:f5:43 +# SHA256 Fingerprint: d4:8d:3d:23:ee:db:50:a4:59:e5:51:97:60:1c:27:77:4b:9d:7b:18:c9:4d:5a:05:95:11:a1:02:50:b9:31:68 +-----BEGIN CERTIFICATE----- +MIIGWzCCBEOgAwIBAgIRAMrpG4nxVQMNo+ZBbcTjpuEwDQYJKoZIhvcNAQELBQAw +WjELMAkGA1UEBhMCRlIxEjAQBgNVBAoMCURoaW15b3RpczEcMBoGA1UECwwTMDAw +MiA0ODE0NjMwODEwMDAzNjEZMBcGA1UEAwwQQ2VydGlnbmEgUm9vdCBDQTAeFw0x +MzEwMDEwODMyMjdaFw0zMzEwMDEwODMyMjdaMFoxCzAJBgNVBAYTAkZSMRIwEAYD +VQQKDAlEaGlteW90aXMxHDAaBgNVBAsMEzAwMDIgNDgxNDYzMDgxMDAwMzYxGTAX +BgNVBAMMEENlcnRpZ25hIFJvb3QgQ0EwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAw +ggIKAoICAQDNGDllGlmx6mQWDoyUJJV8g9PFOSbcDO8WV43X2KyjQn+Cyu3NW9sO +ty3tRQgXstmzy9YXUnIo245Onoq2C/mehJpNdt4iKVzSs9IGPjA5qXSjklYcoW9M +CiBtnyN6tMbaLOQdLNyzKNAT8kxOAkmhVECe5uUFoC2EyP+YbNDrihqECB63aCPu +I9Vwzm1RaRDuoXrC0SIxwoKF0vJVdlB8JXrJhFwLrN1CTivngqIkicuQstDuI7pm +TLtipPlTWmR7fJj6o0ieD5Wupxj0auwuA0Wv8HT4Ks16XdG+RCYyKfHx9WzMfgIh +C59vpD++nVPiz32pLHxYGpfhPTc3GGYo0kDFUYqMwy3OU4gkWGQwFsWq4NYKpkDf +ePb1BHxpE4S80dGnBs8B92jAqFe7OmGtBIyT46388NtEbVncSVmurJqZNjBBe3Yz +IoejwpKGbvlw7q6Hh5UbxHq9MfPU0uWZ/75I7HX1eBYdpnDBfzwboZL7z8g81sWT +Co/1VTp2lc5ZmIoJlXcymoO6LAQ6l73UL77XbJuiyn1tJslV1c/DeVIICZkHJC1k +JWumIWmbat10TWuXekG9qxf5kBdIjzb5LdXF2+6qhUVB+s06RbFo5jZMm5BX7CO5 +hwjCxAnxl4YqKE3idMDaxIzb3+KhF1nOJFl0Mdp//TBt2dzhauH8XwIDAQABo4IB +GjCCARYwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYE +FBiHVuBud+4kNTxOc5of1uHieX4rMB8GA1UdIwQYMBaAFBiHVuBud+4kNTxOc5of +1uHieX4rMEQGA1UdIAQ9MDswOQYEVR0gADAxMC8GCCsGAQUFBwIBFiNodHRwczov +L3d3d3cuY2VydGlnbmEuZnIvYXV0b3JpdGVzLzBtBgNVHR8EZjBkMC+gLaArhilo +dHRwOi8vY3JsLmNlcnRpZ25hLmZyL2NlcnRpZ25hcm9vdGNhLmNybDAxoC+gLYYr +aHR0cDovL2NybC5kaGlteW90aXMuY29tL2NlcnRpZ25hcm9vdGNhLmNybDANBgkq +hkiG9w0BAQsFAAOCAgEAlLieT/DjlQgi581oQfccVdV8AOItOoldaDgvUSILSo3L +6btdPrtcPbEo/uRTVRPPoZAbAh1fZkYJMyjhDSSXcNMQH+pkV5a7XdrnxIxPTGRG +HVyH41neQtGbqH6mid2PHMkwgu07nM3A6RngatgCdTer9zQoKJHyBApPNeNgJgH6 +0BGM+RFq7q89w1DTj18zeTyGqHNFkIwgtnJzFyO+B2XleJINugHA64wcZr+shncB +lA2c5uk5jR+mUYyZDDl34bSb+hxnV29qao6pK0xXeXpXIs/NX2NGjVxZOob4Mkdi +o2cNGJHc+6Zr9UhhcyNZjgKnvETq9Emd8VRY+WCv2hikLyhF3HqgiIZd8zvn/yk1 +gPxkQ5Tm4xxvvq0OKmOZK8l+hfZx6AYDlf7ej0gcWtSS6Cvu5zHbugRqh5jnxV/v +faci9wHYTfmJ0A6aBVmknpjZbyvKcL5kwlWj9Omvw5Ip3IgWJJk8jSaYtlu3zM63 +Nwf9JtmYhST/WSMDmu2dnajkXjjO11INb9I/bbEFa0nOipFGc/T2L/Coc3cOZayh +jWZSaX5LaAzHHjcng6WMxwLkFM1JAbBzs/3GkDpv0mztO+7skb6iQ12LAEpmJURw +3kAP+HwV96LOPNdeE4yBFxgX0b3xdxA61GU5wSesVywlVP+i2k+KYTlerj1KjL0= +-----END CERTIFICATE----- diff --git a/pipenv/patched/notpip/_vendor/certifi/core.py b/pipenv/patched/notpip/_vendor/certifi/core.py index eab9d1d1..2d02ea44 100644 --- a/pipenv/patched/notpip/_vendor/certifi/core.py +++ b/pipenv/patched/notpip/_vendor/certifi/core.py @@ -8,14 +8,6 @@ certifi.py This module returns the installation location of cacert.pem. """ import os -import warnings - - -class DeprecatedBundleWarning(DeprecationWarning): - """ - The weak security bundle is being deprecated. Please bother your service - provider to get them to stop using cross-signed roots. - """ def where(): @@ -24,14 +16,5 @@ def where(): return os.path.join(f, 'cacert.pem') -def old_where(): - warnings.warn( - "The weak security bundle has been removed. certifi.old_where() is now an alias " - "of certifi.where(). Please update your code to use certifi.where() instead. " - "certifi.old_where() will be removed in 2018.", - DeprecatedBundleWarning - ) - return where() - if __name__ == '__main__': print(where()) diff --git a/pipenv/patched/notpip/_vendor/colorama/LICENSE.txt b/pipenv/patched/notpip/_vendor/colorama/LICENSE.txt index 5f567799..3105888e 100644 --- a/pipenv/patched/notpip/_vendor/colorama/LICENSE.txt +++ b/pipenv/patched/notpip/_vendor/colorama/LICENSE.txt @@ -25,4 +25,3 @@ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - diff --git a/pipenv/patched/notpip/_vendor/colorama/__init__.py b/pipenv/patched/notpip/_vendor/colorama/__init__.py index f4d9ce21..2a3bf471 100644 --- a/pipenv/patched/notpip/_vendor/colorama/__init__.py +++ b/pipenv/patched/notpip/_vendor/colorama/__init__.py @@ -3,5 +3,4 @@ from .initialise import init, deinit, reinit, colorama_text from .ansi import Fore, Back, Style, Cursor from .ansitowin32 import AnsiToWin32 -__version__ = '0.3.9' - +__version__ = '0.4.1' diff --git a/pipenv/patched/notpip/_vendor/colorama/ansitowin32.py b/pipenv/patched/notpip/_vendor/colorama/ansitowin32.py index 1d6e6059..359c92be 100644 --- a/pipenv/patched/notpip/_vendor/colorama/ansitowin32.py +++ b/pipenv/patched/notpip/_vendor/colorama/ansitowin32.py @@ -13,14 +13,6 @@ if windll is not None: winterm = WinTerm() -def is_stream_closed(stream): - return not hasattr(stream, 'closed') or stream.closed - - -def is_a_tty(stream): - return hasattr(stream, 'isatty') and stream.isatty() - - class StreamWrapper(object): ''' Wraps a stream (such as stdout), acting as a transparent proxy for all @@ -36,9 +28,38 @@ class StreamWrapper(object): def __getattr__(self, name): return getattr(self.__wrapped, name) + def __enter__(self, *args, **kwargs): + # special method lookup bypasses __getattr__/__getattribute__, see + # https://stackoverflow.com/questions/12632894/why-doesnt-getattr-work-with-exit + # thus, contextlib magic methods are not proxied via __getattr__ + return self.__wrapped.__enter__(*args, **kwargs) + + def __exit__(self, *args, **kwargs): + return self.__wrapped.__exit__(*args, **kwargs) + def write(self, text): self.__convertor.write(text) + def isatty(self): + stream = self.__wrapped + if 'PYCHARM_HOSTED' in os.environ: + if stream is not None and (stream is sys.__stdout__ or stream is sys.__stderr__): + return True + try: + stream_isatty = stream.isatty + except AttributeError: + return False + else: + return stream_isatty() + + @property + def closed(self): + stream = self.__wrapped + try: + return stream.closed + except AttributeError: + return True + class AnsiToWin32(object): ''' @@ -68,12 +89,12 @@ class AnsiToWin32(object): # should we strip ANSI sequences from our output? if strip is None: - strip = conversion_supported or (not is_stream_closed(wrapped) and not is_a_tty(wrapped)) + strip = conversion_supported or (not self.stream.closed and not self.stream.isatty()) self.strip = strip # should we should convert ANSI sequences into win32 calls? if convert is None: - convert = conversion_supported and not is_stream_closed(wrapped) and is_a_tty(wrapped) + convert = conversion_supported and not self.stream.closed and self.stream.isatty() self.convert = convert # dict of ansi codes to win32 functions and parameters @@ -149,7 +170,7 @@ class AnsiToWin32(object): def reset_all(self): if self.convert: self.call_win32('m', (0,)) - elif not self.strip and not is_stream_closed(self.wrapped): + elif not self.strip and not self.stream.closed: self.wrapped.write(Style.RESET_ALL) diff --git a/pipenv/patched/notpip/_vendor/colorama/initialise.py b/pipenv/patched/notpip/_vendor/colorama/initialise.py index 834962a3..430d0668 100644 --- a/pipenv/patched/notpip/_vendor/colorama/initialise.py +++ b/pipenv/patched/notpip/_vendor/colorama/initialise.py @@ -78,5 +78,3 @@ def wrap_stream(stream, convert, strip, autoreset, wrap): if wrapper.should_wrap(): stream = wrapper.stream return stream - - diff --git a/pipenv/patched/notpip/_vendor/colorama/win32.py b/pipenv/patched/notpip/_vendor/colorama/win32.py index 8262e350..c2d83603 100644 --- a/pipenv/patched/notpip/_vendor/colorama/win32.py +++ b/pipenv/patched/notpip/_vendor/colorama/win32.py @@ -89,11 +89,6 @@ else: ] _SetConsoleTitleW.restype = wintypes.BOOL - handles = { - STDOUT: _GetStdHandle(STDOUT), - STDERR: _GetStdHandle(STDERR), - } - def _winapi_test(handle): csbi = CONSOLE_SCREEN_BUFFER_INFO() success = _GetConsoleScreenBufferInfo( @@ -101,17 +96,18 @@ else: return bool(success) def winapi_test(): - return any(_winapi_test(h) for h in handles.values()) + return any(_winapi_test(h) for h in + (_GetStdHandle(STDOUT), _GetStdHandle(STDERR))) def GetConsoleScreenBufferInfo(stream_id=STDOUT): - handle = handles[stream_id] + handle = _GetStdHandle(stream_id) csbi = CONSOLE_SCREEN_BUFFER_INFO() success = _GetConsoleScreenBufferInfo( handle, byref(csbi)) return csbi def SetConsoleTextAttribute(stream_id, attrs): - handle = handles[stream_id] + handle = _GetStdHandle(stream_id) return _SetConsoleTextAttribute(handle, attrs) def SetConsoleCursorPosition(stream_id, position, adjust=True): @@ -129,11 +125,11 @@ else: adjusted_position.Y += sr.Top adjusted_position.X += sr.Left # Resume normal processing - handle = handles[stream_id] + handle = _GetStdHandle(stream_id) return _SetConsoleCursorPosition(handle, adjusted_position) def FillConsoleOutputCharacter(stream_id, char, length, start): - handle = handles[stream_id] + handle = _GetStdHandle(stream_id) char = c_char(char.encode()) length = wintypes.DWORD(length) num_written = wintypes.DWORD(0) @@ -144,7 +140,7 @@ else: def FillConsoleOutputAttribute(stream_id, attr, length, start): ''' FillConsoleOutputAttribute( hConsole, csbi.wAttributes, dwConSize, coordScreen, &cCharsWritten )''' - handle = handles[stream_id] + handle = _GetStdHandle(stream_id) attribute = wintypes.WORD(attr) length = wintypes.DWORD(length) num_written = wintypes.DWORD(0) diff --git a/pipenv/patched/notpip/_vendor/colorama/winterm.py b/pipenv/patched/notpip/_vendor/colorama/winterm.py index 60309d3c..0fdb4ec4 100644 --- a/pipenv/patched/notpip/_vendor/colorama/winterm.py +++ b/pipenv/patched/notpip/_vendor/colorama/winterm.py @@ -44,6 +44,7 @@ class WinTerm(object): def reset_all(self, on_stderr=None): self.set_attrs(self._default) self.set_console(attrs=self._default) + self._light = 0 def fore(self, fore=None, light=False, on_stderr=False): if fore is None: @@ -122,12 +123,15 @@ class WinTerm(object): if mode == 0: from_coord = csbi.dwCursorPosition cells_to_erase = cells_in_screen - cells_before_cursor - if mode == 1: + elif mode == 1: from_coord = win32.COORD(0, 0) cells_to_erase = cells_before_cursor elif mode == 2: from_coord = win32.COORD(0, 0) cells_to_erase = cells_in_screen + else: + # invalid mode + return # fill the entire screen with blanks win32.FillConsoleOutputCharacter(handle, ' ', cells_to_erase, from_coord) # now set the buffer's attributes accordingly @@ -147,12 +151,15 @@ class WinTerm(object): if mode == 0: from_coord = csbi.dwCursorPosition cells_to_erase = csbi.dwSize.X - csbi.dwCursorPosition.X - if mode == 1: + elif mode == 1: from_coord = win32.COORD(0, csbi.dwCursorPosition.Y) cells_to_erase = csbi.dwCursorPosition.X elif mode == 2: from_coord = win32.COORD(0, csbi.dwCursorPosition.Y) cells_to_erase = csbi.dwSize.X + else: + # invalid mode + return # fill the entire screen with blanks win32.FillConsoleOutputCharacter(handle, ' ', cells_to_erase, from_coord) # now set the buffer's attributes accordingly diff --git a/pipenv/patched/notpip/_vendor/distlib/__init__.py b/pipenv/patched/notpip/_vendor/distlib/__init__.py index d4aab453..a786b4d3 100644 --- a/pipenv/patched/notpip/_vendor/distlib/__init__.py +++ b/pipenv/patched/notpip/_vendor/distlib/__init__.py @@ -6,7 +6,7 @@ # import logging -__version__ = '0.2.7' +__version__ = '0.2.8' class DistlibException(Exception): pass diff --git a/pipenv/patched/notpip/_vendor/distlib/database.py b/pipenv/patched/notpip/_vendor/distlib/database.py index a19905e2..b13cdac9 100644 --- a/pipenv/patched/notpip/_vendor/distlib/database.py +++ b/pipenv/patched/notpip/_vendor/distlib/database.py @@ -20,7 +20,8 @@ import zipimport from . import DistlibException, resources from .compat import StringIO from .version import get_scheme, UnsupportedVersionError -from .metadata import Metadata, METADATA_FILENAME, WHEEL_METADATA_FILENAME +from .metadata import (Metadata, METADATA_FILENAME, WHEEL_METADATA_FILENAME, + LEGACY_METADATA_FILENAME) from .util import (parse_requirement, cached_property, parse_name_and_version, read_exports, write_exports, CSVReader, CSVWriter) @@ -132,7 +133,9 @@ class DistributionPath(object): if not r or r.path in seen: continue if self._include_dist and entry.endswith(DISTINFO_EXT): - possible_filenames = [METADATA_FILENAME, WHEEL_METADATA_FILENAME] + possible_filenames = [METADATA_FILENAME, + WHEEL_METADATA_FILENAME, + LEGACY_METADATA_FILENAME] for metadata_filename in possible_filenames: metadata_path = posixpath.join(entry, metadata_filename) pydist = finder.find(metadata_path) diff --git a/pipenv/patched/notpip/_vendor/distlib/locators.py b/pipenv/patched/notpip/_vendor/distlib/locators.py index cb05b184..a7ed9469 100644 --- a/pipenv/patched/notpip/_vendor/distlib/locators.py +++ b/pipenv/patched/notpip/_vendor/distlib/locators.py @@ -255,7 +255,9 @@ class Locator(object): if path.endswith('.whl'): try: wheel = Wheel(path) - if is_compatible(wheel, self.wheel_tags): + if not is_compatible(wheel, self.wheel_tags): + logger.debug('Wheel not compatible: %s', path) + else: if project_name is None: include = True else: @@ -613,6 +615,7 @@ class SimpleScrapingLocator(Locator): # as it is for coordinating our internal threads - the ones created # in _prepare_threads. self._gplock = threading.RLock() + self.platform_check = False # See issue #112 def _prepare_threads(self): """ @@ -658,8 +661,8 @@ class SimpleScrapingLocator(Locator): del self.result return result - platform_dependent = re.compile(r'\b(linux-(i\d86|x86_64|arm\w+)|' - r'win(32|-amd64)|macosx-?\d+)\b', re.I) + platform_dependent = re.compile(r'\b(linux_(i\d86|x86_64|arm\w+)|' + r'win(32|_amd64)|macosx_?\d+)\b', re.I) def _is_platform_dependent(self, url): """ @@ -677,7 +680,7 @@ class SimpleScrapingLocator(Locator): Note that the return value isn't actually used other than as a boolean value. """ - if self._is_platform_dependent(url): + if self.platform_check and self._is_platform_dependent(url): info = None else: info = self.convert_url_to_download_info(url, self.project_name) diff --git a/pipenv/patched/notpip/_vendor/distlib/metadata.py b/pipenv/patched/notpip/_vendor/distlib/metadata.py index 6d6470ff..77eed7f9 100644 --- a/pipenv/patched/notpip/_vendor/distlib/metadata.py +++ b/pipenv/patched/notpip/_vendor/distlib/metadata.py @@ -91,7 +91,9 @@ _426_FIELDS = ('Metadata-Version', 'Name', 'Version', 'Platform', _426_MARKERS = ('Private-Version', 'Provides-Extra', 'Obsoleted-By', 'Setup-Requires-Dist', 'Extension') -_566_FIELDS = _426_FIELDS + ('Description-Content-Type',) +# See issue #106: Sometimes 'Requires' occurs wrongly in the metadata. Include +# it in the tuple literal below to allow it (for now) +_566_FIELDS = _426_FIELDS + ('Description-Content-Type', 'Requires') _566_MARKERS = ('Description-Content-Type',) @@ -377,8 +379,8 @@ class LegacyMetadata(object): value = msg[field] if value is not None and value != 'UNKNOWN': self.set(field, value) - logger.debug('Attempting to set metadata for %s', self) - self.set_metadata_version() + # logger.debug('Attempting to set metadata for %s', self) + # self.set_metadata_version() def write(self, filepath, skip_unknown=False): """Write the metadata fields to filepath.""" @@ -648,6 +650,7 @@ class LegacyMetadata(object): METADATA_FILENAME = 'pydist.json' WHEEL_METADATA_FILENAME = 'metadata.json' +LEGACY_METADATA_FILENAME = 'METADATA' class Metadata(object): diff --git a/pipenv/patched/notpip/_vendor/distlib/scripts.py b/pipenv/patched/notpip/_vendor/distlib/scripts.py index 0b7c3d0b..8e22cb91 100644 --- a/pipenv/patched/notpip/_vendor/distlib/scripts.py +++ b/pipenv/patched/notpip/_vendor/distlib/scripts.py @@ -236,8 +236,10 @@ class ScriptMaker(object): def _write_script(self, names, shebang, script_bytes, filenames, ext): use_launcher = self.add_launchers and self._is_nt linesep = os.linesep.encode('utf-8') + if not shebang.endswith(linesep): + shebang += linesep if not use_launcher: - script_bytes = shebang + linesep + script_bytes + script_bytes = shebang + script_bytes else: # pragma: no cover if ext == 'py': launcher = self._get_launcher('t') @@ -247,7 +249,7 @@ class ScriptMaker(object): with ZipFile(stream, 'w') as zf: zf.writestr('__main__.py', script_bytes) zip_data = stream.getvalue() - script_bytes = launcher + shebang + linesep + zip_data + script_bytes = launcher + shebang + zip_data for name in names: outname = os.path.join(self.target_dir, name) if use_launcher: # pragma: no cover diff --git a/pipenv/patched/notpip/_vendor/distlib/util.py b/pipenv/patched/notpip/_vendor/distlib/util.py index 0b14a93b..9d4bfd3b 100644 --- a/pipenv/patched/notpip/_vendor/distlib/util.py +++ b/pipenv/patched/notpip/_vendor/distlib/util.py @@ -545,16 +545,14 @@ class FileOperator(object): def write_binary_file(self, path, data): self.ensure_dir(os.path.dirname(path)) if not self.dry_run: + if os.path.exists(path): + os.remove(path) with open(path, 'wb') as f: f.write(data) self.record_as_written(path) def write_text_file(self, path, data, encoding): - self.ensure_dir(os.path.dirname(path)) - if not self.dry_run: - with open(path, 'wb') as f: - f.write(data.encode(encoding)) - self.record_as_written(path) + self.write_binary_file(path, data.encode(encoding)) def set_mode(self, bits, mask, files): if os.name == 'posix' or (os.name == 'java' and os._name == 'posix'): @@ -582,7 +580,7 @@ class FileOperator(object): if self.record: self.dirs_created.add(path) - def byte_compile(self, path, optimize=False, force=False, prefix=None): + def byte_compile(self, path, optimize=False, force=False, prefix=None, hashed_invalidation=False): dpath = cache_from_source(path, not optimize) logger.info('Byte-compiling %s to %s', path, dpath) if not self.dry_run: @@ -592,7 +590,10 @@ class FileOperator(object): else: assert path.startswith(prefix) diagpath = path[len(prefix):] - py_compile.compile(path, dpath, diagpath, True) # raise error + compile_kwargs = {} + if hashed_invalidation and hasattr(py_compile, 'PycInvalidationMode'): + compile_kwargs['invalidation_mode'] = py_compile.PycInvalidationMode.CHECKED_HASH + py_compile.compile(path, dpath, diagpath, True, **compile_kwargs) # raise error self.record_as_written(dpath) return dpath diff --git a/pipenv/patched/notpip/_vendor/distlib/wheel.py b/pipenv/patched/notpip/_vendor/distlib/wheel.py index 77372235..b04bfaef 100644 --- a/pipenv/patched/notpip/_vendor/distlib/wheel.py +++ b/pipenv/patched/notpip/_vendor/distlib/wheel.py @@ -442,7 +442,9 @@ class Wheel(object): This can be used to issue any warnings to raise any exceptions. If kwarg ``lib_only`` is True, only the purelib/platlib files are installed, and the headers, scripts, data and dist-info metadata are - not written. + not written. If kwarg ``bytecode_hashed_invalidation`` is True, written + bytecode will try to use file-hash based invalidation (PEP-552) on + supported interpreter versions (CPython 2.7+). The return value is a :class:`InstalledDistribution` instance unless ``options.lib_only`` is True, in which case the return value is ``None``. @@ -451,6 +453,7 @@ class Wheel(object): dry_run = maker.dry_run warner = kwargs.get('warner') lib_only = kwargs.get('lib_only', False) + bc_hashed_invalidation = kwargs.get('bytecode_hashed_invalidation', False) pathname = os.path.join(self.dirname, self.filename) name_ver = '%s-%s' % (self.name, self.version) @@ -557,7 +560,8 @@ class Wheel(object): '%s' % outfile) if bc and outfile.endswith('.py'): try: - pyc = fileop.byte_compile(outfile) + pyc = fileop.byte_compile(outfile, + hashed_invalidation=bc_hashed_invalidation) outfiles.append(pyc) except Exception: # Don't give up if byte-compilation fails, diff --git a/pipenv/patched/notpip/_vendor/idna/core.py b/pipenv/patched/notpip/_vendor/idna/core.py index 090c2c18..104624ad 100644 --- a/pipenv/patched/notpip/_vendor/idna/core.py +++ b/pipenv/patched/notpip/_vendor/idna/core.py @@ -267,10 +267,7 @@ def alabel(label): try: label = label.encode('ascii') - try: - ulabel(label) - except IDNAError: - raise IDNAError('The label {0} is not a valid A-label'.format(label)) + ulabel(label) if not valid_label_length(label): raise IDNAError('Label too long') return label diff --git a/pipenv/patched/notpip/_vendor/idna/idnadata.py b/pipenv/patched/notpip/_vendor/idna/idnadata.py index 17974e23..a80c959d 100644 --- a/pipenv/patched/notpip/_vendor/idna/idnadata.py +++ b/pipenv/patched/notpip/_vendor/idna/idnadata.py @@ -1,6 +1,6 @@ # This file is automatically generated by tools/idna-data -__version__ = "10.0.0" +__version__ = "11.0.0" scripts = { 'Greek': ( 0x37000000374, @@ -49,7 +49,7 @@ scripts = { 0x30210000302a, 0x30380000303c, 0x340000004db6, - 0x4e0000009feb, + 0x4e0000009ff0, 0xf9000000fa6e, 0xfa700000fada, 0x200000002a6d7, @@ -62,7 +62,7 @@ scripts = { 'Hebrew': ( 0x591000005c8, 0x5d0000005eb, - 0x5f0000005f5, + 0x5ef000005f5, 0xfb1d0000fb37, 0xfb380000fb3d, 0xfb3e0000fb3f, @@ -248,6 +248,7 @@ joining_types = { 0x6fb: 68, 0x6fc: 68, 0x6ff: 68, + 0x70f: 84, 0x710: 82, 0x712: 68, 0x713: 68, @@ -522,6 +523,7 @@ joining_types = { 0x1875: 68, 0x1876: 68, 0x1877: 68, + 0x1878: 68, 0x1880: 85, 0x1881: 85, 0x1882: 85, @@ -690,6 +692,70 @@ joining_types = { 0x10bad: 68, 0x10bae: 68, 0x10baf: 85, + 0x10d00: 76, + 0x10d01: 68, + 0x10d02: 68, + 0x10d03: 68, + 0x10d04: 68, + 0x10d05: 68, + 0x10d06: 68, + 0x10d07: 68, + 0x10d08: 68, + 0x10d09: 68, + 0x10d0a: 68, + 0x10d0b: 68, + 0x10d0c: 68, + 0x10d0d: 68, + 0x10d0e: 68, + 0x10d0f: 68, + 0x10d10: 68, + 0x10d11: 68, + 0x10d12: 68, + 0x10d13: 68, + 0x10d14: 68, + 0x10d15: 68, + 0x10d16: 68, + 0x10d17: 68, + 0x10d18: 68, + 0x10d19: 68, + 0x10d1a: 68, + 0x10d1b: 68, + 0x10d1c: 68, + 0x10d1d: 68, + 0x10d1e: 68, + 0x10d1f: 68, + 0x10d20: 68, + 0x10d21: 68, + 0x10d22: 82, + 0x10d23: 68, + 0x10f30: 68, + 0x10f31: 68, + 0x10f32: 68, + 0x10f33: 82, + 0x10f34: 68, + 0x10f35: 68, + 0x10f36: 68, + 0x10f37: 68, + 0x10f38: 68, + 0x10f39: 68, + 0x10f3a: 68, + 0x10f3b: 68, + 0x10f3c: 68, + 0x10f3d: 68, + 0x10f3e: 68, + 0x10f3f: 68, + 0x10f40: 68, + 0x10f41: 68, + 0x10f42: 68, + 0x10f43: 68, + 0x10f44: 68, + 0x10f45: 85, + 0x10f51: 68, + 0x10f52: 68, + 0x10f53: 68, + 0x10f54: 82, + 0x110bd: 85, + 0x110cd: 85, 0x1e900: 68, 0x1e901: 68, 0x1e902: 68, @@ -1034,14 +1100,15 @@ codepoint_classes = { 0x52d0000052e, 0x52f00000530, 0x5590000055a, - 0x56100000587, + 0x56000000587, + 0x58800000589, 0x591000005be, 0x5bf000005c0, 0x5c1000005c3, 0x5c4000005c6, 0x5c7000005c8, 0x5d0000005eb, - 0x5f0000005f3, + 0x5ef000005f3, 0x6100000061b, 0x62000000640, 0x64100000660, @@ -1054,12 +1121,13 @@ codepoint_classes = { 0x7100000074b, 0x74d000007b2, 0x7c0000007f6, + 0x7fd000007fe, 0x8000000082e, 0x8400000085c, 0x8600000086b, 0x8a0000008b5, 0x8b6000008be, - 0x8d4000008e2, + 0x8d3000008e2, 0x8e300000958, 0x96000000964, 0x96600000970, @@ -1077,6 +1145,7 @@ codepoint_classes = { 0x9e0000009e4, 0x9e6000009f2, 0x9fc000009fd, + 0x9fe000009ff, 0xa0100000a04, 0xa0500000a0b, 0xa0f00000a11, @@ -1136,8 +1205,7 @@ codepoint_classes = { 0xbd000000bd1, 0xbd700000bd8, 0xbe600000bf0, - 0xc0000000c04, - 0xc0500000c0d, + 0xc0000000c0d, 0xc0e00000c11, 0xc1200000c29, 0xc2a00000c3a, @@ -1276,7 +1344,7 @@ codepoint_classes = { 0x17dc000017de, 0x17e0000017ea, 0x18100000181a, - 0x182000001878, + 0x182000001879, 0x1880000018ab, 0x18b0000018f6, 0x19000000191f, @@ -1544,11 +1612,11 @@ codepoint_classes = { 0x309d0000309f, 0x30a1000030fb, 0x30fc000030ff, - 0x31050000312f, + 0x310500003130, 0x31a0000031bb, 0x31f000003200, 0x340000004db6, - 0x4e0000009feb, + 0x4e0000009ff0, 0xa0000000a48d, 0xa4d00000a4fe, 0xa5000000a60d, @@ -1655,8 +1723,10 @@ codepoint_classes = { 0xa7a50000a7a6, 0xa7a70000a7a8, 0xa7a90000a7aa, + 0xa7af0000a7b0, 0xa7b50000a7b6, 0xa7b70000a7b8, + 0xa7b90000a7ba, 0xa7f70000a7f8, 0xa7fa0000a828, 0xa8400000a874, @@ -1664,8 +1734,7 @@ codepoint_classes = { 0xa8d00000a8da, 0xa8e00000a8f8, 0xa8fb0000a8fc, - 0xa8fd0000a8fe, - 0xa9000000a92e, + 0xa8fd0000a92e, 0xa9300000a954, 0xa9800000a9c1, 0xa9cf0000a9da, @@ -1743,7 +1812,7 @@ codepoint_classes = { 0x10a0500010a07, 0x10a0c00010a14, 0x10a1500010a18, - 0x10a1900010a34, + 0x10a1900010a36, 0x10a3800010a3b, 0x10a3f00010a40, 0x10a6000010a7d, @@ -1756,6 +1825,11 @@ codepoint_classes = { 0x10b8000010b92, 0x10c0000010c49, 0x10cc000010cf3, + 0x10d0000010d28, + 0x10d3000010d3a, + 0x10f0000010f1d, + 0x10f2700010f28, + 0x10f3000010f51, 0x1100000011047, 0x1106600011070, 0x1107f000110bb, @@ -1763,10 +1837,11 @@ codepoint_classes = { 0x110f0000110fa, 0x1110000011135, 0x1113600011140, + 0x1114400011147, 0x1115000011174, 0x1117600011177, 0x11180000111c5, - 0x111ca000111cd, + 0x111c9000111cd, 0x111d0000111db, 0x111dc000111dd, 0x1120000011212, @@ -1786,7 +1861,7 @@ codepoint_classes = { 0x1132a00011331, 0x1133200011334, 0x113350001133a, - 0x1133c00011345, + 0x1133b00011345, 0x1134700011349, 0x1134b0001134e, 0x1135000011351, @@ -1796,6 +1871,7 @@ codepoint_classes = { 0x1137000011375, 0x114000001144b, 0x114500001145a, + 0x1145e0001145f, 0x11480000114c6, 0x114c7000114c8, 0x114d0000114da, @@ -1807,15 +1883,17 @@ codepoint_classes = { 0x116500001165a, 0x11680000116b8, 0x116c0000116ca, - 0x117000001171a, + 0x117000001171b, 0x1171d0001172c, 0x117300001173a, + 0x118000001183b, 0x118c0000118ea, 0x118ff00011900, 0x11a0000011a3f, 0x11a4700011a48, 0x11a5000011a84, 0x11a8600011a9a, + 0x11a9d00011a9e, 0x11ac000011af9, 0x11c0000011c09, 0x11c0a00011c37, @@ -1831,6 +1909,13 @@ codepoint_classes = { 0x11d3c00011d3e, 0x11d3f00011d48, 0x11d5000011d5a, + 0x11d6000011d66, + 0x11d6700011d69, + 0x11d6a00011d8f, + 0x11d9000011d92, + 0x11d9300011d99, + 0x11da000011daa, + 0x11ee000011ef7, 0x120000001239a, 0x1248000012544, 0x130000001342f, @@ -1845,11 +1930,12 @@ codepoint_classes = { 0x16b5000016b5a, 0x16b6300016b78, 0x16b7d00016b90, + 0x16e6000016e80, 0x16f0000016f45, 0x16f5000016f7f, 0x16f8f00016fa0, 0x16fe000016fe2, - 0x17000000187ed, + 0x17000000187f2, 0x1880000018af3, 0x1b0000001b11f, 0x1b1700001b2fc, diff --git a/pipenv/patched/notpip/_vendor/idna/package_data.py b/pipenv/patched/notpip/_vendor/idna/package_data.py index 39c192ba..257e8989 100644 --- a/pipenv/patched/notpip/_vendor/idna/package_data.py +++ b/pipenv/patched/notpip/_vendor/idna/package_data.py @@ -1,2 +1,2 @@ -__version__ = '2.7' +__version__ = '2.8' diff --git a/pipenv/patched/notpip/_vendor/idna/uts46data.py b/pipenv/patched/notpip/_vendor/idna/uts46data.py index 79731cb9..a68ed4c0 100644 --- a/pipenv/patched/notpip/_vendor/idna/uts46data.py +++ b/pipenv/patched/notpip/_vendor/idna/uts46data.py @@ -4,7 +4,7 @@ """IDNA Mapping Table from UTS46.""" -__version__ = "10.0.0" +__version__ = "11.0.0" def _seg_0(): return [ (0x0, '3'), @@ -1029,11 +1029,8 @@ def _seg_9(): (0x556, 'M', u'ֆ'), (0x557, 'X'), (0x559, 'V'), - (0x560, 'X'), - (0x561, 'V'), (0x587, 'M', u'եւ'), - (0x588, 'X'), - (0x589, 'V'), + (0x588, 'V'), (0x58B, 'X'), (0x58D, 'V'), (0x590, 'X'), @@ -1041,15 +1038,15 @@ def _seg_9(): (0x5C8, 'X'), (0x5D0, 'V'), (0x5EB, 'X'), - (0x5F0, 'V'), + (0x5EF, 'V'), (0x5F5, 'X'), + (0x606, 'V'), + (0x61C, 'X'), + (0x61E, 'V'), ] def _seg_10(): return [ - (0x606, 'V'), - (0x61C, 'X'), - (0x61E, 'V'), (0x675, 'M', u'اٴ'), (0x676, 'M', u'وٴ'), (0x677, 'M', u'ۇٴ'), @@ -1064,7 +1061,7 @@ def _seg_10(): (0x7B2, 'X'), (0x7C0, 'V'), (0x7FB, 'X'), - (0x800, 'V'), + (0x7FD, 'V'), (0x82E, 'X'), (0x830, 'V'), (0x83F, 'X'), @@ -1078,7 +1075,7 @@ def _seg_10(): (0x8B5, 'X'), (0x8B6, 'V'), (0x8BE, 'X'), - (0x8D4, 'V'), + (0x8D3, 'V'), (0x8E2, 'X'), (0x8E3, 'V'), (0x958, 'M', u'क़'), @@ -1118,7 +1115,7 @@ def _seg_10(): (0x9E0, 'V'), (0x9E4, 'X'), (0x9E6, 'V'), - (0x9FE, 'X'), + (0x9FF, 'X'), (0xA01, 'V'), (0xA04, 'X'), (0xA05, 'V'), @@ -1147,19 +1144,19 @@ def _seg_10(): (0xA4E, 'X'), (0xA51, 'V'), (0xA52, 'X'), + (0xA59, 'M', u'ਖ਼'), + (0xA5A, 'M', u'ਗ਼'), + (0xA5B, 'M', u'ਜ਼'), ] def _seg_11(): return [ - (0xA59, 'M', u'ਖ਼'), - (0xA5A, 'M', u'ਗ਼'), - (0xA5B, 'M', u'ਜ਼'), (0xA5C, 'V'), (0xA5D, 'X'), (0xA5E, 'M', u'ਫ਼'), (0xA5F, 'X'), (0xA66, 'V'), - (0xA76, 'X'), + (0xA77, 'X'), (0xA81, 'V'), (0xA84, 'X'), (0xA85, 'V'), @@ -1250,16 +1247,14 @@ def _seg_11(): (0xBE6, 'V'), (0xBFB, 'X'), (0xC00, 'V'), - (0xC04, 'X'), - ] - -def _seg_12(): - return [ - (0xC05, 'V'), (0xC0D, 'X'), (0xC0E, 'V'), (0xC11, 'X'), (0xC12, 'V'), + ] + +def _seg_12(): + return [ (0xC29, 'X'), (0xC2A, 'V'), (0xC3A, 'X'), @@ -1278,8 +1273,6 @@ def _seg_12(): (0xC66, 'V'), (0xC70, 'X'), (0xC78, 'V'), - (0xC84, 'X'), - (0xC85, 'V'), (0xC8D, 'X'), (0xC8E, 'V'), (0xC91, 'X'), @@ -1355,10 +1348,6 @@ def _seg_12(): (0xE83, 'X'), (0xE84, 'V'), (0xE85, 'X'), - ] - -def _seg_13(): - return [ (0xE87, 'V'), (0xE89, 'X'), (0xE8A, 'V'), @@ -1366,6 +1355,10 @@ def _seg_13(): (0xE8D, 'V'), (0xE8E, 'X'), (0xE94, 'V'), + ] + +def _seg_13(): + return [ (0xE98, 'X'), (0xE99, 'V'), (0xEA0, 'X'), @@ -1459,10 +1452,6 @@ def _seg_13(): (0x124E, 'X'), (0x1250, 'V'), (0x1257, 'X'), - ] - -def _seg_14(): - return [ (0x1258, 'V'), (0x1259, 'X'), (0x125A, 'V'), @@ -1470,6 +1459,10 @@ def _seg_14(): (0x1260, 'V'), (0x1289, 'X'), (0x128A, 'V'), + ] + +def _seg_14(): + return [ (0x128E, 'X'), (0x1290, 'V'), (0x12B1, 'X'), @@ -1538,7 +1531,7 @@ def _seg_14(): (0x1810, 'V'), (0x181A, 'X'), (0x1820, 'V'), - (0x1878, 'X'), + (0x1879, 'X'), (0x1880, 'V'), (0x18AB, 'X'), (0x18B0, 'V'), @@ -1563,10 +1556,6 @@ def _seg_14(): (0x19DB, 'X'), (0x19DE, 'V'), (0x1A1C, 'X'), - ] - -def _seg_15(): - return [ (0x1A1E, 'V'), (0x1A5F, 'X'), (0x1A60, 'V'), @@ -1574,6 +1563,10 @@ def _seg_15(): (0x1A7F, 'V'), (0x1A8A, 'X'), (0x1A90, 'V'), + ] + +def _seg_15(): + return [ (0x1A9A, 'X'), (0x1AA0, 'V'), (0x1AAE, 'X'), @@ -1667,10 +1660,6 @@ def _seg_15(): (0x1D68, 'M', u'ρ'), (0x1D69, 'M', u'φ'), (0x1D6A, 'M', u'χ'), - ] - -def _seg_16(): - return [ (0x1D6B, 'V'), (0x1D78, 'M', u'н'), (0x1D79, 'V'), @@ -1678,6 +1667,10 @@ def _seg_16(): (0x1D9C, 'M', u'c'), (0x1D9D, 'M', u'ɕ'), (0x1D9E, 'M', u'ð'), + ] + +def _seg_16(): + return [ (0x1D9F, 'M', u'ɜ'), (0x1DA0, 'M', u'f'), (0x1DA1, 'M', u'ɟ'), @@ -1771,10 +1764,6 @@ def _seg_16(): (0x1E36, 'M', u'ḷ'), (0x1E37, 'V'), (0x1E38, 'M', u'ḹ'), - ] - -def _seg_17(): - return [ (0x1E39, 'V'), (0x1E3A, 'M', u'ḻ'), (0x1E3B, 'V'), @@ -1782,6 +1771,10 @@ def _seg_17(): (0x1E3D, 'V'), (0x1E3E, 'M', u'ḿ'), (0x1E3F, 'V'), + ] + +def _seg_17(): + return [ (0x1E40, 'M', u'ṁ'), (0x1E41, 'V'), (0x1E42, 'M', u'ṃ'), @@ -1875,10 +1868,6 @@ def _seg_17(): (0x1E9F, 'V'), (0x1EA0, 'M', u'ạ'), (0x1EA1, 'V'), - ] - -def _seg_18(): - return [ (0x1EA2, 'M', u'ả'), (0x1EA3, 'V'), (0x1EA4, 'M', u'ấ'), @@ -1886,6 +1875,10 @@ def _seg_18(): (0x1EA6, 'M', u'ầ'), (0x1EA7, 'V'), (0x1EA8, 'M', u'ẩ'), + ] + +def _seg_18(): + return [ (0x1EA9, 'V'), (0x1EAA, 'M', u'ẫ'), (0x1EAB, 'V'), @@ -1979,10 +1972,6 @@ def _seg_18(): (0x1F0B, 'M', u'ἃ'), (0x1F0C, 'M', u'ἄ'), (0x1F0D, 'M', u'ἅ'), - ] - -def _seg_19(): - return [ (0x1F0E, 'M', u'ἆ'), (0x1F0F, 'M', u'ἇ'), (0x1F10, 'V'), @@ -1990,6 +1979,10 @@ def _seg_19(): (0x1F18, 'M', u'ἐ'), (0x1F19, 'M', u'ἑ'), (0x1F1A, 'M', u'ἒ'), + ] + +def _seg_19(): + return [ (0x1F1B, 'M', u'ἓ'), (0x1F1C, 'M', u'ἔ'), (0x1F1D, 'M', u'ἕ'), @@ -2083,10 +2076,6 @@ def _seg_19(): (0x1F9A, 'M', u'ἢι'), (0x1F9B, 'M', u'ἣι'), (0x1F9C, 'M', u'ἤι'), - ] - -def _seg_20(): - return [ (0x1F9D, 'M', u'ἥι'), (0x1F9E, 'M', u'ἦι'), (0x1F9F, 'M', u'ἧι'), @@ -2094,6 +2083,10 @@ def _seg_20(): (0x1FA1, 'M', u'ὡι'), (0x1FA2, 'M', u'ὢι'), (0x1FA3, 'M', u'ὣι'), + ] + +def _seg_20(): + return [ (0x1FA4, 'M', u'ὤι'), (0x1FA5, 'M', u'ὥι'), (0x1FA6, 'M', u'ὦι'), @@ -2187,10 +2180,6 @@ def _seg_20(): (0x2024, 'X'), (0x2027, 'V'), (0x2028, 'X'), - ] - -def _seg_21(): - return [ (0x202F, '3', u' '), (0x2030, 'V'), (0x2033, 'M', u'′′'), @@ -2198,6 +2187,10 @@ def _seg_21(): (0x2035, 'V'), (0x2036, 'M', u'‵‵'), (0x2037, 'M', u'‵‵‵'), + ] + +def _seg_21(): + return [ (0x2038, 'V'), (0x203C, '3', u'!!'), (0x203D, 'V'), @@ -2291,10 +2284,6 @@ def _seg_21(): (0x2120, 'M', u'sm'), (0x2121, 'M', u'tel'), (0x2122, 'M', u'tm'), - ] - -def _seg_22(): - return [ (0x2123, 'V'), (0x2124, 'M', u'z'), (0x2125, 'V'), @@ -2302,6 +2291,10 @@ def _seg_22(): (0x2127, 'V'), (0x2128, 'M', u'z'), (0x2129, 'V'), + ] + +def _seg_22(): + return [ (0x212A, 'M', u'k'), (0x212B, 'M', u'å'), (0x212C, 'M', u'b'), @@ -2395,10 +2388,6 @@ def _seg_22(): (0x226E, '3'), (0x2270, 'V'), (0x2329, 'M', u'〈'), - ] - -def _seg_23(): - return [ (0x232A, 'M', u'〉'), (0x232B, 'V'), (0x2427, 'X'), @@ -2406,6 +2395,10 @@ def _seg_23(): (0x244B, 'X'), (0x2460, 'M', u'1'), (0x2461, 'M', u'2'), + ] + +def _seg_23(): + return [ (0x2462, 'M', u'3'), (0x2463, 'M', u'4'), (0x2464, 'M', u'5'), @@ -2499,10 +2492,6 @@ def _seg_23(): (0x24CF, 'M', u'z'), (0x24D0, 'M', u'a'), (0x24D1, 'M', u'b'), - ] - -def _seg_24(): - return [ (0x24D2, 'M', u'c'), (0x24D3, 'M', u'd'), (0x24D4, 'M', u'e'), @@ -2510,6 +2499,10 @@ def _seg_24(): (0x24D6, 'M', u'g'), (0x24D7, 'M', u'h'), (0x24D8, 'M', u'i'), + ] + +def _seg_24(): + return [ (0x24D9, 'M', u'j'), (0x24DA, 'M', u'k'), (0x24DB, 'M', u'l'), @@ -2541,13 +2534,9 @@ def _seg_24(): (0x2B76, 'V'), (0x2B96, 'X'), (0x2B98, 'V'), - (0x2BBA, 'X'), - (0x2BBD, 'V'), (0x2BC9, 'X'), (0x2BCA, 'V'), - (0x2BD3, 'X'), - (0x2BEC, 'V'), - (0x2BF0, 'X'), + (0x2BFF, 'X'), (0x2C00, 'M', u'ⰰ'), (0x2C01, 'M', u'ⰱ'), (0x2C02, 'M', u'ⰲ'), @@ -2603,10 +2592,6 @@ def _seg_24(): (0x2C62, 'M', u'ɫ'), (0x2C63, 'M', u'ᵽ'), (0x2C64, 'M', u'ɽ'), - ] - -def _seg_25(): - return [ (0x2C65, 'V'), (0x2C67, 'M', u'ⱨ'), (0x2C68, 'V'), @@ -2618,6 +2603,10 @@ def _seg_25(): (0x2C6E, 'M', u'ɱ'), (0x2C6F, 'M', u'ɐ'), (0x2C70, 'M', u'ɒ'), + ] + +def _seg_25(): + return [ (0x2C71, 'V'), (0x2C72, 'M', u'ⱳ'), (0x2C73, 'V'), @@ -2707,10 +2696,6 @@ def _seg_25(): (0x2CCD, 'V'), (0x2CCE, 'M', u'ⳏ'), (0x2CCF, 'V'), - ] - -def _seg_26(): - return [ (0x2CD0, 'M', u'ⳑ'), (0x2CD1, 'V'), (0x2CD2, 'M', u'ⳓ'), @@ -2722,6 +2707,10 @@ def _seg_26(): (0x2CD8, 'M', u'ⳙ'), (0x2CD9, 'V'), (0x2CDA, 'M', u'ⳛ'), + ] + +def _seg_26(): + return [ (0x2CDB, 'V'), (0x2CDC, 'M', u'ⳝ'), (0x2CDD, 'V'), @@ -2768,7 +2757,7 @@ def _seg_26(): (0x2DD8, 'V'), (0x2DDF, 'X'), (0x2DE0, 'V'), - (0x2E4A, 'X'), + (0x2E4F, 'X'), (0x2E80, 'V'), (0x2E9A, 'X'), (0x2E9B, 'V'), @@ -2811,10 +2800,6 @@ def _seg_26(): (0x2F20, 'M', u'士'), (0x2F21, 'M', u'夂'), (0x2F22, 'M', u'夊'), - ] - -def _seg_27(): - return [ (0x2F23, 'M', u'夕'), (0x2F24, 'M', u'大'), (0x2F25, 'M', u'女'), @@ -2826,6 +2811,10 @@ def _seg_27(): (0x2F2B, 'M', u'尸'), (0x2F2C, 'M', u'屮'), (0x2F2D, 'M', u'山'), + ] + +def _seg_27(): + return [ (0x2F2E, 'M', u'巛'), (0x2F2F, 'M', u'工'), (0x2F30, 'M', u'己'), @@ -2915,10 +2904,6 @@ def _seg_27(): (0x2F84, 'M', u'至'), (0x2F85, 'M', u'臼'), (0x2F86, 'M', u'舌'), - ] - -def _seg_28(): - return [ (0x2F87, 'M', u'舛'), (0x2F88, 'M', u'舟'), (0x2F89, 'M', u'艮'), @@ -2930,6 +2915,10 @@ def _seg_28(): (0x2F8F, 'M', u'行'), (0x2F90, 'M', u'衣'), (0x2F91, 'M', u'襾'), + ] + +def _seg_28(): + return [ (0x2F92, 'M', u'見'), (0x2F93, 'M', u'角'), (0x2F94, 'M', u'言'), @@ -3019,13 +3008,9 @@ def _seg_28(): (0x309F, 'M', u'より'), (0x30A0, 'V'), (0x30FF, 'M', u'コト'), - ] - -def _seg_29(): - return [ (0x3100, 'X'), (0x3105, 'V'), - (0x312F, 'X'), + (0x3130, 'X'), (0x3131, 'M', u'ᄀ'), (0x3132, 'M', u'ᄁ'), (0x3133, 'M', u'ᆪ'), @@ -3034,6 +3019,10 @@ def _seg_29(): (0x3136, 'M', u'ᆭ'), (0x3137, 'M', u'ᄃ'), (0x3138, 'M', u'ᄄ'), + ] + +def _seg_29(): + return [ (0x3139, 'M', u'ᄅ'), (0x313A, 'M', u'ᆰ'), (0x313B, 'M', u'ᆱ'), @@ -3123,10 +3112,6 @@ def _seg_29(): (0x318F, 'X'), (0x3190, 'V'), (0x3192, 'M', u'一'), - ] - -def _seg_30(): - return [ (0x3193, 'M', u'二'), (0x3194, 'M', u'三'), (0x3195, 'M', u'四'), @@ -3138,6 +3123,10 @@ def _seg_30(): (0x319B, 'M', u'丙'), (0x319C, 'M', u'丁'), (0x319D, 'M', u'天'), + ] + +def _seg_30(): + return [ (0x319E, 'M', u'地'), (0x319F, 'M', u'人'), (0x31A0, 'V'), @@ -3227,10 +3216,6 @@ def _seg_30(): (0x3256, 'M', u'26'), (0x3257, 'M', u'27'), (0x3258, 'M', u'28'), - ] - -def _seg_31(): - return [ (0x3259, 'M', u'29'), (0x325A, 'M', u'30'), (0x325B, 'M', u'31'), @@ -3242,6 +3227,10 @@ def _seg_31(): (0x3261, 'M', u'ᄂ'), (0x3262, 'M', u'ᄃ'), (0x3263, 'M', u'ᄅ'), + ] + +def _seg_31(): + return [ (0x3264, 'M', u'ᄆ'), (0x3265, 'M', u'ᄇ'), (0x3266, 'M', u'ᄉ'), @@ -3331,10 +3320,6 @@ def _seg_31(): (0x32BA, 'M', u'45'), (0x32BB, 'M', u'46'), (0x32BC, 'M', u'47'), - ] - -def _seg_32(): - return [ (0x32BD, 'M', u'48'), (0x32BE, 'M', u'49'), (0x32BF, 'M', u'50'), @@ -3346,6 +3331,10 @@ def _seg_32(): (0x32C5, 'M', u'6月'), (0x32C6, 'M', u'7月'), (0x32C7, 'M', u'8月'), + ] + +def _seg_32(): + return [ (0x32C8, 'M', u'9月'), (0x32C9, 'M', u'10月'), (0x32CA, 'M', u'11月'), @@ -3435,10 +3424,6 @@ def _seg_32(): (0x331E, 'M', u'コーポ'), (0x331F, 'M', u'サイクル'), (0x3320, 'M', u'サンチーム'), - ] - -def _seg_33(): - return [ (0x3321, 'M', u'シリング'), (0x3322, 'M', u'センチ'), (0x3323, 'M', u'セント'), @@ -3450,6 +3435,10 @@ def _seg_33(): (0x3329, 'M', u'ノット'), (0x332A, 'M', u'ハイツ'), (0x332B, 'M', u'パーセント'), + ] + +def _seg_33(): + return [ (0x332C, 'M', u'パーツ'), (0x332D, 'M', u'バーレル'), (0x332E, 'M', u'ピアストル'), @@ -3539,10 +3528,6 @@ def _seg_33(): (0x3382, 'M', u'μa'), (0x3383, 'M', u'ma'), (0x3384, 'M', u'ka'), - ] - -def _seg_34(): - return [ (0x3385, 'M', u'kb'), (0x3386, 'M', u'mb'), (0x3387, 'M', u'gb'), @@ -3554,6 +3539,10 @@ def _seg_34(): (0x338D, 'M', u'μg'), (0x338E, 'M', u'mg'), (0x338F, 'M', u'kg'), + ] + +def _seg_34(): + return [ (0x3390, 'M', u'hz'), (0x3391, 'M', u'khz'), (0x3392, 'M', u'mhz'), @@ -3643,10 +3632,6 @@ def _seg_34(): (0x33E6, 'M', u'7日'), (0x33E7, 'M', u'8日'), (0x33E8, 'M', u'9日'), - ] - -def _seg_35(): - return [ (0x33E9, 'M', u'10日'), (0x33EA, 'M', u'11日'), (0x33EB, 'M', u'12日'), @@ -3658,6 +3643,10 @@ def _seg_35(): (0x33F1, 'M', u'18日'), (0x33F2, 'M', u'19日'), (0x33F3, 'M', u'20日'), + ] + +def _seg_35(): + return [ (0x33F4, 'M', u'21日'), (0x33F5, 'M', u'22日'), (0x33F6, 'M', u'23日'), @@ -3673,7 +3662,7 @@ def _seg_35(): (0x3400, 'V'), (0x4DB6, 'X'), (0x4DC0, 'V'), - (0x9FEB, 'X'), + (0x9FF0, 'X'), (0xA000, 'V'), (0xA48D, 'X'), (0xA490, 'V'), @@ -3747,10 +3736,6 @@ def _seg_35(): (0xA692, 'M', u'ꚓ'), (0xA693, 'V'), (0xA694, 'M', u'ꚕ'), - ] - -def _seg_36(): - return [ (0xA695, 'V'), (0xA696, 'M', u'ꚗ'), (0xA697, 'V'), @@ -3762,6 +3747,10 @@ def _seg_36(): (0xA69D, 'M', u'ь'), (0xA69E, 'V'), (0xA6F8, 'X'), + ] + +def _seg_36(): + return [ (0xA700, 'V'), (0xA722, 'M', u'ꜣ'), (0xA723, 'V'), @@ -3851,10 +3840,6 @@ def _seg_36(): (0xA780, 'M', u'ꞁ'), (0xA781, 'V'), (0xA782, 'M', u'ꞃ'), - ] - -def _seg_37(): - return [ (0xA783, 'V'), (0xA784, 'M', u'ꞅ'), (0xA785, 'V'), @@ -3866,6 +3851,10 @@ def _seg_37(): (0xA78E, 'V'), (0xA790, 'M', u'ꞑ'), (0xA791, 'V'), + ] + +def _seg_37(): + return [ (0xA792, 'M', u'ꞓ'), (0xA793, 'V'), (0xA796, 'M', u'ꞗ'), @@ -3893,7 +3882,7 @@ def _seg_37(): (0xA7AC, 'M', u'ɡ'), (0xA7AD, 'M', u'ɬ'), (0xA7AE, 'M', u'ɪ'), - (0xA7AF, 'X'), + (0xA7AF, 'V'), (0xA7B0, 'M', u'ʞ'), (0xA7B1, 'M', u'ʇ'), (0xA7B2, 'M', u'ʝ'), @@ -3903,6 +3892,8 @@ def _seg_37(): (0xA7B6, 'M', u'ꞷ'), (0xA7B7, 'V'), (0xA7B8, 'X'), + (0xA7B9, 'V'), + (0xA7BA, 'X'), (0xA7F7, 'V'), (0xA7F8, 'M', u'ħ'), (0xA7F9, 'M', u'œ'), @@ -3917,8 +3908,6 @@ def _seg_37(): (0xA8CE, 'V'), (0xA8DA, 'X'), (0xA8E0, 'V'), - (0xA8FE, 'X'), - (0xA900, 'V'), (0xA954, 'X'), (0xA95F, 'V'), (0xA97D, 'X'), @@ -3955,10 +3944,6 @@ def _seg_37(): (0xAB5F, 'M', u'ꭒ'), (0xAB60, 'V'), (0xAB66, 'X'), - ] - -def _seg_38(): - return [ (0xAB70, 'M', u'Ꭰ'), (0xAB71, 'M', u'Ꭱ'), (0xAB72, 'M', u'Ꭲ'), @@ -3970,6 +3955,10 @@ def _seg_38(): (0xAB78, 'M', u'Ꭸ'), (0xAB79, 'M', u'Ꭹ'), (0xAB7A, 'M', u'Ꭺ'), + ] + +def _seg_38(): + return [ (0xAB7B, 'M', u'Ꭻ'), (0xAB7C, 'M', u'Ꭼ'), (0xAB7D, 'M', u'Ꭽ'), @@ -4059,10 +4048,6 @@ def _seg_38(): (0xF907, 'M', u'龜'), (0xF909, 'M', u'契'), (0xF90A, 'M', u'金'), - ] - -def _seg_39(): - return [ (0xF90B, 'M', u'喇'), (0xF90C, 'M', u'奈'), (0xF90D, 'M', u'懶'), @@ -4074,6 +4059,10 @@ def _seg_39(): (0xF913, 'M', u'邏'), (0xF914, 'M', u'樂'), (0xF915, 'M', u'洛'), + ] + +def _seg_39(): + return [ (0xF916, 'M', u'烙'), (0xF917, 'M', u'珞'), (0xF918, 'M', u'落'), @@ -4163,10 +4152,6 @@ def _seg_39(): (0xF96C, 'M', u'塞'), (0xF96D, 'M', u'省'), (0xF96E, 'M', u'葉'), - ] - -def _seg_40(): - return [ (0xF96F, 'M', u'說'), (0xF970, 'M', u'殺'), (0xF971, 'M', u'辰'), @@ -4178,6 +4163,10 @@ def _seg_40(): (0xF977, 'M', u'亮'), (0xF978, 'M', u'兩'), (0xF979, 'M', u'凉'), + ] + +def _seg_40(): + return [ (0xF97A, 'M', u'梁'), (0xF97B, 'M', u'糧'), (0xF97C, 'M', u'良'), @@ -4267,10 +4256,6 @@ def _seg_40(): (0xF9D0, 'M', u'類'), (0xF9D1, 'M', u'六'), (0xF9D2, 'M', u'戮'), - ] - -def _seg_41(): - return [ (0xF9D3, 'M', u'陸'), (0xF9D4, 'M', u'倫'), (0xF9D5, 'M', u'崙'), @@ -4282,6 +4267,10 @@ def _seg_41(): (0xF9DB, 'M', u'率'), (0xF9DC, 'M', u'隆'), (0xF9DD, 'M', u'利'), + ] + +def _seg_41(): + return [ (0xF9DE, 'M', u'吏'), (0xF9DF, 'M', u'履'), (0xF9E0, 'M', u'易'), @@ -4371,10 +4360,6 @@ def _seg_41(): (0xFA39, 'M', u'塀'), (0xFA3A, 'M', u'墨'), (0xFA3B, 'M', u'層'), - ] - -def _seg_42(): - return [ (0xFA3C, 'M', u'屮'), (0xFA3D, 'M', u'悔'), (0xFA3E, 'M', u'慨'), @@ -4386,6 +4371,10 @@ def _seg_42(): (0xFA44, 'M', u'梅'), (0xFA45, 'M', u'海'), (0xFA46, 'M', u'渚'), + ] + +def _seg_42(): + return [ (0xFA47, 'M', u'漢'), (0xFA48, 'M', u'煮'), (0xFA49, 'M', u'爫'), @@ -4475,10 +4464,6 @@ def _seg_42(): (0xFA9F, 'M', u'犯'), (0xFAA0, 'M', u'猪'), (0xFAA1, 'M', u'瑱'), - ] - -def _seg_43(): - return [ (0xFAA2, 'M', u'甆'), (0xFAA3, 'M', u'画'), (0xFAA4, 'M', u'瘝'), @@ -4490,6 +4475,10 @@ def _seg_43(): (0xFAAA, 'M', u'着'), (0xFAAB, 'M', u'磌'), (0xFAAC, 'M', u'窱'), + ] + +def _seg_43(): + return [ (0xFAAD, 'M', u'節'), (0xFAAE, 'M', u'类'), (0xFAAF, 'M', u'絛'), @@ -4579,10 +4568,6 @@ def _seg_43(): (0xFB38, 'M', u'טּ'), (0xFB39, 'M', u'יּ'), (0xFB3A, 'M', u'ךּ'), - ] - -def _seg_44(): - return [ (0xFB3B, 'M', u'כּ'), (0xFB3C, 'M', u'לּ'), (0xFB3D, 'X'), @@ -4594,6 +4579,10 @@ def _seg_44(): (0xFB43, 'M', u'ףּ'), (0xFB44, 'M', u'פּ'), (0xFB45, 'X'), + ] + +def _seg_44(): + return [ (0xFB46, 'M', u'צּ'), (0xFB47, 'M', u'קּ'), (0xFB48, 'M', u'רּ'), @@ -4683,10 +4672,6 @@ def _seg_44(): (0xFC19, 'M', u'خج'), (0xFC1A, 'M', u'خح'), (0xFC1B, 'M', u'خم'), - ] - -def _seg_45(): - return [ (0xFC1C, 'M', u'سج'), (0xFC1D, 'M', u'سح'), (0xFC1E, 'M', u'سخ'), @@ -4698,6 +4683,10 @@ def _seg_45(): (0xFC24, 'M', u'ضخ'), (0xFC25, 'M', u'ضم'), (0xFC26, 'M', u'طح'), + ] + +def _seg_45(): + return [ (0xFC27, 'M', u'طم'), (0xFC28, 'M', u'ظم'), (0xFC29, 'M', u'عج'), @@ -4787,10 +4776,6 @@ def _seg_45(): (0xFC7D, 'M', u'في'), (0xFC7E, 'M', u'قى'), (0xFC7F, 'M', u'قي'), - ] - -def _seg_46(): - return [ (0xFC80, 'M', u'كا'), (0xFC81, 'M', u'كل'), (0xFC82, 'M', u'كم'), @@ -4802,6 +4787,10 @@ def _seg_46(): (0xFC88, 'M', u'ما'), (0xFC89, 'M', u'مم'), (0xFC8A, 'M', u'نر'), + ] + +def _seg_46(): + return [ (0xFC8B, 'M', u'نز'), (0xFC8C, 'M', u'نم'), (0xFC8D, 'M', u'نن'), @@ -4891,10 +4880,6 @@ def _seg_46(): (0xFCE1, 'M', u'بم'), (0xFCE2, 'M', u'به'), (0xFCE3, 'M', u'تم'), - ] - -def _seg_47(): - return [ (0xFCE4, 'M', u'ته'), (0xFCE5, 'M', u'ثم'), (0xFCE6, 'M', u'ثه'), @@ -4906,6 +4891,10 @@ def _seg_47(): (0xFCEC, 'M', u'كم'), (0xFCED, 'M', u'لم'), (0xFCEE, 'M', u'نم'), + ] + +def _seg_47(): + return [ (0xFCEF, 'M', u'نه'), (0xFCF0, 'M', u'يم'), (0xFCF1, 'M', u'يه'), @@ -4995,10 +4984,6 @@ def _seg_47(): (0xFD57, 'M', u'تمخ'), (0xFD58, 'M', u'جمح'), (0xFD5A, 'M', u'حمي'), - ] - -def _seg_48(): - return [ (0xFD5B, 'M', u'حمى'), (0xFD5C, 'M', u'سحج'), (0xFD5D, 'M', u'سجح'), @@ -5010,6 +4995,10 @@ def _seg_48(): (0xFD66, 'M', u'صمم'), (0xFD67, 'M', u'شحم'), (0xFD69, 'M', u'شجي'), + ] + +def _seg_48(): + return [ (0xFD6A, 'M', u'شمخ'), (0xFD6C, 'M', u'شمم'), (0xFD6E, 'M', u'ضحى'), @@ -5099,10 +5088,6 @@ def _seg_48(): (0xFDF3, 'M', u'اكبر'), (0xFDF4, 'M', u'محمد'), (0xFDF5, 'M', u'صلعم'), - ] - -def _seg_49(): - return [ (0xFDF6, 'M', u'رسول'), (0xFDF7, 'M', u'عليه'), (0xFDF8, 'M', u'وسلم'), @@ -5114,6 +5099,10 @@ def _seg_49(): (0xFDFE, 'X'), (0xFE00, 'I'), (0xFE10, '3', u','), + ] + +def _seg_49(): + return [ (0xFE11, 'M', u'、'), (0xFE12, 'X'), (0xFE13, '3', u':'), @@ -5203,10 +5192,6 @@ def _seg_49(): (0xFE8F, 'M', u'ب'), (0xFE93, 'M', u'ة'), (0xFE95, 'M', u'ت'), - ] - -def _seg_50(): - return [ (0xFE99, 'M', u'ث'), (0xFE9D, 'M', u'ج'), (0xFEA1, 'M', u'ح'), @@ -5218,6 +5203,10 @@ def _seg_50(): (0xFEB1, 'M', u'س'), (0xFEB5, 'M', u'ش'), (0xFEB9, 'M', u'ص'), + ] + +def _seg_50(): + return [ (0xFEBD, 'M', u'ض'), (0xFEC1, 'M', u'ط'), (0xFEC5, 'M', u'ظ'), @@ -5307,10 +5296,6 @@ def _seg_50(): (0xFF41, 'M', u'a'), (0xFF42, 'M', u'b'), (0xFF43, 'M', u'c'), - ] - -def _seg_51(): - return [ (0xFF44, 'M', u'd'), (0xFF45, 'M', u'e'), (0xFF46, 'M', u'f'), @@ -5322,6 +5307,10 @@ def _seg_51(): (0xFF4C, 'M', u'l'), (0xFF4D, 'M', u'm'), (0xFF4E, 'M', u'n'), + ] + +def _seg_51(): + return [ (0xFF4F, 'M', u'o'), (0xFF50, 'M', u'p'), (0xFF51, 'M', u'q'), @@ -5411,10 +5400,6 @@ def _seg_51(): (0xFFA5, 'M', u'ᆬ'), (0xFFA6, 'M', u'ᆭ'), (0xFFA7, 'M', u'ᄃ'), - ] - -def _seg_52(): - return [ (0xFFA8, 'M', u'ᄄ'), (0xFFA9, 'M', u'ᄅ'), (0xFFAA, 'M', u'ᆰ'), @@ -5426,6 +5411,10 @@ def _seg_52(): (0xFFB0, 'M', u'ᄚ'), (0xFFB1, 'M', u'ᄆ'), (0xFFB2, 'M', u'ᄇ'), + ] + +def _seg_52(): + return [ (0xFFB3, 'M', u'ᄈ'), (0xFFB4, 'M', u'ᄡ'), (0xFFB5, 'M', u'ᄉ'), @@ -5515,10 +5504,6 @@ def _seg_52(): (0x10300, 'V'), (0x10324, 'X'), (0x1032D, 'V'), - ] - -def _seg_53(): - return [ (0x1034B, 'X'), (0x10350, 'V'), (0x1037B, 'X'), @@ -5530,6 +5515,10 @@ def _seg_53(): (0x103D6, 'X'), (0x10400, 'M', u'𐐨'), (0x10401, 'M', u'𐐩'), + ] + +def _seg_53(): + return [ (0x10402, 'M', u'𐐪'), (0x10403, 'M', u'𐐫'), (0x10404, 'M', u'𐐬'), @@ -5619,10 +5608,6 @@ def _seg_53(): (0x10570, 'X'), (0x10600, 'V'), (0x10737, 'X'), - ] - -def _seg_54(): - return [ (0x10740, 'V'), (0x10756, 'X'), (0x10760, 'V'), @@ -5634,6 +5619,10 @@ def _seg_54(): (0x1080A, 'V'), (0x10836, 'X'), (0x10837, 'V'), + ] + +def _seg_54(): + return [ (0x10839, 'X'), (0x1083C, 'V'), (0x1083D, 'X'), @@ -5666,11 +5655,11 @@ def _seg_54(): (0x10A15, 'V'), (0x10A18, 'X'), (0x10A19, 'V'), - (0x10A34, 'X'), + (0x10A36, 'X'), (0x10A38, 'V'), (0x10A3B, 'X'), (0x10A3F, 'V'), - (0x10A48, 'X'), + (0x10A49, 'X'), (0x10A50, 'V'), (0x10A59, 'X'), (0x10A60, 'V'), @@ -5723,10 +5712,6 @@ def _seg_54(): (0x10C9B, 'M', u'𐳛'), (0x10C9C, 'M', u'𐳜'), (0x10C9D, 'M', u'𐳝'), - ] - -def _seg_55(): - return [ (0x10C9E, 'M', u'𐳞'), (0x10C9F, 'M', u'𐳟'), (0x10CA0, 'M', u'𐳠'), @@ -5738,6 +5723,10 @@ def _seg_55(): (0x10CA6, 'M', u'𐳦'), (0x10CA7, 'M', u'𐳧'), (0x10CA8, 'M', u'𐳨'), + ] + +def _seg_55(): + return [ (0x10CA9, 'M', u'𐳩'), (0x10CAA, 'M', u'𐳪'), (0x10CAB, 'M', u'𐳫'), @@ -5752,9 +5741,15 @@ def _seg_55(): (0x10CC0, 'V'), (0x10CF3, 'X'), (0x10CFA, 'V'), - (0x10D00, 'X'), + (0x10D28, 'X'), + (0x10D30, 'V'), + (0x10D3A, 'X'), (0x10E60, 'V'), (0x10E7F, 'X'), + (0x10F00, 'V'), + (0x10F28, 'X'), + (0x10F30, 'V'), + (0x10F5A, 'X'), (0x11000, 'V'), (0x1104E, 'X'), (0x11052, 'V'), @@ -5770,7 +5765,7 @@ def _seg_55(): (0x11100, 'V'), (0x11135, 'X'), (0x11136, 'V'), - (0x11144, 'X'), + (0x11147, 'X'), (0x11150, 'V'), (0x11177, 'X'), (0x11180, 'V'), @@ -5811,7 +5806,7 @@ def _seg_55(): (0x11334, 'X'), (0x11335, 'V'), (0x1133A, 'X'), - (0x1133C, 'V'), + (0x1133B, 'V'), (0x11345, 'X'), (0x11347, 'V'), (0x11349, 'X'), @@ -5827,16 +5822,16 @@ def _seg_55(): (0x1136D, 'X'), (0x11370, 'V'), (0x11375, 'X'), - ] - -def _seg_56(): - return [ (0x11400, 'V'), (0x1145A, 'X'), (0x1145B, 'V'), (0x1145C, 'X'), (0x1145D, 'V'), - (0x1145E, 'X'), + ] + +def _seg_56(): + return [ + (0x1145F, 'X'), (0x11480, 'V'), (0x114C8, 'X'), (0x114D0, 'V'), @@ -5856,11 +5851,13 @@ def _seg_56(): (0x116C0, 'V'), (0x116CA, 'X'), (0x11700, 'V'), - (0x1171A, 'X'), + (0x1171B, 'X'), (0x1171D, 'V'), (0x1172C, 'X'), (0x11730, 'V'), (0x11740, 'X'), + (0x11800, 'V'), + (0x1183C, 'X'), (0x118A0, 'M', u'𑣀'), (0x118A1, 'M', u'𑣁'), (0x118A2, 'M', u'𑣂'), @@ -5902,8 +5899,6 @@ def _seg_56(): (0x11A50, 'V'), (0x11A84, 'X'), (0x11A86, 'V'), - (0x11A9D, 'X'), - (0x11A9E, 'V'), (0x11AA3, 'X'), (0x11AC0, 'V'), (0x11AF9, 'X'), @@ -5931,14 +5926,28 @@ def _seg_56(): (0x11D3B, 'X'), (0x11D3C, 'V'), (0x11D3E, 'X'), - ] - -def _seg_57(): - return [ (0x11D3F, 'V'), (0x11D48, 'X'), (0x11D50, 'V'), (0x11D5A, 'X'), + (0x11D60, 'V'), + ] + +def _seg_57(): + return [ + (0x11D66, 'X'), + (0x11D67, 'V'), + (0x11D69, 'X'), + (0x11D6A, 'V'), + (0x11D8F, 'X'), + (0x11D90, 'V'), + (0x11D92, 'X'), + (0x11D93, 'V'), + (0x11D99, 'X'), + (0x11DA0, 'V'), + (0x11DAA, 'X'), + (0x11EE0, 'V'), + (0x11EF9, 'X'), (0x12000, 'V'), (0x1239A, 'X'), (0x12400, 'V'), @@ -5973,6 +5982,8 @@ def _seg_57(): (0x16B78, 'X'), (0x16B7D, 'V'), (0x16B90, 'X'), + (0x16E60, 'V'), + (0x16E9B, 'X'), (0x16F00, 'V'), (0x16F45, 'X'), (0x16F50, 'V'), @@ -5982,7 +5993,7 @@ def _seg_57(): (0x16FE0, 'V'), (0x16FE2, 'X'), (0x17000, 'V'), - (0x187ED, 'X'), + (0x187F2, 'X'), (0x18800, 'V'), (0x18AF3, 'X'), (0x1B000, 'V'), @@ -6024,21 +6035,23 @@ def _seg_57(): (0x1D1C1, 'V'), (0x1D1E9, 'X'), (0x1D200, 'V'), + ] + +def _seg_58(): + return [ (0x1D246, 'X'), + (0x1D2E0, 'V'), + (0x1D2F4, 'X'), (0x1D300, 'V'), (0x1D357, 'X'), (0x1D360, 'V'), - (0x1D372, 'X'), + (0x1D379, 'X'), (0x1D400, 'M', u'a'), (0x1D401, 'M', u'b'), (0x1D402, 'M', u'c'), (0x1D403, 'M', u'd'), (0x1D404, 'M', u'e'), (0x1D405, 'M', u'f'), - ] - -def _seg_58(): - return [ (0x1D406, 'M', u'g'), (0x1D407, 'M', u'h'), (0x1D408, 'M', u'i'), @@ -6126,6 +6139,10 @@ def _seg_58(): (0x1D45A, 'M', u'm'), (0x1D45B, 'M', u'n'), (0x1D45C, 'M', u'o'), + ] + +def _seg_59(): + return [ (0x1D45D, 'M', u'p'), (0x1D45E, 'M', u'q'), (0x1D45F, 'M', u'r'), @@ -6139,10 +6156,6 @@ def _seg_58(): (0x1D467, 'M', u'z'), (0x1D468, 'M', u'a'), (0x1D469, 'M', u'b'), - ] - -def _seg_59(): - return [ (0x1D46A, 'M', u'c'), (0x1D46B, 'M', u'd'), (0x1D46C, 'M', u'e'), @@ -6230,6 +6243,10 @@ def _seg_59(): (0x1D4C1, 'M', u'l'), (0x1D4C2, 'M', u'm'), (0x1D4C3, 'M', u'n'), + ] + +def _seg_60(): + return [ (0x1D4C4, 'X'), (0x1D4C5, 'M', u'p'), (0x1D4C6, 'M', u'q'), @@ -6243,10 +6260,6 @@ def _seg_59(): (0x1D4CE, 'M', u'y'), (0x1D4CF, 'M', u'z'), (0x1D4D0, 'M', u'a'), - ] - -def _seg_60(): - return [ (0x1D4D1, 'M', u'b'), (0x1D4D2, 'M', u'c'), (0x1D4D3, 'M', u'd'), @@ -6334,6 +6347,10 @@ def _seg_60(): (0x1D526, 'M', u'i'), (0x1D527, 'M', u'j'), (0x1D528, 'M', u'k'), + ] + +def _seg_61(): + return [ (0x1D529, 'M', u'l'), (0x1D52A, 'M', u'm'), (0x1D52B, 'M', u'n'), @@ -6347,10 +6364,6 @@ def _seg_60(): (0x1D533, 'M', u'v'), (0x1D534, 'M', u'w'), (0x1D535, 'M', u'x'), - ] - -def _seg_61(): - return [ (0x1D536, 'M', u'y'), (0x1D537, 'M', u'z'), (0x1D538, 'M', u'a'), @@ -6438,6 +6451,10 @@ def _seg_61(): (0x1D58C, 'M', u'g'), (0x1D58D, 'M', u'h'), (0x1D58E, 'M', u'i'), + ] + +def _seg_62(): + return [ (0x1D58F, 'M', u'j'), (0x1D590, 'M', u'k'), (0x1D591, 'M', u'l'), @@ -6451,10 +6468,6 @@ def _seg_61(): (0x1D599, 'M', u't'), (0x1D59A, 'M', u'u'), (0x1D59B, 'M', u'v'), - ] - -def _seg_62(): - return [ (0x1D59C, 'M', u'w'), (0x1D59D, 'M', u'x'), (0x1D59E, 'M', u'y'), @@ -6542,6 +6555,10 @@ def _seg_62(): (0x1D5F0, 'M', u'c'), (0x1D5F1, 'M', u'd'), (0x1D5F2, 'M', u'e'), + ] + +def _seg_63(): + return [ (0x1D5F3, 'M', u'f'), (0x1D5F4, 'M', u'g'), (0x1D5F5, 'M', u'h'), @@ -6555,10 +6572,6 @@ def _seg_62(): (0x1D5FD, 'M', u'p'), (0x1D5FE, 'M', u'q'), (0x1D5FF, 'M', u'r'), - ] - -def _seg_63(): - return [ (0x1D600, 'M', u's'), (0x1D601, 'M', u't'), (0x1D602, 'M', u'u'), @@ -6646,6 +6659,10 @@ def _seg_63(): (0x1D654, 'M', u'y'), (0x1D655, 'M', u'z'), (0x1D656, 'M', u'a'), + ] + +def _seg_64(): + return [ (0x1D657, 'M', u'b'), (0x1D658, 'M', u'c'), (0x1D659, 'M', u'd'), @@ -6659,10 +6676,6 @@ def _seg_63(): (0x1D661, 'M', u'l'), (0x1D662, 'M', u'm'), (0x1D663, 'M', u'n'), - ] - -def _seg_64(): - return [ (0x1D664, 'M', u'o'), (0x1D665, 'M', u'p'), (0x1D666, 'M', u'q'), @@ -6750,6 +6763,10 @@ def _seg_64(): (0x1D6B9, 'M', u'θ'), (0x1D6BA, 'M', u'σ'), (0x1D6BB, 'M', u'τ'), + ] + +def _seg_65(): + return [ (0x1D6BC, 'M', u'υ'), (0x1D6BD, 'M', u'φ'), (0x1D6BE, 'M', u'χ'), @@ -6763,10 +6780,6 @@ def _seg_64(): (0x1D6C6, 'M', u'ε'), (0x1D6C7, 'M', u'ζ'), (0x1D6C8, 'M', u'η'), - ] - -def _seg_65(): - return [ (0x1D6C9, 'M', u'θ'), (0x1D6CA, 'M', u'ι'), (0x1D6CB, 'M', u'κ'), @@ -6854,6 +6867,10 @@ def _seg_65(): (0x1D71F, 'M', u'δ'), (0x1D720, 'M', u'ε'), (0x1D721, 'M', u'ζ'), + ] + +def _seg_66(): + return [ (0x1D722, 'M', u'η'), (0x1D723, 'M', u'θ'), (0x1D724, 'M', u'ι'), @@ -6867,10 +6884,6 @@ def _seg_65(): (0x1D72C, 'M', u'ρ'), (0x1D72D, 'M', u'θ'), (0x1D72E, 'M', u'σ'), - ] - -def _seg_66(): - return [ (0x1D72F, 'M', u'τ'), (0x1D730, 'M', u'υ'), (0x1D731, 'M', u'φ'), @@ -6958,6 +6971,10 @@ def _seg_66(): (0x1D785, 'M', u'φ'), (0x1D786, 'M', u'χ'), (0x1D787, 'M', u'ψ'), + ] + +def _seg_67(): + return [ (0x1D788, 'M', u'ω'), (0x1D789, 'M', u'∂'), (0x1D78A, 'M', u'ε'), @@ -6971,10 +6988,6 @@ def _seg_66(): (0x1D792, 'M', u'γ'), (0x1D793, 'M', u'δ'), (0x1D794, 'M', u'ε'), - ] - -def _seg_67(): - return [ (0x1D795, 'M', u'ζ'), (0x1D796, 'M', u'η'), (0x1D797, 'M', u'θ'), @@ -7062,6 +7075,10 @@ def _seg_67(): (0x1D7EC, 'M', u'0'), (0x1D7ED, 'M', u'1'), (0x1D7EE, 'M', u'2'), + ] + +def _seg_68(): + return [ (0x1D7EF, 'M', u'3'), (0x1D7F0, 'M', u'4'), (0x1D7F1, 'M', u'5'), @@ -7075,10 +7092,6 @@ def _seg_67(): (0x1D7F9, 'M', u'3'), (0x1D7FA, 'M', u'4'), (0x1D7FB, 'M', u'5'), - ] - -def _seg_68(): - return [ (0x1D7FC, 'M', u'6'), (0x1D7FD, 'M', u'7'), (0x1D7FE, 'M', u'8'), @@ -7143,6 +7156,8 @@ def _seg_68(): (0x1E95A, 'X'), (0x1E95E, 'V'), (0x1E960, 'X'), + (0x1EC71, 'V'), + (0x1ECB5, 'X'), (0x1EE00, 'M', u'ا'), (0x1EE01, 'M', u'ب'), (0x1EE02, 'M', u'ج'), @@ -7164,6 +7179,10 @@ def _seg_68(): (0x1EE12, 'M', u'ق'), (0x1EE13, 'M', u'ر'), (0x1EE14, 'M', u'ش'), + ] + +def _seg_69(): + return [ (0x1EE15, 'M', u'ت'), (0x1EE16, 'M', u'ث'), (0x1EE17, 'M', u'خ'), @@ -7179,10 +7198,6 @@ def _seg_68(): (0x1EE21, 'M', u'ب'), (0x1EE22, 'M', u'ج'), (0x1EE23, 'X'), - ] - -def _seg_69(): - return [ (0x1EE24, 'M', u'ه'), (0x1EE25, 'X'), (0x1EE27, 'M', u'ح'), @@ -7268,6 +7283,10 @@ def _seg_69(): (0x1EE81, 'M', u'ب'), (0x1EE82, 'M', u'ج'), (0x1EE83, 'M', u'د'), + ] + +def _seg_70(): + return [ (0x1EE84, 'M', u'ه'), (0x1EE85, 'M', u'و'), (0x1EE86, 'M', u'ز'), @@ -7283,10 +7302,6 @@ def _seg_69(): (0x1EE90, 'M', u'ف'), (0x1EE91, 'M', u'ص'), (0x1EE92, 'M', u'ق'), - ] - -def _seg_70(): - return [ (0x1EE93, 'M', u'ر'), (0x1EE94, 'M', u'ش'), (0x1EE95, 'M', u'ت'), @@ -7372,6 +7387,10 @@ def _seg_70(): (0x1F122, '3', u'(s)'), (0x1F123, '3', u'(t)'), (0x1F124, '3', u'(u)'), + ] + +def _seg_71(): + return [ (0x1F125, '3', u'(v)'), (0x1F126, '3', u'(w)'), (0x1F127, '3', u'(x)'), @@ -7382,15 +7401,11 @@ def _seg_70(): (0x1F12C, 'M', u'r'), (0x1F12D, 'M', u'cd'), (0x1F12E, 'M', u'wz'), - (0x1F12F, 'X'), + (0x1F12F, 'V'), (0x1F130, 'M', u'a'), (0x1F131, 'M', u'b'), (0x1F132, 'M', u'c'), (0x1F133, 'M', u'd'), - ] - -def _seg_71(): - return [ (0x1F134, 'M', u'e'), (0x1F135, 'M', u'f'), (0x1F136, 'M', u'g'), @@ -7476,6 +7491,10 @@ def _seg_71(): (0x1F239, 'M', u'割'), (0x1F23A, 'M', u'営'), (0x1F23B, 'M', u'配'), + ] + +def _seg_72(): + return [ (0x1F23C, 'X'), (0x1F240, 'M', u'〔本〕'), (0x1F241, 'M', u'〔三〕'), @@ -7491,21 +7510,17 @@ def _seg_71(): (0x1F251, 'M', u'可'), (0x1F252, 'X'), (0x1F260, 'V'), - ] - -def _seg_72(): - return [ (0x1F266, 'X'), (0x1F300, 'V'), (0x1F6D5, 'X'), (0x1F6E0, 'V'), (0x1F6ED, 'X'), (0x1F6F0, 'V'), - (0x1F6F9, 'X'), + (0x1F6FA, 'X'), (0x1F700, 'V'), (0x1F774, 'X'), (0x1F780, 'V'), - (0x1F7D5, 'X'), + (0x1F7D9, 'X'), (0x1F800, 'V'), (0x1F80C, 'X'), (0x1F810, 'V'), @@ -7521,15 +7536,21 @@ def _seg_72(): (0x1F910, 'V'), (0x1F93F, 'X'), (0x1F940, 'V'), - (0x1F94D, 'X'), - (0x1F950, 'V'), - (0x1F96C, 'X'), - (0x1F980, 'V'), - (0x1F998, 'X'), + (0x1F971, 'X'), + (0x1F973, 'V'), + (0x1F977, 'X'), + (0x1F97A, 'V'), + (0x1F97B, 'X'), + (0x1F97C, 'V'), + (0x1F9A3, 'X'), + (0x1F9B0, 'V'), + (0x1F9BA, 'X'), (0x1F9C0, 'V'), - (0x1F9C1, 'X'), + (0x1F9C3, 'X'), (0x1F9D0, 'V'), - (0x1F9E7, 'X'), + (0x1FA00, 'X'), + (0x1FA60, 'V'), + (0x1FA6E, 'X'), (0x20000, 'V'), (0x2A6D7, 'X'), (0x2A700, 'V'), @@ -7574,6 +7595,10 @@ def _seg_72(): (0x2F81F, 'M', u'㓟'), (0x2F820, 'M', u'刻'), (0x2F821, 'M', u'剆'), + ] + +def _seg_73(): + return [ (0x2F822, 'M', u'割'), (0x2F823, 'M', u'剷'), (0x2F824, 'M', u'㔕'), @@ -7595,10 +7620,6 @@ def _seg_72(): (0x2F836, 'M', u'及'), (0x2F837, 'M', u'叟'), (0x2F838, 'M', u'𠭣'), - ] - -def _seg_73(): - return [ (0x2F839, 'M', u'叫'), (0x2F83A, 'M', u'叱'), (0x2F83B, 'M', u'吆'), @@ -7678,6 +7699,10 @@ def _seg_73(): (0x2F887, 'M', u'幩'), (0x2F888, 'M', u'㡢'), (0x2F889, 'M', u'𢆃'), + ] + +def _seg_74(): + return [ (0x2F88A, 'M', u'㡼'), (0x2F88B, 'M', u'庰'), (0x2F88C, 'M', u'庳'), @@ -7699,10 +7724,6 @@ def _seg_73(): (0x2F89E, 'M', u'志'), (0x2F89F, 'M', u'忹'), (0x2F8A0, 'M', u'悁'), - ] - -def _seg_74(): - return [ (0x2F8A1, 'M', u'㤺'), (0x2F8A2, 'M', u'㤜'), (0x2F8A3, 'M', u'悔'), @@ -7782,6 +7803,10 @@ def _seg_74(): (0x2F8ED, 'M', u'櫛'), (0x2F8EE, 'M', u'㰘'), (0x2F8EF, 'M', u'次'), + ] + +def _seg_75(): + return [ (0x2F8F0, 'M', u'𣢧'), (0x2F8F1, 'M', u'歔'), (0x2F8F2, 'M', u'㱎'), @@ -7803,10 +7828,6 @@ def _seg_74(): (0x2F902, 'M', u'流'), (0x2F903, 'M', u'浩'), (0x2F904, 'M', u'浸'), - ] - -def _seg_75(): - return [ (0x2F905, 'M', u'涅'), (0x2F906, 'M', u'𣴞'), (0x2F907, 'M', u'洴'), @@ -7886,6 +7907,10 @@ def _seg_75(): (0x2F953, 'M', u'祖'), (0x2F954, 'M', u'𥚚'), (0x2F955, 'M', u'𥛅'), + ] + +def _seg_76(): + return [ (0x2F956, 'M', u'福'), (0x2F957, 'M', u'秫'), (0x2F958, 'M', u'䄯'), @@ -7907,10 +7932,6 @@ def _seg_75(): (0x2F969, 'M', u'糣'), (0x2F96A, 'M', u'紀'), (0x2F96B, 'M', u'𥾆'), - ] - -def _seg_76(): - return [ (0x2F96C, 'M', u'絣'), (0x2F96D, 'M', u'䌁'), (0x2F96E, 'M', u'緇'), @@ -7990,6 +8011,10 @@ def _seg_76(): (0x2F9B8, 'M', u'蚈'), (0x2F9B9, 'M', u'蜎'), (0x2F9BA, 'M', u'蛢'), + ] + +def _seg_77(): + return [ (0x2F9BB, 'M', u'蝹'), (0x2F9BC, 'M', u'蜨'), (0x2F9BD, 'M', u'蝫'), @@ -8011,10 +8036,6 @@ def _seg_76(): (0x2F9CD, 'M', u'䚾'), (0x2F9CE, 'M', u'䛇'), (0x2F9CF, 'M', u'誠'), - ] - -def _seg_77(): - return [ (0x2F9D0, 'M', u'諭'), (0x2F9D1, 'M', u'變'), (0x2F9D2, 'M', u'豕'), @@ -8094,6 +8115,10 @@ def _seg_77(): (0x2FA1D, 'M', u'𪘀'), (0x2FA1E, 'X'), (0xE0100, 'I'), + ] + +def _seg_78(): + return [ (0xE01F0, 'X'), ] @@ -8176,4 +8201,5 @@ uts46data = tuple( + _seg_75() + _seg_76() + _seg_77() + + _seg_78() ) diff --git a/pipenv/patched/notpip/_vendor/packaging/__about__.py b/pipenv/patched/notpip/_vendor/packaging/__about__.py index 21fc6ce3..7481c9e2 100644 --- a/pipenv/patched/notpip/_vendor/packaging/__about__.py +++ b/pipenv/patched/notpip/_vendor/packaging/__about__.py @@ -4,18 +4,24 @@ from __future__ import absolute_import, division, print_function __all__ = [ - "__title__", "__summary__", "__uri__", "__version__", "__author__", - "__email__", "__license__", "__copyright__", + "__title__", + "__summary__", + "__uri__", + "__version__", + "__author__", + "__email__", + "__license__", + "__copyright__", ] __title__ = "packaging" __summary__ = "Core utilities for Python packages" __uri__ = "https://github.com/pypa/packaging" -__version__ = "18.0" +__version__ = "19.0" __author__ = "Donald Stufft and individual contributors" __email__ = "donald@stufft.io" __license__ = "BSD or Apache License, Version 2.0" -__copyright__ = "Copyright 2014-2018 %s" % __author__ +__copyright__ = "Copyright 2014-2019 %s" % __author__ diff --git a/pipenv/patched/notpip/_vendor/packaging/__init__.py b/pipenv/patched/notpip/_vendor/packaging/__init__.py index 5ee62202..a0cf67df 100644 --- a/pipenv/patched/notpip/_vendor/packaging/__init__.py +++ b/pipenv/patched/notpip/_vendor/packaging/__init__.py @@ -4,11 +4,23 @@ from __future__ import absolute_import, division, print_function from .__about__ import ( - __author__, __copyright__, __email__, __license__, __summary__, __title__, - __uri__, __version__ + __author__, + __copyright__, + __email__, + __license__, + __summary__, + __title__, + __uri__, + __version__, ) __all__ = [ - "__title__", "__summary__", "__uri__", "__version__", "__author__", - "__email__", "__license__", "__copyright__", + "__title__", + "__summary__", + "__uri__", + "__version__", + "__author__", + "__email__", + "__license__", + "__copyright__", ] diff --git a/pipenv/patched/notpip/_vendor/packaging/_compat.py b/pipenv/patched/notpip/_vendor/packaging/_compat.py index 210bb80b..25da473c 100644 --- a/pipenv/patched/notpip/_vendor/packaging/_compat.py +++ b/pipenv/patched/notpip/_vendor/packaging/_compat.py @@ -12,9 +12,9 @@ PY3 = sys.version_info[0] == 3 # flake8: noqa if PY3: - string_types = str, + string_types = (str,) else: - string_types = basestring, + string_types = (basestring,) def with_metaclass(meta, *bases): @@ -27,4 +27,5 @@ def with_metaclass(meta, *bases): class metaclass(meta): def __new__(cls, name, this_bases, d): return meta(name, bases, d) - return type.__new__(metaclass, 'temporary_class', (), {}) + + return type.__new__(metaclass, "temporary_class", (), {}) diff --git a/pipenv/patched/notpip/_vendor/packaging/_structures.py b/pipenv/patched/notpip/_vendor/packaging/_structures.py index e9fc4a04..68dcca63 100644 --- a/pipenv/patched/notpip/_vendor/packaging/_structures.py +++ b/pipenv/patched/notpip/_vendor/packaging/_structures.py @@ -5,7 +5,6 @@ from __future__ import absolute_import, division, print_function class Infinity(object): - def __repr__(self): return "Infinity" @@ -38,7 +37,6 @@ Infinity = Infinity() class NegativeInfinity(object): - def __repr__(self): return "-Infinity" diff --git a/pipenv/patched/notpip/_vendor/packaging/markers.py b/pipenv/patched/notpip/_vendor/packaging/markers.py index dc3cef81..50a08091 100644 --- a/pipenv/patched/notpip/_vendor/packaging/markers.py +++ b/pipenv/patched/notpip/_vendor/packaging/markers.py @@ -17,8 +17,11 @@ from .specifiers import Specifier, InvalidSpecifier __all__ = [ - "InvalidMarker", "UndefinedComparison", "UndefinedEnvironmentName", - "Marker", "default_environment", + "InvalidMarker", + "UndefinedComparison", + "UndefinedEnvironmentName", + "Marker", + "default_environment", ] @@ -42,7 +45,6 @@ class UndefinedEnvironmentName(ValueError): class Node(object): - def __init__(self, value): self.value = value @@ -57,62 +59,52 @@ class Node(object): class Variable(Node): - def serialize(self): return str(self) class Value(Node): - def serialize(self): return '"{0}"'.format(self) class Op(Node): - def serialize(self): return str(self) VARIABLE = ( - L("implementation_version") | - L("platform_python_implementation") | - L("implementation_name") | - L("python_full_version") | - L("platform_release") | - L("platform_version") | - L("platform_machine") | - L("platform_system") | - L("python_version") | - L("sys_platform") | - L("os_name") | - L("os.name") | # PEP-345 - L("sys.platform") | # PEP-345 - L("platform.version") | # PEP-345 - L("platform.machine") | # PEP-345 - L("platform.python_implementation") | # PEP-345 - L("python_implementation") | # undocumented setuptools legacy - L("extra") + L("implementation_version") + | L("platform_python_implementation") + | L("implementation_name") + | L("python_full_version") + | L("platform_release") + | L("platform_version") + | L("platform_machine") + | L("platform_system") + | L("python_version") + | L("sys_platform") + | L("os_name") + | L("os.name") + | L("sys.platform") # PEP-345 + | L("platform.version") # PEP-345 + | L("platform.machine") # PEP-345 + | L("platform.python_implementation") # PEP-345 + | L("python_implementation") # PEP-345 + | L("extra") # undocumented setuptools legacy ) ALIASES = { - 'os.name': 'os_name', - 'sys.platform': 'sys_platform', - 'platform.version': 'platform_version', - 'platform.machine': 'platform_machine', - 'platform.python_implementation': 'platform_python_implementation', - 'python_implementation': 'platform_python_implementation' + "os.name": "os_name", + "sys.platform": "sys_platform", + "platform.version": "platform_version", + "platform.machine": "platform_machine", + "platform.python_implementation": "platform_python_implementation", + "python_implementation": "platform_python_implementation", } VARIABLE.setParseAction(lambda s, l, t: Variable(ALIASES.get(t[0], t[0]))) VERSION_CMP = ( - L("===") | - L("==") | - L(">=") | - L("<=") | - L("!=") | - L("~=") | - L(">") | - L("<") + L("===") | L("==") | L(">=") | L("<=") | L("!=") | L("~=") | L(">") | L("<") ) MARKER_OP = VERSION_CMP | L("not in") | L("in") @@ -152,8 +144,11 @@ def _format_marker(marker, first=True): # where the single item is itself it's own list. In that case we want skip # the rest of this function so that we don't get extraneous () on the # outside. - if (isinstance(marker, list) and len(marker) == 1 and - isinstance(marker[0], (list, tuple))): + if ( + isinstance(marker, list) + and len(marker) == 1 + and isinstance(marker[0], (list, tuple)) + ): return _format_marker(marker[0]) if isinstance(marker, list): @@ -239,20 +234,20 @@ def _evaluate_markers(markers, environment): def format_full_version(info): - version = '{0.major}.{0.minor}.{0.micro}'.format(info) + version = "{0.major}.{0.minor}.{0.micro}".format(info) kind = info.releaselevel - if kind != 'final': + if kind != "final": version += kind[0] + str(info.serial) return version def default_environment(): - if hasattr(sys, 'implementation'): + if hasattr(sys, "implementation"): iver = format_full_version(sys.implementation.version) implementation_name = sys.implementation.name else: - iver = '0' - implementation_name = '' + iver = "0" + implementation_name = "" return { "implementation_name": implementation_name, @@ -270,13 +265,13 @@ def default_environment(): class Marker(object): - def __init__(self, marker): try: self._markers = _coerce_parse_result(MARKER.parseString(marker)) except ParseException as e: err_str = "Invalid marker: {0!r}, parse error at {1!r}".format( - marker, marker[e.loc:e.loc + 8]) + marker, marker[e.loc : e.loc + 8] + ) raise InvalidMarker(err_str) def __str__(self): diff --git a/pipenv/patched/notpip/_vendor/packaging/requirements.py b/pipenv/patched/notpip/_vendor/packaging/requirements.py index 5ec5d74a..a3de7673 100644 --- a/pipenv/patched/notpip/_vendor/packaging/requirements.py +++ b/pipenv/patched/notpip/_vendor/packaging/requirements.py @@ -38,8 +38,8 @@ IDENTIFIER = Combine(ALPHANUM + ZeroOrMore(IDENTIFIER_END)) NAME = IDENTIFIER("name") EXTRA = IDENTIFIER -URI = Regex(r'[^ ]+')("url") -URL = (AT + URI) +URI = Regex(r"[^ ]+")("url") +URL = AT + URI EXTRAS_LIST = EXTRA + ZeroOrMore(COMMA + EXTRA) EXTRAS = (LBRACKET + Optional(EXTRAS_LIST) + RBRACKET)("extras") @@ -48,17 +48,18 @@ VERSION_PEP440 = Regex(Specifier._regex_str, re.VERBOSE | re.IGNORECASE) VERSION_LEGACY = Regex(LegacySpecifier._regex_str, re.VERBOSE | re.IGNORECASE) VERSION_ONE = VERSION_PEP440 ^ VERSION_LEGACY -VERSION_MANY = Combine(VERSION_ONE + ZeroOrMore(COMMA + VERSION_ONE), - joinString=",", adjacent=False)("_raw_spec") +VERSION_MANY = Combine( + VERSION_ONE + ZeroOrMore(COMMA + VERSION_ONE), joinString=",", adjacent=False +)("_raw_spec") _VERSION_SPEC = Optional(((LPAREN + VERSION_MANY + RPAREN) | VERSION_MANY)) -_VERSION_SPEC.setParseAction(lambda s, l, t: t._raw_spec or '') +_VERSION_SPEC.setParseAction(lambda s, l, t: t._raw_spec or "") VERSION_SPEC = originalTextFor(_VERSION_SPEC)("specifier") VERSION_SPEC.setParseAction(lambda s, l, t: t[1]) MARKER_EXPR = originalTextFor(MARKER_EXPR())("marker") MARKER_EXPR.setParseAction( - lambda s, l, t: Marker(s[t._original_start:t._original_end]) + lambda s, l, t: Marker(s[t._original_start : t._original_end]) ) MARKER_SEPARATOR = SEMICOLON MARKER = MARKER_SEPARATOR + MARKER_EXPR @@ -66,8 +67,7 @@ MARKER = MARKER_SEPARATOR + MARKER_EXPR VERSION_AND_MARKER = VERSION_SPEC + Optional(MARKER) URL_AND_MARKER = URL + Optional(MARKER) -NAMED_REQUIREMENT = \ - NAME + Optional(EXTRAS) + (URL_AND_MARKER | VERSION_AND_MARKER) +NAMED_REQUIREMENT = NAME + Optional(EXTRAS) + (URL_AND_MARKER | VERSION_AND_MARKER) REQUIREMENT = stringStart + NAMED_REQUIREMENT + stringEnd # pyparsing isn't thread safe during initialization, so we do it eagerly, see @@ -92,15 +92,21 @@ class Requirement(object): try: req = REQUIREMENT.parseString(requirement_string) except ParseException as e: - raise InvalidRequirement("Parse error at \"{0!r}\": {1}".format( - requirement_string[e.loc:e.loc + 8], e.msg - )) + raise InvalidRequirement( + 'Parse error at "{0!r}": {1}'.format( + requirement_string[e.loc : e.loc + 8], e.msg + ) + ) self.name = req.name if req.url: parsed_url = urlparse.urlparse(req.url) - if not (parsed_url.scheme and parsed_url.netloc) or ( - not parsed_url.scheme and not parsed_url.netloc): + if parsed_url.scheme == "file": + if urlparse.urlunparse(parsed_url) != req.url: + raise InvalidRequirement("Invalid URL given") + elif not (parsed_url.scheme and parsed_url.netloc) or ( + not parsed_url.scheme and not parsed_url.netloc + ): raise InvalidRequirement("Invalid URL: {0}".format(req.url)) self.url = req.url else: @@ -120,6 +126,8 @@ class Requirement(object): if self.url: parts.append("@ {0}".format(self.url)) + if self.marker: + parts.append(" ") if self.marker: parts.append("; {0}".format(self.marker)) diff --git a/pipenv/patched/notpip/_vendor/packaging/specifiers.py b/pipenv/patched/notpip/_vendor/packaging/specifiers.py index 4c798999..743576a0 100644 --- a/pipenv/patched/notpip/_vendor/packaging/specifiers.py +++ b/pipenv/patched/notpip/_vendor/packaging/specifiers.py @@ -19,7 +19,6 @@ class InvalidSpecifier(ValueError): class BaseSpecifier(with_metaclass(abc.ABCMeta, object)): - @abc.abstractmethod def __str__(self): """ @@ -84,10 +83,7 @@ class _IndividualSpecifier(BaseSpecifier): if not match: raise InvalidSpecifier("Invalid specifier: '{0}'".format(spec)) - self._spec = ( - match.group("operator").strip(), - match.group("version").strip(), - ) + self._spec = (match.group("operator").strip(), match.group("version").strip()) # Store whether or not this Specifier should accept prereleases self._prereleases = prereleases @@ -99,11 +95,7 @@ class _IndividualSpecifier(BaseSpecifier): else "" ) - return "<{0}({1!r}{2})>".format( - self.__class__.__name__, - str(self), - pre, - ) + return "<{0}({1!r}{2})>".format(self.__class__.__name__, str(self), pre) def __str__(self): return "{0}{1}".format(*self._spec) @@ -194,8 +186,9 @@ class _IndividualSpecifier(BaseSpecifier): # If our version is a prerelease, and we were not set to allow # prereleases, then we'll store it for later incase nothing # else matches this specifier. - if (parsed_version.is_prerelease and not - (prereleases or self.prereleases)): + if parsed_version.is_prerelease and not ( + prereleases or self.prereleases + ): found_prereleases.append(version) # Either this is not a prerelease, or we should have been # accepting prereleases from the beginning. @@ -213,8 +206,7 @@ class _IndividualSpecifier(BaseSpecifier): class LegacySpecifier(_IndividualSpecifier): - _regex_str = ( - r""" + _regex_str = r""" (?P(==|!=|<=|>=|<|>)) \s* (?P @@ -225,10 +217,8 @@ class LegacySpecifier(_IndividualSpecifier): # them, and a comma since it's a version separator. ) """ - ) - _regex = re.compile( - r"^\s*" + _regex_str + r"\s*$", re.VERBOSE | re.IGNORECASE) + _regex = re.compile(r"^\s*" + _regex_str + r"\s*$", re.VERBOSE | re.IGNORECASE) _operators = { "==": "equal", @@ -269,13 +259,13 @@ def _require_version_compare(fn): if not isinstance(prospective, Version): return False return fn(self, prospective, spec) + return wrapped class Specifier(_IndividualSpecifier): - _regex_str = ( - r""" + _regex_str = r""" (?P(~=|==|!=|<=|>=|<|>|===)) (?P (?: @@ -367,10 +357,8 @@ class Specifier(_IndividualSpecifier): ) ) """ - ) - _regex = re.compile( - r"^\s*" + _regex_str + r"\s*$", re.VERBOSE | re.IGNORECASE) + _regex = re.compile(r"^\s*" + _regex_str + r"\s*$", re.VERBOSE | re.IGNORECASE) _operators = { "~=": "compatible", @@ -397,8 +385,7 @@ class Specifier(_IndividualSpecifier): prefix = ".".join( list( itertools.takewhile( - lambda x: (not x.startswith("post") and not - x.startswith("dev")), + lambda x: (not x.startswith("post") and not x.startswith("dev")), _version_split(spec), ) )[:-1] @@ -407,8 +394,9 @@ class Specifier(_IndividualSpecifier): # Add the prefix notation to the end of our string prefix += ".*" - return (self._get_operator(">=")(prospective, spec) and - self._get_operator("==")(prospective, prefix)) + return self._get_operator(">=")(prospective, spec) and self._get_operator("==")( + prospective, prefix + ) @_require_version_compare def _compare_equal(self, prospective, spec): @@ -428,7 +416,7 @@ class Specifier(_IndividualSpecifier): # Shorten the prospective version to be the same length as the spec # so that we can determine if the specifier is a prefix of the # prospective version or not. - prospective = prospective[:len(spec)] + prospective = prospective[: len(spec)] # Pad out our two sides with zeros so that they both equal the same # length. @@ -567,27 +555,17 @@ def _pad_version(left, right): right_split.append(list(itertools.takewhile(lambda x: x.isdigit(), right))) # Get the rest of our versions - left_split.append(left[len(left_split[0]):]) - right_split.append(right[len(right_split[0]):]) + left_split.append(left[len(left_split[0]) :]) + right_split.append(right[len(right_split[0]) :]) # Insert our padding - left_split.insert( - 1, - ["0"] * max(0, len(right_split[0]) - len(left_split[0])), - ) - right_split.insert( - 1, - ["0"] * max(0, len(left_split[0]) - len(right_split[0])), - ) + left_split.insert(1, ["0"] * max(0, len(right_split[0]) - len(left_split[0]))) + right_split.insert(1, ["0"] * max(0, len(left_split[0]) - len(right_split[0]))) - return ( - list(itertools.chain(*left_split)), - list(itertools.chain(*right_split)), - ) + return (list(itertools.chain(*left_split)), list(itertools.chain(*right_split))) class SpecifierSet(BaseSpecifier): - def __init__(self, specifiers="", prereleases=None): # Split on , to break each indidivual specifier into it's own item, and # strip each item to remove leading/trailing whitespace. @@ -721,10 +699,7 @@ class SpecifierSet(BaseSpecifier): # given version is contained within all of them. # Note: This use of all() here means that an empty set of specifiers # will always return True, this is an explicit design decision. - return all( - s.contains(item, prereleases=prereleases) - for s in self._specs - ) + return all(s.contains(item, prereleases=prereleases) for s in self._specs) def filter(self, iterable, prereleases=None): # Determine if we're forcing a prerelease or not, if we're not forcing diff --git a/pipenv/patched/notpip/_vendor/packaging/utils.py b/pipenv/patched/notpip/_vendor/packaging/utils.py index 4b94a82f..88418786 100644 --- a/pipenv/patched/notpip/_vendor/packaging/utils.py +++ b/pipenv/patched/notpip/_vendor/packaging/utils.py @@ -36,13 +36,7 @@ def canonicalize_version(version): # Release segment # NB: This strips trailing '.0's to normalize - parts.append( - re.sub( - r'(\.0)+$', - '', - ".".join(str(x) for x in version.release) - ) - ) + parts.append(re.sub(r"(\.0)+$", "", ".".join(str(x) for x in version.release))) # Pre-release if version.pre is not None: diff --git a/pipenv/patched/notpip/_vendor/packaging/version.py b/pipenv/patched/notpip/_vendor/packaging/version.py index 6ed5cbbd..95157a1f 100644 --- a/pipenv/patched/notpip/_vendor/packaging/version.py +++ b/pipenv/patched/notpip/_vendor/packaging/version.py @@ -10,14 +10,11 @@ import re from ._structures import Infinity -__all__ = [ - "parse", "Version", "LegacyVersion", "InvalidVersion", "VERSION_PATTERN" -] +__all__ = ["parse", "Version", "LegacyVersion", "InvalidVersion", "VERSION_PATTERN"] _Version = collections.namedtuple( - "_Version", - ["epoch", "release", "dev", "pre", "post", "local"], + "_Version", ["epoch", "release", "dev", "pre", "post", "local"] ) @@ -40,7 +37,6 @@ class InvalidVersion(ValueError): class _BaseVersion(object): - def __hash__(self): return hash(self._key) @@ -70,7 +66,6 @@ class _BaseVersion(object): class LegacyVersion(_BaseVersion): - def __init__(self, version): self._version = str(version) self._key = _legacy_cmpkey(self._version) @@ -126,12 +121,14 @@ class LegacyVersion(_BaseVersion): return False -_legacy_version_component_re = re.compile( - r"(\d+ | [a-z]+ | \.| -)", re.VERBOSE, -) +_legacy_version_component_re = re.compile(r"(\d+ | [a-z]+ | \.| -)", re.VERBOSE) _legacy_version_replacement_map = { - "pre": "c", "preview": "c", "-": "final-", "rc": "c", "dev": "@", + "pre": "c", + "preview": "c", + "-": "final-", + "rc": "c", + "dev": "@", } @@ -215,10 +212,7 @@ VERSION_PATTERN = r""" class Version(_BaseVersion): - _regex = re.compile( - r"^\s*" + VERSION_PATTERN + r"\s*$", - re.VERBOSE | re.IGNORECASE, - ) + _regex = re.compile(r"^\s*" + VERSION_PATTERN + r"\s*$", re.VERBOSE | re.IGNORECASE) def __init__(self, version): # Validate the version and parse it into pieces @@ -230,18 +224,11 @@ class Version(_BaseVersion): self._version = _Version( epoch=int(match.group("epoch")) if match.group("epoch") else 0, release=tuple(int(i) for i in match.group("release").split(".")), - pre=_parse_letter_version( - match.group("pre_l"), - match.group("pre_n"), - ), + pre=_parse_letter_version(match.group("pre_l"), match.group("pre_n")), post=_parse_letter_version( - match.group("post_l"), - match.group("post_n1") or match.group("post_n2"), - ), - dev=_parse_letter_version( - match.group("dev_l"), - match.group("dev_n"), + match.group("post_l"), match.group("post_n1") or match.group("post_n2") ), + dev=_parse_letter_version(match.group("dev_l"), match.group("dev_n")), local=_parse_local_version(match.group("local")), ) @@ -395,12 +382,7 @@ def _cmpkey(epoch, release, pre, post, dev, local): # re-reverse it back into the correct order and make it a tuple and use # that for our sorting key. release = tuple( - reversed(list( - itertools.dropwhile( - lambda x: x == 0, - reversed(release), - ) - )) + reversed(list(itertools.dropwhile(lambda x: x == 0, reversed(release)))) ) # We need to "trick" the sorting algorithm to put 1.0.dev0 before 1.0a0. @@ -433,9 +415,6 @@ def _cmpkey(epoch, release, pre, post, dev, local): # - Numeric segments sort numerically # - Shorter versions sort before longer versions when the prefixes # match exactly - local = tuple( - (i, "") if isinstance(i, int) else (-Infinity, i) - for i in local - ) + local = tuple((i, "") if isinstance(i, int) else (-Infinity, i) for i in local) return epoch, release, pre, post, dev, local diff --git a/pipenv/patched/notpip/_vendor/pep517/__init__.py b/pipenv/patched/notpip/_vendor/pep517/__init__.py index 8beedea4..9c1a098f 100644 --- a/pipenv/patched/notpip/_vendor/pep517/__init__.py +++ b/pipenv/patched/notpip/_vendor/pep517/__init__.py @@ -1,4 +1,4 @@ """Wrappers to build Python packages using PEP 517 hooks """ -__version__ = '0.2' +__version__ = '0.5.0' diff --git a/pipenv/patched/notpip/_vendor/pep517/_in_process.py b/pipenv/patched/notpip/_vendor/pep517/_in_process.py index baa14d38..d6524b66 100644 --- a/pipenv/patched/notpip/_vendor/pep517/_in_process.py +++ b/pipenv/patched/notpip/_vendor/pep517/_in_process.py @@ -21,19 +21,28 @@ import sys # This is run as a script, not a module, so it can't do a relative import import compat + +class BackendUnavailable(Exception): + """Raised if we cannot import the backend""" + + def _build_backend(): """Find and load the build backend""" ep = os.environ['PEP517_BUILD_BACKEND'] mod_path, _, obj_path = ep.partition(':') - obj = import_module(mod_path) + try: + obj = import_module(mod_path) + except ImportError: + raise BackendUnavailable if obj_path: for path_part in obj_path.split('.'): obj = getattr(obj, path_part) return obj + def get_requires_for_build_wheel(config_settings): """Invoke the optional get_requires_for_build_wheel hook - + Returns [] if the hook is not defined. """ backend = _build_backend() @@ -44,9 +53,10 @@ def get_requires_for_build_wheel(config_settings): else: return hook(config_settings) + def prepare_metadata_for_build_wheel(metadata_directory, config_settings): """Invoke optional prepare_metadata_for_build_wheel - + Implements a fallback by building a wheel if the hook isn't defined. """ backend = _build_backend() @@ -58,8 +68,10 @@ def prepare_metadata_for_build_wheel(metadata_directory, config_settings): else: return hook(metadata_directory, config_settings) + WHEEL_BUILT_MARKER = 'PEP517_ALREADY_BUILT_WHEEL' + def _dist_info_files(whl_zip): """Identify the .dist-info folder inside a wheel ZipFile.""" res = [] @@ -71,11 +83,13 @@ def _dist_info_files(whl_zip): return res raise Exception("No .dist-info folder found in wheel") -def _get_wheel_metadata_from_wheel(backend, metadata_directory, config_settings): + +def _get_wheel_metadata_from_wheel( + backend, metadata_directory, config_settings): """Build a wheel and extract the metadata from it. - - Fallback for when the build backend does not define the 'get_wheel_metadata' - hook. + + Fallback for when the build backend does not + define the 'get_wheel_metadata' hook. """ from zipfile import ZipFile whl_basename = backend.build_wheel(metadata_directory, config_settings) @@ -88,6 +102,7 @@ def _get_wheel_metadata_from_wheel(backend, metadata_directory, config_settings) zipf.extractall(path=metadata_directory, members=dist_info) return dist_info[0].split('/')[0] + def _find_already_built_wheel(metadata_directory): """Check for a wheel already built during the get_wheel_metadata hook. """ @@ -105,14 +120,16 @@ def _find_already_built_wheel(metadata_directory): print('Found multiple .whl files; unspecified behaviour. ' 'Will call build_wheel.') return None - + # Exactly one .whl file return whl_files[0] + def build_wheel(wheel_directory, config_settings, metadata_directory=None): """Invoke the mandatory build_wheel hook. - - If a wheel was already built in the prepare_metadata_for_build_wheel fallback, this + + If a wheel was already built in the + prepare_metadata_for_build_wheel fallback, this will copy it rather than rebuilding the wheel. """ prebuilt_whl = _find_already_built_wheel(metadata_directory) @@ -137,12 +154,15 @@ def get_requires_for_build_sdist(config_settings): else: return hook(config_settings) + class _DummyException(Exception): """Nothing should ever raise this exception""" + class GotUnsupportedOperation(Exception): """For internal use when backend raises UnsupportedOperation""" + def build_sdist(sdist_directory, config_settings): """Invoke the mandatory build_sdist hook.""" backend = _build_backend() @@ -151,6 +171,7 @@ def build_sdist(sdist_directory, config_settings): except getattr(backend, 'UnsupportedOperation', _DummyException): raise GotUnsupportedOperation + HOOK_NAMES = { 'get_requires_for_build_wheel', 'prepare_metadata_for_build_wheel', @@ -159,6 +180,7 @@ HOOK_NAMES = { 'build_sdist', } + def main(): if len(sys.argv) < 3: sys.exit("Needs args: hook_name, control_dir") @@ -173,10 +195,13 @@ def main(): json_out = {'unsupported': False, 'return_val': None} try: json_out['return_val'] = hook(**hook_input['kwargs']) + except BackendUnavailable: + json_out['no_backend'] = True except GotUnsupportedOperation: json_out['unsupported'] = True - + compat.write_json(json_out, pjoin(control_dir, 'output.json'), indent=2) + if __name__ == '__main__': main() diff --git a/pipenv/patched/notpip/_vendor/pep517/build.py b/pipenv/patched/notpip/_vendor/pep517/build.py new file mode 100644 index 00000000..db9a0799 --- /dev/null +++ b/pipenv/patched/notpip/_vendor/pep517/build.py @@ -0,0 +1,108 @@ +"""Build a project using PEP 517 hooks. +""" +import argparse +import logging +import os +import contextlib +from pipenv.patched.notpip._vendor import pytoml +import shutil +import errno +import tempfile + +from .envbuild import BuildEnvironment +from .wrappers import Pep517HookCaller + +log = logging.getLogger(__name__) + + +@contextlib.contextmanager +def tempdir(): + td = tempfile.mkdtemp() + try: + yield td + finally: + shutil.rmtree(td) + + +def _do_build(hooks, env, dist, dest): + get_requires_name = 'get_requires_for_build_{dist}'.format(**locals()) + get_requires = getattr(hooks, get_requires_name) + reqs = get_requires({}) + log.info('Got build requires: %s', reqs) + + env.pip_install(reqs) + log.info('Installed dynamic build dependencies') + + with tempdir() as td: + log.info('Trying to build %s in %s', dist, td) + build_name = 'build_{dist}'.format(**locals()) + build = getattr(hooks, build_name) + filename = build(td, {}) + source = os.path.join(td, filename) + shutil.move(source, os.path.join(dest, os.path.basename(filename))) + + +def mkdir_p(*args, **kwargs): + """Like `mkdir`, but does not raise an exception if the + directory already exists. + """ + try: + return os.mkdir(*args, **kwargs) + except OSError as exc: + if exc.errno != errno.EEXIST: + raise + + +def build(source_dir, dist, dest=None): + pyproject = os.path.join(source_dir, 'pyproject.toml') + dest = os.path.join(source_dir, dest or 'dist') + mkdir_p(dest) + + with open(pyproject) as f: + pyproject_data = pytoml.load(f) + # Ensure the mandatory data can be loaded + buildsys = pyproject_data['build-system'] + requires = buildsys['requires'] + backend = buildsys['build-backend'] + + hooks = Pep517HookCaller(source_dir, backend) + + with BuildEnvironment() as env: + env.pip_install(requires) + _do_build(hooks, env, dist, dest) + + +parser = argparse.ArgumentParser() +parser.add_argument( + 'source_dir', + help="A directory containing pyproject.toml", +) +parser.add_argument( + '--binary', '-b', + action='store_true', + default=False, +) +parser.add_argument( + '--source', '-s', + action='store_true', + default=False, +) +parser.add_argument( + '--out-dir', '-o', + help="Destination in which to save the builds relative to source dir", +) + + +def main(args): + # determine which dists to build + dists = list(filter(None, ( + 'sdist' if args.source or not args.binary else None, + 'wheel' if args.binary or not args.source else None, + ))) + + for dist in dists: + build(args.source_dir, dist, args.out_dir) + + +if __name__ == '__main__': + main(parser.parse_args()) diff --git a/pipenv/patched/notpip/_vendor/pep517/check.py b/pipenv/patched/notpip/_vendor/pep517/check.py index 3dffd2e0..9d28ba44 100644 --- a/pipenv/patched/notpip/_vendor/pep517/check.py +++ b/pipenv/patched/notpip/_vendor/pep517/check.py @@ -18,10 +18,11 @@ from .wrappers import Pep517HookCaller log = logging.getLogger(__name__) -def check_build_sdist(hooks): + +def check_build_sdist(hooks, build_sys_requires): with BuildEnvironment() as env: try: - env.pip_install(hooks.build_sys_requires) + env.pip_install(build_sys_requires) log.info('Installed static build dependencies') except CalledProcessError: log.error('Failed to install static build dependencies') @@ -30,7 +31,7 @@ def check_build_sdist(hooks): try: reqs = hooks.get_requires_for_build_sdist({}) log.info('Got build requires: %s', reqs) - except: + except Exception: log.error('Failure in get_requires_for_build_sdist', exc_info=True) return False @@ -47,12 +48,13 @@ def check_build_sdist(hooks): try: filename = hooks.build_sdist(td, {}) log.info('build_sdist returned %r', filename) - except: + except Exception: log.info('Failure in build_sdist', exc_info=True) return False if not filename.endswith('.tar.gz'): - log.error("Filename %s doesn't have .tar.gz extension", filename) + log.error( + "Filename %s doesn't have .tar.gz extension", filename) return False path = pjoin(td, filename) @@ -73,10 +75,11 @@ def check_build_sdist(hooks): return True -def check_build_wheel(hooks): + +def check_build_wheel(hooks, build_sys_requires): with BuildEnvironment() as env: try: - env.pip_install(hooks.build_sys_requires) + env.pip_install(build_sys_requires) log.info('Installed static build dependencies') except CalledProcessError: log.error('Failed to install static build dependencies') @@ -85,7 +88,7 @@ def check_build_wheel(hooks): try: reqs = hooks.get_requires_for_build_wheel({}) log.info('Got build requires: %s', reqs) - except: + except Exception: log.error('Failure in get_requires_for_build_sdist', exc_info=True) return False @@ -102,7 +105,7 @@ def check_build_wheel(hooks): try: filename = hooks.build_wheel(td, {}) log.info('build_wheel returned %r', filename) - except: + except Exception: log.info('Failure in build_wheel', exc_info=True) return False @@ -151,8 +154,8 @@ def check(source_dir): hooks = Pep517HookCaller(source_dir, backend) - sdist_ok = check_build_sdist(hooks) - wheel_ok = check_build_wheel(hooks) + sdist_ok = check_build_sdist(hooks, requires) + wheel_ok = check_build_wheel(hooks, requires) if not sdist_ok: log.warning('Sdist checks failed; scroll up to see') @@ -164,7 +167,8 @@ def check(source_dir): def main(argv=None): ap = argparse.ArgumentParser() - ap.add_argument('source_dir', + ap.add_argument( + 'source_dir', help="A directory containing pyproject.toml") args = ap.parse_args(argv) @@ -178,17 +182,21 @@ def main(argv=None): print(ansi('Checks failed', 'red')) sys.exit(1) + ansi_codes = { 'reset': '\x1b[0m', 'bold': '\x1b[1m', 'red': '\x1b[31m', 'green': '\x1b[32m', } + + def ansi(s, attr): if os.name != 'nt' and sys.stdout.isatty(): return ansi_codes[attr] + str(s) + ansi_codes['reset'] else: return str(s) + if __name__ == '__main__': main() diff --git a/pipenv/patched/notpip/_vendor/pep517/colorlog.py b/pipenv/patched/notpip/_vendor/pep517/colorlog.py index 26cf7480..69c8a59d 100644 --- a/pipenv/patched/notpip/_vendor/pep517/colorlog.py +++ b/pipenv/patched/notpip/_vendor/pep517/colorlog.py @@ -24,6 +24,7 @@ try: except ImportError: curses = None + def _stderr_supports_color(): color = False if curses and hasattr(sys.stderr, 'isatty') and sys.stderr.isatty(): @@ -35,13 +36,14 @@ def _stderr_supports_color(): pass return color + class LogFormatter(logging.Formatter): """Log formatter with colour support """ DEFAULT_COLORS = { - logging.INFO: 2, # Green - logging.WARNING: 3, # Yellow - logging.ERROR: 1, # Red + logging.INFO: 2, # Green + logging.WARNING: 3, # Yellow + logging.ERROR: 1, # Red logging.CRITICAL: 1, } @@ -75,7 +77,8 @@ class LogFormatter(logging.Formatter): fg_color = str(fg_color, "ascii") for levelno, code in self.DEFAULT_COLORS.items(): - self._colors[levelno] = str(curses.tparm(fg_color, code), "ascii") + self._colors[levelno] = str( + curses.tparm(fg_color, code), "ascii") self._normal = str(curses.tigetstr("sgr0"), "ascii") scr = curses.initscr() @@ -83,15 +86,16 @@ class LogFormatter(logging.Formatter): curses.endwin() else: self._normal = '' - # Default width is usually 80, but too wide is worse than too narrow + # Default width is usually 80, but too wide is + # worse than too narrow self.termwidth = 70 def formatMessage(self, record): - l = len(record.message) + mlen = len(record.message) right_text = '{initial}-{name}'.format(initial=record.levelname[0], name=record.name) - if l + len(right_text) < self.termwidth: - space = ' ' * (self.termwidth - (l + len(right_text))) + if mlen + len(right_text) < self.termwidth: + space = ' ' * (self.termwidth - (mlen + len(right_text))) else: space = ' ' @@ -103,6 +107,7 @@ class LogFormatter(logging.Formatter): return record.message + space + start_color + right_text + end_color + def enable_colourful_output(level=logging.INFO): handler = logging.StreamHandler() handler.setFormatter(LogFormatter()) diff --git a/pipenv/patched/notpip/_vendor/pep517/envbuild.py b/pipenv/patched/notpip/_vendor/pep517/envbuild.py index c54d3585..8a5ad4d7 100644 --- a/pipenv/patched/notpip/_vendor/pep517/envbuild.py +++ b/pipenv/patched/notpip/_vendor/pep517/envbuild.py @@ -14,6 +14,7 @@ from .wrappers import Pep517HookCaller log = logging.getLogger(__name__) + def _load_pyproject(source_dir): with open(os.path.join(source_dir, 'pyproject.toml')) as f: pyproject_data = pytoml.load(f) @@ -89,11 +90,17 @@ class BuildEnvironment(object): if not reqs: return log.info('Calling pip to install %s', reqs) - check_call([sys.executable, '-m', 'pip', 'install', '--ignore-installed', - '--prefix', self.path] + list(reqs)) + check_call([ + sys.executable, '-m', 'pip', 'install', '--ignore-installed', + '--prefix', self.path] + list(reqs)) def __exit__(self, exc_type, exc_val, exc_tb): - if self._cleanup and (self.path is not None) and os.path.isdir(self.path): + needs_cleanup = ( + self._cleanup and + self.path is not None and + os.path.isdir(self.path) + ) + if needs_cleanup: shutil.rmtree(self.path) if self.save_path is None: @@ -106,6 +113,7 @@ class BuildEnvironment(object): else: os.environ['PYTHONPATH'] = self.save_pythonpath + def build_wheel(source_dir, wheel_dir, config_settings=None): """Build a wheel from a source directory using PEP 517 hooks. diff --git a/pipenv/patched/notpip/_vendor/pep517/wrappers.py b/pipenv/patched/notpip/_vendor/pep517/wrappers.py index 28260f32..b14b8991 100644 --- a/pipenv/patched/notpip/_vendor/pep517/wrappers.py +++ b/pipenv/patched/notpip/_vendor/pep517/wrappers.py @@ -10,6 +10,7 @@ from . import compat _in_proc_script = pjoin(dirname(abspath(__file__)), '_in_process.py') + @contextmanager def tempdir(): td = mkdtemp() @@ -18,9 +19,24 @@ def tempdir(): finally: shutil.rmtree(td) + +class BackendUnavailable(Exception): + """Will be raised if the backend cannot be imported in the hook process.""" + + class UnsupportedOperation(Exception): """May be raised by build_sdist if the backend indicates that it can't.""" + +def default_subprocess_runner(cmd, cwd=None, extra_environ=None): + """The default method of calling the wrapper subprocess.""" + env = os.environ.copy() + if extra_environ: + env.update(extra_environ) + + check_call(cmd, cwd=cwd, env=env) + + class Pep517HookCaller(object): """A wrapper around a source directory to be built with a PEP 517 backend. @@ -30,6 +46,16 @@ class Pep517HookCaller(object): def __init__(self, source_dir, build_backend): self.source_dir = abspath(source_dir) self.build_backend = build_backend + self._subprocess_runner = default_subprocess_runner + + # TODO: Is this over-engineered? Maybe frontends only need to + # set this when creating the wrapper, not on every call. + @contextmanager + def subprocess_runner(self, runner): + prev = self._subprocess_runner + self._subprocess_runner = runner + yield + self._subprocess_runner = prev def get_requires_for_build_wheel(self, config_settings=None): """Identify packages required for building a wheel @@ -45,7 +71,8 @@ class Pep517HookCaller(object): 'config_settings': config_settings }) - def prepare_metadata_for_build_wheel(self, metadata_directory, config_settings=None): + def prepare_metadata_for_build_wheel( + self, metadata_directory, config_settings=None): """Prepare a *.dist-info folder with metadata for this project. Returns the name of the newly created folder. @@ -59,7 +86,9 @@ class Pep517HookCaller(object): 'config_settings': config_settings, }) - def build_wheel(self, wheel_directory, config_settings=None, metadata_directory=None): + def build_wheel( + self, wheel_directory, config_settings=None, + metadata_directory=None): """Build a wheel from this project. Returns the name of the newly created file. @@ -103,10 +132,7 @@ class Pep517HookCaller(object): 'config_settings': config_settings, }) - def _call_hook(self, hook_name, kwargs): - env = os.environ.copy() - # On Python 2, pytoml returns Unicode values (which is correct) but the # environment passed to check_call needs to contain string values. We # convert here by encoding using ASCII (the backend can only contain @@ -118,17 +144,20 @@ class Pep517HookCaller(object): else: build_backend = self.build_backend - env['PEP517_BUILD_BACKEND'] = build_backend with tempdir() as td: compat.write_json({'kwargs': kwargs}, pjoin(td, 'input.json'), indent=2) # Run the hook in a subprocess - check_call([sys.executable, _in_proc_script, hook_name, td], - cwd=self.source_dir, env=env) + self._subprocess_runner( + [sys.executable, _in_proc_script, hook_name, td], + cwd=self.source_dir, + extra_environ={'PEP517_BUILD_BACKEND': build_backend} + ) data = compat.read_json(pjoin(td, 'output.json')) if data.get('unsupported'): raise UnsupportedOperation + if data.get('no_backend'): + raise BackendUnavailable return data['return_val'] - diff --git a/pipenv/patched/notpip/_vendor/pkg_resources/__init__.py b/pipenv/patched/notpip/_vendor/pkg_resources/__init__.py index ac893b66..0459a7da 100644 --- a/pipenv/patched/notpip/_vendor/pkg_resources/__init__.py +++ b/pipenv/patched/notpip/_vendor/pkg_resources/__init__.py @@ -238,6 +238,9 @@ __all__ = [ 'register_finder', 'register_namespace_handler', 'register_loader_type', 'fixup_namespace_packages', 'get_importer', + # Warnings + 'PkgResourcesDeprecationWarning', + # Deprecated/backward compatibility only 'run_main', 'AvailableDistributions', ] @@ -2228,7 +2231,18 @@ register_namespace_handler(object, null_ns_handler) def normalize_path(filename): """Normalize a file/dir name for comparison purposes""" - return os.path.normcase(os.path.realpath(filename)) + return os.path.normcase(os.path.realpath(os.path.normpath(_cygwin_patch(filename)))) + + +def _cygwin_patch(filename): # pragma: nocover + """ + Contrary to POSIX 2008, on Cygwin, getcwd (3) contains + symlink components. Using + os.path.abspath() works around this limitation. A fix in os.getcwd() + would probably better, in Cygwin even more so, except + that this seems to be by design... + """ + return os.path.abspath(filename) if sys.platform == 'cygwin' else filename def _normalize_cached(filename, _cache={}): @@ -2324,7 +2338,7 @@ class EntryPoint: warnings.warn( "Parameters to load are deprecated. Call .resolve and " ".require separately.", - DeprecationWarning, + PkgResourcesDeprecationWarning, stacklevel=2, ) if require: @@ -3147,3 +3161,11 @@ def _initialize_master_working_set(): # match order list(map(working_set.add_entry, sys.path)) globals().update(locals()) + +class PkgResourcesDeprecationWarning(Warning): + """ + Base class for warning about deprecations in ``pkg_resources`` + + This class is not derived from ``DeprecationWarning``, and as such is + visible by default. + """ diff --git a/pipenv/patched/notpip/_vendor/pyparsing.py b/pipenv/patched/notpip/_vendor/pyparsing.py index 455d1151..3972b370 100644 --- a/pipenv/patched/notpip/_vendor/pyparsing.py +++ b/pipenv/patched/notpip/_vendor/pyparsing.py @@ -1,6 +1,7 @@ +#-*- coding: utf-8 -*- # module pyparsing.py # -# Copyright (c) 2003-2018 Paul T. McGuire +# Copyright (c) 2003-2019 Paul T. McGuire # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -27,15 +28,18 @@ __doc__ = \ pyparsing module - Classes and methods to define and execute parsing grammars ============================================================================= -The pyparsing module is an alternative approach to creating and executing simple grammars, -vs. the traditional lex/yacc approach, or the use of regular expressions. With pyparsing, you -don't need to learn a new syntax for defining grammars or matching expressions - the parsing module -provides a library of classes that you use to construct the grammar directly in Python. +The pyparsing module is an alternative approach to creating and +executing simple grammars, vs. the traditional lex/yacc approach, or the +use of regular expressions. With pyparsing, you don't need to learn +a new syntax for defining grammars or matching expressions - the parsing +module provides a library of classes that you use to construct the +grammar directly in Python. -Here is a program to parse "Hello, World!" (or any greeting of the form -C{", !"}), built up using L{Word}, L{Literal}, and L{And} elements -(L{'+'} operator gives L{And} expressions, strings are auto-converted to -L{Literal} expressions):: +Here is a program to parse "Hello, World!" (or any greeting of the form +``", !"``), built up using :class:`Word`, +:class:`Literal`, and :class:`And` elements +(the :class:`'+'` operators create :class:`And` expressions, +and the strings are auto-converted to :class:`Literal` expressions):: from pipenv.patched.notpip._vendor.pyparsing import Word, alphas @@ -49,33 +53,48 @@ The program outputs the following:: Hello, World! -> ['Hello', ',', 'World', '!'] -The Python representation of the grammar is quite readable, owing to the self-explanatory -class names, and the use of '+', '|' and '^' operators. +The Python representation of the grammar is quite readable, owing to the +self-explanatory class names, and the use of '+', '|' and '^' operators. -The L{ParseResults} object returned from L{ParserElement.parseString} can be accessed as a nested list, a dictionary, or an -object with named attributes. +The :class:`ParseResults` object returned from +:class:`ParserElement.parseString` can be +accessed as a nested list, a dictionary, or an object with named +attributes. -The pyparsing module handles some of the problems that are typically vexing when writing text parsers: - - extra or missing whitespace (the above program will also handle "Hello,World!", "Hello , World !", etc.) - - quoted strings - - embedded comments +The pyparsing module handles some of the problems that are typically +vexing when writing text parsers: + + - extra or missing whitespace (the above program will also handle + "Hello,World!", "Hello , World !", etc.) + - quoted strings + - embedded comments Getting Started - ----------------- -Visit the classes L{ParserElement} and L{ParseResults} to see the base classes that most other pyparsing +Visit the classes :class:`ParserElement` and :class:`ParseResults` to +see the base classes that most other pyparsing classes inherit from. Use the docstrings for examples of how to: - - construct literal match expressions from L{Literal} and L{CaselessLiteral} classes - - construct character word-group expressions using the L{Word} class - - see how to create repetitive expressions using L{ZeroOrMore} and L{OneOrMore} classes - - use L{'+'}, L{'|'}, L{'^'}, and L{'&'} operators to combine simple expressions into more complex ones - - associate names with your parsed results using L{ParserElement.setResultsName} - - find some helpful expression short-cuts like L{delimitedList} and L{oneOf} - - find more useful common expressions in the L{pyparsing_common} namespace class + + - construct literal match expressions from :class:`Literal` and + :class:`CaselessLiteral` classes + - construct character word-group expressions using the :class:`Word` + class + - see how to create repetitive expressions using :class:`ZeroOrMore` + and :class:`OneOrMore` classes + - use :class:`'+'`, :class:`'|'`, :class:`'^'`, + and :class:`'&'` operators to combine simple expressions into + more complex ones + - associate names with your parsed results using + :class:`ParserElement.setResultsName` + - find some helpful expression short-cuts like :class:`delimitedList` + and :class:`oneOf` + - find more useful common expressions in the :class:`pyparsing_common` + namespace class """ -__version__ = "2.2.1" -__versionTime__ = "18 Sep 2018 00:49 UTC" +__version__ = "2.3.1" +__versionTime__ = "09 Jan 2019 23:26 UTC" __author__ = "Paul McGuire " import string @@ -91,6 +110,12 @@ import traceback import types from datetime import datetime +try: + # Python 3 + from itertools import filterfalse +except ImportError: + from itertools import ifilterfalse as filterfalse + try: from _thread import RLock except ImportError: @@ -113,27 +138,33 @@ except ImportError: except ImportError: _OrderedDict = None +try: + from types import SimpleNamespace +except ImportError: + class SimpleNamespace: pass + + #~ sys.stderr.write( "testing pyparsing module, version %s, %s\n" % (__version__,__versionTime__ ) ) __all__ = [ 'And', 'CaselessKeyword', 'CaselessLiteral', 'CharsNotIn', 'Combine', 'Dict', 'Each', 'Empty', 'FollowedBy', 'Forward', 'GoToColumn', 'Group', 'Keyword', 'LineEnd', 'LineStart', 'Literal', -'MatchFirst', 'NoMatch', 'NotAny', 'OneOrMore', 'OnlyOnce', 'Optional', 'Or', +'PrecededBy', 'MatchFirst', 'NoMatch', 'NotAny', 'OneOrMore', 'OnlyOnce', 'Optional', 'Or', 'ParseBaseException', 'ParseElementEnhance', 'ParseException', 'ParseExpression', 'ParseFatalException', 'ParseResults', 'ParseSyntaxException', 'ParserElement', 'QuotedString', 'RecursiveGrammarException', -'Regex', 'SkipTo', 'StringEnd', 'StringStart', 'Suppress', 'Token', 'TokenConverter', -'White', 'Word', 'WordEnd', 'WordStart', 'ZeroOrMore', +'Regex', 'SkipTo', 'StringEnd', 'StringStart', 'Suppress', 'Token', 'TokenConverter', +'White', 'Word', 'WordEnd', 'WordStart', 'ZeroOrMore', 'Char', 'alphanums', 'alphas', 'alphas8bit', 'anyCloseTag', 'anyOpenTag', 'cStyleComment', 'col', 'commaSeparatedList', 'commonHTMLEntity', 'countedArray', 'cppStyleComment', 'dblQuotedString', 'dblSlashComment', 'delimitedList', 'dictOf', 'downcaseTokens', 'empty', 'hexnums', 'htmlComment', 'javaStyleComment', 'line', 'lineEnd', 'lineStart', 'lineno', 'makeHTMLTags', 'makeXMLTags', 'matchOnlyAtCol', 'matchPreviousExpr', 'matchPreviousLiteral', 'nestedExpr', 'nullDebugAction', 'nums', 'oneOf', 'opAssoc', 'operatorPrecedence', 'printables', -'punc8bit', 'pythonStyleComment', 'quotedString', 'removeQuotes', 'replaceHTMLEntity', +'punc8bit', 'pythonStyleComment', 'quotedString', 'removeQuotes', 'replaceHTMLEntity', 'replaceWith', 'restOfLine', 'sglQuotedString', 'srange', 'stringEnd', 'stringStart', 'traceParseAction', 'unicodeString', 'upcaseTokens', 'withAttribute', 'indentedBlock', 'originalTextFor', 'ungroup', 'infixNotation','locatedExpr', 'withClass', -'CloseMatch', 'tokenMap', 'pyparsing_common', +'CloseMatch', 'tokenMap', 'pyparsing_common', 'pyparsing_unicode', 'unicode_set', ] system_version = tuple(sys.version_info)[:3] @@ -142,6 +173,7 @@ if PY_3: _MAX_INT = sys.maxsize basestring = str unichr = chr + unicode = str _ustr = str # build list of single arg builtins, that can be used as parse actions @@ -152,9 +184,11 @@ else: range = xrange def _ustr(obj): - """Drop-in replacement for str(obj) that tries to be Unicode friendly. It first tries - str(obj). If that fails with a UnicodeEncodeError, then it tries unicode(obj). It - then < returns the unicode object | encodes it with the default encoding | ... >. + """Drop-in replacement for str(obj) that tries to be Unicode + friendly. It first tries str(obj). If that fails with + a UnicodeEncodeError, then it tries unicode(obj). It then + < returns the unicode object | encodes it with the default + encoding | ... >. """ if isinstance(obj,unicode): return obj @@ -179,9 +213,9 @@ else: singleArgBuiltins.append(getattr(__builtin__,fname)) except AttributeError: continue - + _generatorType = type((y for y in range(1))) - + def _xml_escape(data): """Escape &, <, >, ", ', etc. in a string of data.""" @@ -192,9 +226,6 @@ def _xml_escape(data): data = data.replace(from_, to_) return data -class _Constants(object): - pass - alphas = string.ascii_uppercase + string.ascii_lowercase nums = "0123456789" hexnums = nums + "ABCDEFabcdef" @@ -220,16 +251,16 @@ class ParseBaseException(Exception): @classmethod def _from_exception(cls, pe): """ - internal factory method to simplify creating one type of ParseException + internal factory method to simplify creating one type of ParseException from another - avoids having __init__ signature conflicts among subclasses """ return cls(pe.pstr, pe.loc, pe.msg, pe.parserElement) def __getattr__( self, aname ): """supported attributes by name are: - - lineno - returns the line number of the exception text - - col - returns the column number of the exception text - - line - returns the line containing the exception text + - lineno - returns the line number of the exception text + - col - returns the column number of the exception text + - line - returns the line containing the exception text """ if( aname == "lineno" ): return lineno( self.loc, self.pstr ) @@ -262,22 +293,94 @@ class ParseException(ParseBaseException): """ Exception thrown when parse expressions don't match class; supported attributes by name are: - - lineno - returns the line number of the exception text - - col - returns the column number of the exception text - - line - returns the line containing the exception text - + - lineno - returns the line number of the exception text + - col - returns the column number of the exception text + - line - returns the line containing the exception text + Example:: + try: Word(nums).setName("integer").parseString("ABC") except ParseException as pe: print(pe) print("column: {}".format(pe.col)) - + prints:: + Expected integer (at char 0), (line:1, col:1) column: 1 + """ - pass + + @staticmethod + def explain(exc, depth=16): + """ + Method to take an exception and translate the Python internal traceback into a list + of the pyparsing expressions that caused the exception to be raised. + + Parameters: + + - exc - exception raised during parsing (need not be a ParseException, in support + of Python exceptions that might be raised in a parse action) + - depth (default=16) - number of levels back in the stack trace to list expression + and function names; if None, the full stack trace names will be listed; if 0, only + the failing input line, marker, and exception string will be shown + + Returns a multi-line string listing the ParserElements and/or function names in the + exception's stack trace. + + Note: the diagnostic output will include string representations of the expressions + that failed to parse. These representations will be more helpful if you use `setName` to + give identifiable names to your expressions. Otherwise they will use the default string + forms, which may be cryptic to read. + + explain() is only supported under Python 3. + """ + import inspect + + if depth is None: + depth = sys.getrecursionlimit() + ret = [] + if isinstance(exc, ParseBaseException): + ret.append(exc.line) + ret.append(' ' * (exc.col - 1) + '^') + ret.append("{0}: {1}".format(type(exc).__name__, exc)) + + if depth > 0: + callers = inspect.getinnerframes(exc.__traceback__, context=depth) + seen = set() + for i, ff in enumerate(callers[-depth:]): + frm = ff.frame + + f_self = frm.f_locals.get('self', None) + if isinstance(f_self, ParserElement): + if frm.f_code.co_name not in ('parseImpl', '_parseNoCache'): + continue + if f_self in seen: + continue + seen.add(f_self) + + self_type = type(f_self) + ret.append("{0}.{1} - {2}".format(self_type.__module__, + self_type.__name__, + f_self)) + elif f_self is not None: + self_type = type(f_self) + ret.append("{0}.{1}".format(self_type.__module__, + self_type.__name__)) + else: + code = frm.f_code + if code.co_name in ('wrapper', ''): + continue + + ret.append("{0}".format(code.co_name)) + + depth -= 1 + if not depth: + break + + return '\n'.join(ret) + class ParseFatalException(ParseBaseException): """user-throwable exception thrown when inconsistent parse content @@ -285,9 +388,11 @@ class ParseFatalException(ParseBaseException): pass class ParseSyntaxException(ParseFatalException): - """just like L{ParseFatalException}, but thrown internally when an - L{ErrorStop} ('-' operator) indicates that parsing is to stop - immediately because an unbacktrackable syntax error has been found""" + """just like :class:`ParseFatalException`, but thrown internally + when an :class:`ErrorStop` ('-' operator) indicates + that parsing is to stop immediately because an unbacktrackable + syntax error has been found. + """ pass #~ class ReparseException(ParseBaseException): @@ -304,7 +409,9 @@ class ParseSyntaxException(ParseFatalException): #~ self.reparseLoc = restartLoc class RecursiveGrammarException(Exception): - """exception thrown by L{ParserElement.validate} if the grammar could be improperly recursive""" + """exception thrown by :class:`ParserElement.validate` if the + grammar could be improperly recursive + """ def __init__( self, parseElementList ): self.parseElementTrace = parseElementList @@ -322,16 +429,18 @@ class _ParseResultsWithOffset(object): self.tup = (self.tup[0],i) class ParseResults(object): - """ - Structured parse results, to provide multiple means of access to the parsed data: - - as a list (C{len(results)}) - - by list index (C{results[0], results[1]}, etc.) - - by attribute (C{results.} - see L{ParserElement.setResultsName}) + """Structured parse results, to provide multiple means of access to + the parsed data: + + - as a list (``len(results)``) + - by list index (``results[0], results[1]``, etc.) + - by attribute (``results.`` - see :class:`ParserElement.setResultsName`) Example:: + integer = Word(nums) - date_str = (integer.setResultsName("year") + '/' - + integer.setResultsName("month") + '/' + date_str = (integer.setResultsName("year") + '/' + + integer.setResultsName("month") + '/' + integer.setResultsName("day")) # equivalent form: # date_str = integer("year") + '/' + integer("month") + '/' + integer("day") @@ -348,7 +457,9 @@ class ParseResults(object): test("'month' in result") test("'minutes' in result") test("result.dump()", str) + prints:: + list(result) -> ['1999', '/', '12', '/', '31'] result[0] -> '1999' result['month'] -> '12' @@ -398,7 +509,7 @@ class ParseResults(object): toklist = [ toklist ] if asList: if isinstance(toklist,ParseResults): - self[name] = _ParseResultsWithOffset(toklist.copy(),0) + self[name] = _ParseResultsWithOffset(ParseResults(toklist.__toklist), 0) else: self[name] = _ParseResultsWithOffset(ParseResults(toklist[0]),0) self[name].__name = name @@ -467,19 +578,19 @@ class ParseResults(object): def _itervalues( self ): return (self[k] for k in self._iterkeys()) - + def _iteritems( self ): return ((k, self[k]) for k in self._iterkeys()) if PY_3: - keys = _iterkeys - """Returns an iterator of all named result keys (Python 3.x only).""" + keys = _iterkeys + """Returns an iterator of all named result keys.""" values = _itervalues - """Returns an iterator of all named result values (Python 3.x only).""" + """Returns an iterator of all named result values.""" items = _iteritems - """Returns an iterator of all named result key-value tuples (Python 3.x only).""" + """Returns an iterator of all named result key-value tuples.""" else: iterkeys = _iterkeys @@ -498,7 +609,7 @@ class ParseResults(object): def values( self ): """Returns all named result values (as a list in Python 2.x, as an iterator in Python 3.x).""" return list(self.itervalues()) - + def items( self ): """Returns all named result key-values (as a list of tuples in Python 2.x, as an iterator in Python 3.x).""" return list(self.iteritems()) @@ -507,19 +618,20 @@ class ParseResults(object): """Since keys() returns an iterator, this method is helpful in bypassing code that looks for the existence of any defined results names.""" return bool(self.__tokdict) - + def pop( self, *args, **kwargs): """ - Removes and returns item at specified index (default=C{last}). - Supports both C{list} and C{dict} semantics for C{pop()}. If passed no - argument or an integer argument, it will use C{list} semantics - and pop tokens from the list of parsed tokens. If passed a - non-integer argument (most likely a string), it will use C{dict} - semantics and pop the corresponding value from any defined - results names. A second default return value argument is - supported, just as in C{dict.pop()}. + Removes and returns item at specified index (default= ``last``). + Supports both ``list`` and ``dict`` semantics for ``pop()``. If + passed no argument or an integer argument, it will use ``list`` + semantics and pop tokens from the list of parsed tokens. If passed + a non-integer argument (most likely a string), it will use ``dict`` + semantics and pop the corresponding value from any defined results + names. A second default return value argument is supported, just as in + ``dict.pop()``. Example:: + def remove_first(tokens): tokens.pop(0) print(OneOrMore(Word(nums)).parseString("0 123 321")) # -> ['0', '123', '321'] @@ -536,7 +648,9 @@ class ParseResults(object): return tokens patt.addParseAction(remove_LABEL) print(patt.parseString("AAB 123 321").dump()) + prints:: + ['AAB', '123', '321'] - LABEL: AAB @@ -549,8 +663,8 @@ class ParseResults(object): args = (args[0], v) else: raise TypeError("pop() got an unexpected keyword argument '%s'" % k) - if (isinstance(args[0], int) or - len(args) == 1 or + if (isinstance(args[0], int) or + len(args) == 1 or args[0] in self): index = args[0] ret = self[index] @@ -563,14 +677,15 @@ class ParseResults(object): def get(self, key, defaultValue=None): """ Returns named result matching the given key, or if there is no - such name, then returns the given C{defaultValue} or C{None} if no - C{defaultValue} is specified. + such name, then returns the given ``defaultValue`` or ``None`` if no + ``defaultValue`` is specified. + + Similar to ``dict.get()``. - Similar to C{dict.get()}. - Example:: + integer = Word(nums) - date_str = integer("year") + '/' + integer("month") + '/' + integer("day") + date_str = integer("year") + '/' + integer("month") + '/' + integer("day") result = date_str.parseString("1999/12/31") print(result.get("year")) # -> '1999' @@ -585,10 +700,11 @@ class ParseResults(object): def insert( self, index, insStr ): """ Inserts new element at location index in the list of parsed tokens. - - Similar to C{list.insert()}. + + Similar to ``list.insert()``. Example:: + print(OneOrMore(Word(nums)).parseString("0 123 321")) # -> ['0', '123', '321'] # use a parse action to insert the parse location in the front of the parsed results @@ -607,8 +723,9 @@ class ParseResults(object): Add single element to end of ParseResults list of elements. Example:: + print(OneOrMore(Word(nums)).parseString("0 123 321")) # -> ['0', '123', '321'] - + # use a parse action to compute the sum of the parsed integers, and add it to the end def append_sum(tokens): tokens.append(sum(map(int, tokens))) @@ -621,8 +738,9 @@ class ParseResults(object): Add sequence of elements to end of ParseResults list of elements. Example:: + patt = OneOrMore(Word(alphas)) - + # use a parse action to append the reverse of the matched strings, to make a palindrome def make_palindrome(tokens): tokens.extend(reversed([t[::-1] for t in tokens])) @@ -646,7 +764,7 @@ class ParseResults(object): return self[name] except KeyError: return "" - + if name in self.__tokdict: if name not in self.__accumNames: return self.__tokdict[name][-1][0] @@ -671,7 +789,7 @@ class ParseResults(object): self[k] = v if isinstance(v[0],ParseResults): v[0].__parent = wkref(self) - + self.__toklist += other.__toklist self.__accumNames.update( other.__accumNames ) return self @@ -683,7 +801,7 @@ class ParseResults(object): else: # this may raise a TypeError - so be it return other + self - + def __repr__( self ): return "(%s, %s)" % ( repr( self.__toklist ), repr( self.__tokdict ) ) @@ -706,11 +824,12 @@ class ParseResults(object): Returns the parse results as a nested list of matching tokens, all converted to strings. Example:: + patt = OneOrMore(Word(alphas)) result = patt.parseString("sldkj lsdkj sldkj") # even though the result prints in string-like form, it is actually a pyparsing ParseResults print(type(result), result) # -> ['sldkj', 'lsdkj', 'sldkj'] - + # Use asList() to create an actual list result_list = result.asList() print(type(result_list), result_list) # -> ['sldkj', 'lsdkj', 'sldkj'] @@ -722,12 +841,13 @@ class ParseResults(object): Returns the named parse results as a nested dictionary. Example:: + integer = Word(nums) date_str = integer("year") + '/' + integer("month") + '/' + integer("day") - + result = date_str.parseString('12/31/1999') print(type(result), repr(result)) # -> (['12', '/', '31', '/', '1999'], {'day': [('1999', 4)], 'year': [('12', 0)], 'month': [('31', 2)]}) - + result_dict = result.asDict() print(type(result_dict), repr(result_dict)) # -> {'day': '1999', 'year': '12', 'month': '31'} @@ -740,7 +860,7 @@ class ParseResults(object): item_fn = self.items else: item_fn = self.iteritems - + def toItem(obj): if isinstance(obj, ParseResults): if obj.haskeys(): @@ -749,15 +869,15 @@ class ParseResults(object): return [toItem(v) for v in obj] else: return obj - + return dict((k,toItem(v)) for k,v in item_fn()) def copy( self ): """ - Returns a new copy of a C{ParseResults} object. + Returns a new copy of a :class:`ParseResults` object. """ ret = ParseResults( self.__toklist ) - ret.__tokdict = self.__tokdict.copy() + ret.__tokdict = dict(self.__tokdict.items()) ret.__parent = self.__parent ret.__accumNames.update( self.__accumNames ) ret.__name = self.__name @@ -833,22 +953,25 @@ class ParseResults(object): def getName(self): r""" - Returns the results name for this token expression. Useful when several + Returns the results name for this token expression. Useful when several different expressions might match at a particular location. Example:: + integer = Word(nums) ssn_expr = Regex(r"\d\d\d-\d\d-\d\d\d\d") house_number_expr = Suppress('#') + Word(nums, alphanums) - user_data = (Group(house_number_expr)("house_number") + user_data = (Group(house_number_expr)("house_number") | Group(ssn_expr)("ssn") | Group(integer)("age")) user_info = OneOrMore(user_data) - + result = user_info.parseString("22 111-22-3333 #221B") for item in result: print(item.getName(), ':', item[0]) + prints:: + age : 22 ssn : 111-22-3333 house_number : 221B @@ -870,17 +993,20 @@ class ParseResults(object): def dump(self, indent='', depth=0, full=True): """ - Diagnostic method for listing out the contents of a C{ParseResults}. - Accepts an optional C{indent} argument so that this string can be embedded - in a nested display of other data. + Diagnostic method for listing out the contents of + a :class:`ParseResults`. Accepts an optional ``indent`` argument so + that this string can be embedded in a nested display of other data. Example:: + integer = Word(nums) date_str = integer("year") + '/' + integer("month") + '/' + integer("day") - + result = date_str.parseString('12/31/1999') print(result.dump()) + prints:: + ['12', '/', '31', '/', '1999'] - day: 1999 - month: 31 @@ -910,16 +1036,18 @@ class ParseResults(object): out.append("\n%s%s[%d]:\n%s%s%s" % (indent,(' '*(depth)),i,indent,(' '*(depth+1)),vv.dump(indent,depth+1) )) else: out.append("\n%s%s[%d]:\n%s%s%s" % (indent,(' '*(depth)),i,indent,(' '*(depth+1)),_ustr(vv))) - + return "".join(out) def pprint(self, *args, **kwargs): """ - Pretty-printer for parsed results as a list, using the C{pprint} module. - Accepts additional positional or keyword args as defined for the - C{pprint.pprint} method. (U{http://docs.python.org/3/library/pprint.html#pprint.pprint}) + Pretty-printer for parsed results as a list, using the + `pprint `_ module. + Accepts additional positional or keyword args as defined for + `pprint.pprint `_ . Example:: + ident = Word(alphas, alphanums) num = Word(nums) func = Forward() @@ -927,7 +1055,9 @@ class ParseResults(object): func <<= ident + Group(Optional(delimitedList(term))) result = func.parseString("fna a,b,(fnb c,d,200),100") result.pprint(width=40) + prints:: + ['fna', ['a', 'b', @@ -970,24 +1100,25 @@ def col (loc,strg): The first column is number 1. Note: the default parsing behavior is to expand tabs in the input string - before starting the parsing process. See L{I{ParserElement.parseString}} for more information - on parsing strings containing C{}s, and suggested methods to maintain a - consistent view of the parsed string, the parse location, and line and column - positions within the parsed string. + before starting the parsing process. See + :class:`ParserElement.parseString` for more + information on parsing strings containing ```` s, and suggested + methods to maintain a consistent view of the parsed string, the parse + location, and line and column positions within the parsed string. """ s = strg return 1 if 0} for more information - on parsing strings containing C{}s, and suggested methods to maintain a - consistent view of the parsed string, the parse location, and line and column - positions within the parsed string. - """ + Note - the default parsing behavior is to expand tabs in the input string + before starting the parsing process. See :class:`ParserElement.parseString` + for more information on parsing strings containing ```` s, and + suggested methods to maintain a consistent view of the parsed string, the + parse location, and line and column positions within the parsed string. + """ return strg.count("\n",0,loc) + 1 def line( loc, strg ): @@ -1041,7 +1172,7 @@ def _trim_arity(func, maxargs=2): return lambda s,l,t: func(t) limit = [0] foundArity = [False] - + # traceback return data structure changed in Py3.5 - normalize back to plain tuples if system_version[:2] >= (3,5): def extract_stack(limit=0): @@ -1056,12 +1187,12 @@ def _trim_arity(func, maxargs=2): else: extract_stack = traceback.extract_stack extract_tb = traceback.extract_tb - - # synthesize what would be returned by traceback.extract_stack at the call to + + # synthesize what would be returned by traceback.extract_stack at the call to # user's parse action 'func', so that we don't incur call penalty at parse time - + LINE_DIFF = 6 - # IF ANY CODE CHANGES, EVEN JUST COMMENTS OR BLANK LINES, BETWEEN THE NEXT LINE AND + # IF ANY CODE CHANGES, EVEN JUST COMMENTS OR BLANK LINES, BETWEEN THE NEXT LINE AND # THE CALL TO FUNC INSIDE WRAPPER, LINE_DIFF MUST BE MODIFIED!!!! this_line = extract_stack(limit=2)[-1] pa_call_line_synth = (this_line[0], this_line[1]+LINE_DIFF) @@ -1092,7 +1223,7 @@ def _trim_arity(func, maxargs=2): # copy func name to wrapper for sensible debug output func_name = "" try: - func_name = getattr(func, '__name__', + func_name = getattr(func, '__name__', getattr(func, '__class__').__name__) except Exception: func_name = str(func) @@ -1111,9 +1242,10 @@ class ParserElement(object): Overrides the default whitespace chars Example:: + # default whitespace chars are space, and newline OneOrMore(Word(alphas)).parseString("abc def\nghi jkl") # -> ['abc', 'def', 'ghi', 'jkl'] - + # change to just treat newline as significant ParserElement.setDefaultWhitespaceChars(" \t") OneOrMore(Word(alphas)).parseString("abc def\nghi jkl") # -> ['abc', 'def'] @@ -1124,18 +1256,19 @@ class ParserElement(object): def inlineLiteralsUsing(cls): """ Set class to be used for inclusion of string literals into a parser. - + Example:: + # default literal class used is Literal integer = Word(nums) - date_str = integer("year") + '/' + integer("month") + '/' + integer("day") + date_str = integer("year") + '/' + integer("month") + '/' + integer("day") date_str.parseString("1999/12/31") # -> ['1999', '/', '12', '/', '31'] # change to Suppress ParserElement.inlineLiteralsUsing(Suppress) - date_str = integer("year") + '/' + integer("month") + '/' + integer("day") + date_str = integer("year") + '/' + integer("month") + '/' + integer("day") date_str.parseString("1999/12/31") # -> ['1999', '12', '31'] """ @@ -1149,7 +1282,7 @@ class ParserElement(object): self.resultsName = None self.saveAsList = savelist self.skipWhitespace = True - self.whiteChars = ParserElement.DEFAULT_WHITE_CHARS + self.whiteChars = set(ParserElement.DEFAULT_WHITE_CHARS) self.copyDefaultWhiteChars = True self.mayReturnEmpty = False # used when checking for left-recursion self.keepTabs = False @@ -1166,18 +1299,24 @@ class ParserElement(object): def copy( self ): """ - Make a copy of this C{ParserElement}. Useful for defining different parse actions - for the same parsing pattern, using copies of the original parse element. - + Make a copy of this :class:`ParserElement`. Useful for defining + different parse actions for the same parsing pattern, using copies of + the original parse element. + Example:: + integer = Word(nums).setParseAction(lambda toks: int(toks[0])) integerK = integer.copy().addParseAction(lambda toks: toks[0]*1024) + Suppress("K") integerM = integer.copy().addParseAction(lambda toks: toks[0]*1024*1024) + Suppress("M") - + print(OneOrMore(integerK | integerM | integer).parseString("5K 100 640K 256M")) + prints:: + [5120, 100, 655360, 268435456] - Equivalent form of C{expr.copy()} is just C{expr()}:: + + Equivalent form of ``expr.copy()`` is just ``expr()``:: + integerM = integer().addParseAction(lambda toks: toks[0]*1024*1024) + Suppress("M") """ cpy = copy.copy( self ) @@ -1190,8 +1329,9 @@ class ParserElement(object): def setName( self, name ): """ Define name for this expression, makes debugging and exception messages clearer. - + Example:: + Word(nums).parseString("ABC") # -> Exception: Expected W:(0123...) (at char 0), (line:1, col:1) Word(nums).setName("integer").parseString("ABC") # -> Exception: Expected integer (at char 0), (line:1, col:1) """ @@ -1205,17 +1345,18 @@ class ParserElement(object): """ Define name for referencing matching tokens as a nested attribute of the returned parse results. - NOTE: this returns a *copy* of the original C{ParserElement} object; + NOTE: this returns a *copy* of the original :class:`ParserElement` object; this is so that the client can define a basic element, such as an integer, and reference it in multiple places with different names. You can also set results names using the abbreviated syntax, - C{expr("name")} in place of C{expr.setResultsName("name")} - - see L{I{__call__}<__call__>}. + ``expr("name")`` in place of ``expr.setResultsName("name")`` + - see :class:`__call__`. Example:: - date_str = (integer.setResultsName("year") + '/' - + integer.setResultsName("month") + '/' + + date_str = (integer.setResultsName("year") + '/' + + integer.setResultsName("month") + '/' + integer.setResultsName("day")) # equivalent form: @@ -1231,7 +1372,7 @@ class ParserElement(object): def setBreak(self,breakFlag = True): """Method to invoke the Python pdb debugger when this element is - about to be parsed. Set C{breakFlag} to True to enable, False to + about to be parsed. Set ``breakFlag`` to True to enable, False to disable. """ if breakFlag: @@ -1250,25 +1391,28 @@ class ParserElement(object): def setParseAction( self, *fns, **kwargs ): """ Define one or more actions to perform when successfully matching parse element definition. - Parse action fn is a callable method with 0-3 arguments, called as C{fn(s,loc,toks)}, - C{fn(loc,toks)}, C{fn(toks)}, or just C{fn()}, where: - - s = the original string being parsed (see note below) - - loc = the location of the matching substring - - toks = a list of the matched tokens, packaged as a C{L{ParseResults}} object + Parse action fn is a callable method with 0-3 arguments, called as ``fn(s,loc,toks)`` , + ``fn(loc,toks)`` , ``fn(toks)`` , or just ``fn()`` , where: + + - s = the original string being parsed (see note below) + - loc = the location of the matching substring + - toks = a list of the matched tokens, packaged as a :class:`ParseResults` object + If the functions in fns modify the tokens, they can return them as the return value from fn, and the modified list of tokens will replace the original. Otherwise, fn does not need to return any value. Optional keyword arguments: - - callDuringTry = (default=C{False}) indicate if parse action should be run during lookaheads and alternate testing + - callDuringTry = (default= ``False`` ) indicate if parse action should be run during lookaheads and alternate testing Note: the default parsing behavior is to expand tabs in the input string - before starting the parsing process. See L{I{parseString}} for more information - on parsing strings containing C{}s, and suggested methods to maintain a - consistent view of the parsed string, the parse location, and line and column - positions within the parsed string. - + before starting the parsing process. See :class:`parseString for more + information on parsing strings containing ```` s, and suggested + methods to maintain a consistent view of the parsed string, the parse + location, and line and column positions within the parsed string. + Example:: + integer = Word(nums) date_str = integer + '/' + integer + '/' + integer @@ -1287,24 +1431,25 @@ class ParserElement(object): def addParseAction( self, *fns, **kwargs ): """ - Add one or more parse actions to expression's list of parse actions. See L{I{setParseAction}}. - - See examples in L{I{copy}}. + Add one or more parse actions to expression's list of parse actions. See :class:`setParseAction`. + + See examples in :class:`copy`. """ self.parseAction += list(map(_trim_arity, list(fns))) self.callDuringTry = self.callDuringTry or kwargs.get("callDuringTry", False) return self def addCondition(self, *fns, **kwargs): - """Add a boolean predicate function to expression's list of parse actions. See - L{I{setParseAction}} for function call signatures. Unlike C{setParseAction}, - functions passed to C{addCondition} need to return boolean success/fail of the condition. + """Add a boolean predicate function to expression's list of parse actions. See + :class:`setParseAction` for function call signatures. Unlike ``setParseAction``, + functions passed to ``addCondition`` need to return boolean success/fail of the condition. Optional keyword arguments: - - message = define a custom message to be used in the raised exception - - fatal = if True, will raise ParseFatalException to stop parsing immediately; otherwise will raise ParseException - + - message = define a custom message to be used in the raised exception + - fatal = if True, will raise ParseFatalException to stop parsing immediately; otherwise will raise ParseException + Example:: + integer = Word(nums).setParseAction(lambda toks: int(toks[0])) year_int = integer.copy() year_int.addCondition(lambda toks: toks[0] >= 2000, message="Only support years 2000 and later") @@ -1315,8 +1460,9 @@ class ParserElement(object): msg = kwargs.get("message", "failed user-defined condition") exc_type = ParseFatalException if kwargs.get("fatal", False) else ParseException for fn in fns: + fn = _trim_arity(fn) def pa(s,l,t): - if not bool(_trim_arity(fn)(s,l,t)): + if not bool(fn(s,l,t)): raise exc_type(s,l,msg) self.parseAction.append(pa) self.callDuringTry = self.callDuringTry or kwargs.get("callDuringTry", False) @@ -1325,12 +1471,12 @@ class ParserElement(object): def setFailAction( self, fn ): """Define action to perform if parsing fails at this expression. Fail acton fn is a callable function that takes the arguments - C{fn(s,loc,expr,err)} where: - - s = string being parsed - - loc = location where expression match was attempted and failed - - expr = the parse expression that failed - - err = the exception thrown - The function returns no value. It may throw C{L{ParseFatalException}} + ``fn(s,loc,expr,err)`` where: + - s = string being parsed + - loc = location where expression match was attempted and failed + - expr = the parse expression that failed + - err = the exception thrown + The function returns no value. It may throw :class:`ParseFatalException` if it is desired to stop parsing immediately.""" self.failAction = fn return self @@ -1412,8 +1558,14 @@ class ParserElement(object): if debugging: try: for fn in self.parseAction: - tokens = fn( instring, tokensStart, retTokens ) - if tokens is not None: + try: + tokens = fn( instring, tokensStart, retTokens ) + except IndexError as parse_action_exc: + exc = ParseException("exception raised in parse action") + exc.__cause__ = parse_action_exc + raise exc + + if tokens is not None and tokens is not retTokens: retTokens = ParseResults( tokens, self.resultsName, asList=self.saveAsList and isinstance(tokens,(ParseResults,list)), @@ -1425,8 +1577,14 @@ class ParserElement(object): raise else: for fn in self.parseAction: - tokens = fn( instring, tokensStart, retTokens ) - if tokens is not None: + try: + tokens = fn( instring, tokensStart, retTokens ) + except IndexError as parse_action_exc: + exc = ParseException("exception raised in parse action") + exc.__cause__ = parse_action_exc + raise exc + + if tokens is not None and tokens is not retTokens: retTokens = ParseResults( tokens, self.resultsName, asList=self.saveAsList and isinstance(tokens,(ParseResults,list)), @@ -1443,7 +1601,7 @@ class ParserElement(object): return self._parse( instring, loc, doActions=False )[0] except ParseFatalException: raise ParseException( instring, loc, self.errmsg, self) - + def canParseNext(self, instring, loc): try: self.tryParse(instring, loc) @@ -1465,7 +1623,7 @@ class ParserElement(object): def clear(self): cache.clear() - + def cache_len(self): return len(cache) @@ -1577,23 +1735,23 @@ class ParserElement(object): often in many complex grammars) can immediately return a cached value, instead of re-executing parsing/validating code. Memoizing is done of both valid results and parsing exceptions. - + Parameters: - - cache_size_limit - (default=C{128}) - if an integer value is provided - will limit the size of the packrat cache; if None is passed, then - the cache size will be unbounded; if 0 is passed, the cache will - be effectively disabled. - + + - cache_size_limit - (default= ``128``) - if an integer value is provided + will limit the size of the packrat cache; if None is passed, then + the cache size will be unbounded; if 0 is passed, the cache will + be effectively disabled. + This speedup may break existing programs that use parse actions that have side-effects. For this reason, packrat parsing is disabled when you first import pyparsing. To activate the packrat feature, your - program must call the class method C{ParserElement.enablePackrat()}. If - your program uses C{psyco} to "compile as you go", you must call - C{enablePackrat} before calling C{psyco.full()}. If you do not do this, - Python will crash. For best results, call C{enablePackrat()} immediately - after importing pyparsing. - + program must call the class method :class:`ParserElement.enablePackrat`. + For best results, call ``enablePackrat()`` immediately after + importing pyparsing. + Example:: + from pipenv.patched.notpip._vendor import pyparsing pyparsing.ParserElement.enablePackrat() """ @@ -1612,23 +1770,25 @@ class ParserElement(object): expression has been built. If you want the grammar to require that the entire input string be - successfully parsed, then set C{parseAll} to True (equivalent to ending - the grammar with C{L{StringEnd()}}). + successfully parsed, then set ``parseAll`` to True (equivalent to ending + the grammar with ``StringEnd()``). - Note: C{parseString} implicitly calls C{expandtabs()} on the input string, + Note: ``parseString`` implicitly calls ``expandtabs()`` on the input string, in order to report proper column numbers in parse actions. If the input string contains tabs and - the grammar uses parse actions that use the C{loc} argument to index into the + the grammar uses parse actions that use the ``loc`` argument to index into the string being parsed, you can ensure you have a consistent view of the input string by: - - calling C{parseWithTabs} on your grammar before calling C{parseString} - (see L{I{parseWithTabs}}) - - define your parse action using the full C{(s,loc,toks)} signature, and - reference the input string using the parse action's C{s} argument - - explictly expand the tabs in your input string before calling - C{parseString} - + + - calling ``parseWithTabs`` on your grammar before calling ``parseString`` + (see :class:`parseWithTabs`) + - define your parse action using the full ``(s,loc,toks)`` signature, and + reference the input string using the parse action's ``s`` argument + - explictly expand the tabs in your input string before calling + ``parseString`` + Example:: + Word('a').parseString('aaaaabaaa') # -> ['aaaaa'] Word('a').parseString('aaaaabaaa', parseAll=True) # -> Exception: Expected end of text """ @@ -1659,22 +1819,23 @@ class ParserElement(object): """ Scan the input string for expression matches. Each match will return the matching tokens, start location, and end location. May be called with optional - C{maxMatches} argument, to clip scanning after 'n' matches are found. If - C{overlap} is specified, then overlapping matches will be reported. + ``maxMatches`` argument, to clip scanning after 'n' matches are found. If + ``overlap`` is specified, then overlapping matches will be reported. Note that the start and end locations are reported relative to the string - being parsed. See L{I{parseString}} for more information on parsing + being parsed. See :class:`parseString` for more information on parsing strings with embedded tabs. Example:: + source = "sldjf123lsdjjkf345sldkjf879lkjsfd987" print(source) for tokens,start,end in Word(alphas).scanString(source): print(' '*start + '^'*(end-start)) print(' '*start + tokens[0]) - + prints:: - + sldjf123lsdjjkf345sldkjf879lkjsfd987 ^^^^^ sldjf @@ -1728,19 +1889,22 @@ class ParserElement(object): def transformString( self, instring ): """ - Extension to C{L{scanString}}, to modify matching text with modified tokens that may - be returned from a parse action. To use C{transformString}, define a grammar and + Extension to :class:`scanString`, to modify matching text with modified tokens that may + be returned from a parse action. To use ``transformString``, define a grammar and attach a parse action to it that modifies the returned token list. - Invoking C{transformString()} on a target string will then scan for matches, + Invoking ``transformString()`` on a target string will then scan for matches, and replace the matched text patterns according to the logic in the parse - action. C{transformString()} returns the resulting transformed string. - + action. ``transformString()`` returns the resulting transformed string. + Example:: + wd = Word(alphas) wd.setParseAction(lambda toks: toks[0].title()) - + print(wd.transformString("now is the winter of our discontent made glorious summer by this sun of york.")) - Prints:: + + prints:: + Now Is The Winter Of Our Discontent Made Glorious Summer By This Sun Of York. """ out = [] @@ -1771,19 +1935,22 @@ class ParserElement(object): def searchString( self, instring, maxMatches=_MAX_INT ): """ - Another extension to C{L{scanString}}, simplifying the access to the tokens found + Another extension to :class:`scanString`, simplifying the access to the tokens found to match the given parse expression. May be called with optional - C{maxMatches} argument, to clip searching after 'n' matches are found. - + ``maxMatches`` argument, to clip searching after 'n' matches are found. + Example:: + # a capitalized word starts with an uppercase letter, followed by zero or more lowercase letters cap_word = Word(alphas.upper(), alphas.lower()) - + print(cap_word.searchString("More than Iron, more than Lead, more than Gold I need Electricity")) # the sum() builtin can be used to merge results into a single ParseResults object print(sum(cap_word.searchString("More than Iron, more than Lead, more than Gold I need Electricity"))) + prints:: + [['More'], ['Iron'], ['Lead'], ['Gold'], ['I'], ['Electricity']] ['More', 'Iron', 'Lead', 'Gold', 'I', 'Electricity'] """ @@ -1799,14 +1966,17 @@ class ParserElement(object): def split(self, instring, maxsplit=_MAX_INT, includeSeparators=False): """ Generator method to split a string using the given expression as a separator. - May be called with optional C{maxsplit} argument, to limit the number of splits; - and the optional C{includeSeparators} argument (default=C{False}), if the separating + May be called with optional ``maxsplit`` argument, to limit the number of splits; + and the optional ``includeSeparators`` argument (default= ``False``), if the separating matching text should be included in the split results. - - Example:: + + Example:: + punc = oneOf(list(".,;:/-!?")) print(list(punc.split("This, this?, this sentence, is badly punctuated!"))) + prints:: + ['This', ' this', '', ' this sentence', ' is badly punctuated', ''] """ splits = 0 @@ -1820,14 +1990,17 @@ class ParserElement(object): def __add__(self, other ): """ - Implementation of + operator - returns C{L{And}}. Adding strings to a ParserElement - converts them to L{Literal}s by default. - + Implementation of + operator - returns :class:`And`. Adding strings to a ParserElement + converts them to :class:`Literal`s by default. + Example:: + greet = Word(alphas) + "," + Word(alphas) + "!" hello = "Hello, World!" print (hello, "->", greet.parseString(hello)) - Prints:: + + prints:: + Hello, World! -> ['Hello', ',', 'World', '!'] """ if isinstance( other, basestring ): @@ -1840,7 +2013,7 @@ class ParserElement(object): def __radd__(self, other ): """ - Implementation of + operator when left operand is not a C{L{ParserElement}} + Implementation of + operator when left operand is not a :class:`ParserElement` """ if isinstance( other, basestring ): other = ParserElement._literalStringClass( other ) @@ -1852,7 +2025,7 @@ class ParserElement(object): def __sub__(self, other): """ - Implementation of - operator, returns C{L{And}} with error stop + Implementation of - operator, returns :class:`And` with error stop """ if isinstance( other, basestring ): other = ParserElement._literalStringClass( other ) @@ -1864,7 +2037,7 @@ class ParserElement(object): def __rsub__(self, other ): """ - Implementation of - operator when left operand is not a C{L{ParserElement}} + Implementation of - operator when left operand is not a :class:`ParserElement` """ if isinstance( other, basestring ): other = ParserElement._literalStringClass( other ) @@ -1876,23 +2049,23 @@ class ParserElement(object): def __mul__(self,other): """ - Implementation of * operator, allows use of C{expr * 3} in place of - C{expr + expr + expr}. Expressions may also me multiplied by a 2-integer - tuple, similar to C{{min,max}} multipliers in regular expressions. Tuples - may also include C{None} as in: - - C{expr*(n,None)} or C{expr*(n,)} is equivalent - to C{expr*n + L{ZeroOrMore}(expr)} - (read as "at least n instances of C{expr}") - - C{expr*(None,n)} is equivalent to C{expr*(0,n)} - (read as "0 to n instances of C{expr}") - - C{expr*(None,None)} is equivalent to C{L{ZeroOrMore}(expr)} - - C{expr*(1,None)} is equivalent to C{L{OneOrMore}(expr)} + Implementation of * operator, allows use of ``expr * 3`` in place of + ``expr + expr + expr``. Expressions may also me multiplied by a 2-integer + tuple, similar to ``{min,max}`` multipliers in regular expressions. Tuples + may also include ``None`` as in: + - ``expr*(n,None)`` or ``expr*(n,)`` is equivalent + to ``expr*n + ZeroOrMore(expr)`` + (read as "at least n instances of ``expr``") + - ``expr*(None,n)`` is equivalent to ``expr*(0,n)`` + (read as "0 to n instances of ``expr``") + - ``expr*(None,None)`` is equivalent to ``ZeroOrMore(expr)`` + - ``expr*(1,None)`` is equivalent to ``OneOrMore(expr)`` - Note that C{expr*(None,n)} does not raise an exception if + Note that ``expr*(None,n)`` does not raise an exception if more than n exprs exist in the input stream; that is, - C{expr*(None,n)} does not enforce a maximum number of expr + ``expr*(None,n)`` does not enforce a maximum number of expr occurrences. If this behavior is desired, then write - C{expr*(None,n) + ~expr} + ``expr*(None,n) + ~expr`` """ if isinstance(other,int): minElements, optElements = other,0 @@ -1947,7 +2120,7 @@ class ParserElement(object): def __or__(self, other ): """ - Implementation of | operator - returns C{L{MatchFirst}} + Implementation of | operator - returns :class:`MatchFirst` """ if isinstance( other, basestring ): other = ParserElement._literalStringClass( other ) @@ -1959,7 +2132,7 @@ class ParserElement(object): def __ror__(self, other ): """ - Implementation of | operator when left operand is not a C{L{ParserElement}} + Implementation of | operator when left operand is not a :class:`ParserElement` """ if isinstance( other, basestring ): other = ParserElement._literalStringClass( other ) @@ -1971,7 +2144,7 @@ class ParserElement(object): def __xor__(self, other ): """ - Implementation of ^ operator - returns C{L{Or}} + Implementation of ^ operator - returns :class:`Or` """ if isinstance( other, basestring ): other = ParserElement._literalStringClass( other ) @@ -1983,7 +2156,7 @@ class ParserElement(object): def __rxor__(self, other ): """ - Implementation of ^ operator when left operand is not a C{L{ParserElement}} + Implementation of ^ operator when left operand is not a :class:`ParserElement` """ if isinstance( other, basestring ): other = ParserElement._literalStringClass( other ) @@ -1995,7 +2168,7 @@ class ParserElement(object): def __and__(self, other ): """ - Implementation of & operator - returns C{L{Each}} + Implementation of & operator - returns :class:`Each` """ if isinstance( other, basestring ): other = ParserElement._literalStringClass( other ) @@ -2007,7 +2180,7 @@ class ParserElement(object): def __rand__(self, other ): """ - Implementation of & operator when left operand is not a C{L{ParserElement}} + Implementation of & operator when left operand is not a :class:`ParserElement` """ if isinstance( other, basestring ): other = ParserElement._literalStringClass( other ) @@ -2019,23 +2192,24 @@ class ParserElement(object): def __invert__( self ): """ - Implementation of ~ operator - returns C{L{NotAny}} + Implementation of ~ operator - returns :class:`NotAny` """ return NotAny( self ) def __call__(self, name=None): """ - Shortcut for C{L{setResultsName}}, with C{listAllMatches=False}. - - If C{name} is given with a trailing C{'*'} character, then C{listAllMatches} will be - passed as C{True}. - - If C{name} is omitted, same as calling C{L{copy}}. + Shortcut for :class:`setResultsName`, with ``listAllMatches=False``. + + If ``name`` is given with a trailing ``'*'`` character, then ``listAllMatches`` will be + passed as ``True``. + + If ``name` is omitted, same as calling :class:`copy`. Example:: + # these are equivalent userdata = Word(alphas).setResultsName("name") + Word(nums+"-").setResultsName("socsecno") - userdata = Word(alphas)("name") + Word(nums+"-")("socsecno") + userdata = Word(alphas)("name") + Word(nums+"-")("socsecno") """ if name is not None: return self.setResultsName(name) @@ -2044,7 +2218,7 @@ class ParserElement(object): def suppress( self ): """ - Suppresses the output of this C{ParserElement}; useful to keep punctuation from + Suppresses the output of this :class:`ParserElement`; useful to keep punctuation from cluttering up returned output. """ return Suppress( self ) @@ -2052,7 +2226,7 @@ class ParserElement(object): def leaveWhitespace( self ): """ Disables the skipping of whitespace before matching the characters in the - C{ParserElement}'s defined pattern. This is normally only used internally by + :class:`ParserElement`'s defined pattern. This is normally only used internally by the pyparsing module, but may be needed in some whitespace-sensitive grammars. """ self.skipWhitespace = False @@ -2069,9 +2243,9 @@ class ParserElement(object): def parseWithTabs( self ): """ - Overrides default behavior to expand C{}s to spaces before parsing the input string. - Must be called before C{parseString} when the input grammar contains elements that - match C{} characters. + Overrides default behavior to expand ````s to spaces before parsing the input string. + Must be called before ``parseString`` when the input grammar contains elements that + match ```` characters. """ self.keepTabs = True return self @@ -2081,11 +2255,12 @@ class ParserElement(object): Define expression to be ignored (e.g., comments) while doing pattern matching; may be called repeatedly, to define multiple comment or other ignorable patterns. - + Example:: + patt = OneOrMore(Word(alphas)) patt.parseString('ablaj /* comment */ lskjd') # -> ['ablaj'] - + patt.ignore(cStyleComment) patt.parseString('ablaj /* comment */ lskjd') # -> ['ablaj', 'lskjd'] """ @@ -2112,19 +2287,21 @@ class ParserElement(object): def setDebug( self, flag=True ): """ Enable display of debugging messages while doing pattern matching. - Set C{flag} to True to enable, False to disable. + Set ``flag`` to True to enable, False to disable. Example:: + wd = Word(alphas).setName("alphaword") integer = Word(nums).setName("numword") term = wd | integer - + # turn on debugging for wd wd.setDebug() OneOrMore(term).parseString("abc 123 xyz 890") - + prints:: + Match alphaword at loc 0(1,1) Matched alphaword -> ['abc'] Match alphaword at loc 3(1,4) @@ -2137,12 +2314,12 @@ class ParserElement(object): Exception raised:Expected alphaword (at char 15), (line:1, col:16) The output shown is that produced by the default debug actions - custom debug actions can be - specified using L{setDebugActions}. Prior to attempting - to match the C{wd} expression, the debugging message C{"Match at loc (,)"} - is shown. Then if the parse succeeds, a C{"Matched"} message is shown, or an C{"Exception raised"} - message is shown. Also note the use of L{setName} to assign a human-readable name to the expression, + specified using :class:`setDebugActions`. Prior to attempting + to match the ``wd`` expression, the debugging message ``"Match at loc (,)"`` + is shown. Then if the parse succeeds, a ``"Matched"`` message is shown, or an ``"Exception raised"`` + message is shown. Also note the use of :class:`setName` to assign a human-readable name to the expression, which makes debugging and exception messages easier to understand - for instance, the default - name created for the C{Word} expression without calling C{setName} is C{"W:(ABCD...)"}. + name created for the :class:`Word` expression without calling ``setName`` is ``"W:(ABCD...)"``. """ if flag: self.setDebugActions( _defaultStartDebugAction, _defaultSuccessDebugAction, _defaultExceptionDebugAction ) @@ -2212,14 +2389,15 @@ class ParserElement(object): def matches(self, testString, parseAll=True): """ - Method for quick testing of a parser against a test string. Good for simple + Method for quick testing of a parser against a test string. Good for simple inline microtests of sub expressions while building up larger parser. - + Parameters: - testString - to test against this expression for a match - - parseAll - (default=C{True}) - flag to pass to C{L{parseString}} when running tests - + - parseAll - (default= ``True``) - flag to pass to :class:`parseString` when running tests + Example:: + expr = Word(nums) assert expr.matches("100") """ @@ -2228,28 +2406,32 @@ class ParserElement(object): return True except ParseBaseException: return False - - def runTests(self, tests, parseAll=True, comment='#', fullDump=True, printResults=True, failureTests=False): + + def runTests(self, tests, parseAll=True, comment='#', + fullDump=True, printResults=True, failureTests=False, postParse=None): """ Execute the parse expression on a series of test strings, showing each test, the parsed results or where the parse failed. Quick and easy way to run a parse expression against a list of sample strings. - + Parameters: - tests - a list of separate test strings, or a multiline string of test strings - - parseAll - (default=C{True}) - flag to pass to C{L{parseString}} when running tests - - comment - (default=C{'#'}) - expression for indicating embedded comments in the test + - parseAll - (default= ``True``) - flag to pass to :class:`parseString` when running tests + - comment - (default= ``'#'``) - expression for indicating embedded comments in the test string; pass None to disable comment filtering - - fullDump - (default=C{True}) - dump results as list followed by results names in nested outline; + - fullDump - (default= ``True``) - dump results as list followed by results names in nested outline; if False, only dump nested list - - printResults - (default=C{True}) prints test output to stdout - - failureTests - (default=C{False}) indicates if these tests are expected to fail parsing + - printResults - (default= ``True``) prints test output to stdout + - failureTests - (default= ``False``) indicates if these tests are expected to fail parsing + - postParse - (default= ``None``) optional callback for successful parse results; called as + `fn(test_string, parse_results)` and returns a string to be added to the test output Returns: a (success, results) tuple, where success indicates that all tests succeeded - (or failed if C{failureTests} is True), and the results contain a list of lines of each + (or failed if ``failureTests`` is True), and the results contain a list of lines of each test's output - + Example:: + number_expr = pyparsing_common.number.copy() result = number_expr.runTests(''' @@ -2273,7 +2455,9 @@ class ParserElement(object): 3.14.159 ''', failureTests=True) print("Success" if result[0] else "Failed!") + prints:: + # unsigned integer 100 [100] @@ -2291,7 +2475,7 @@ class ParserElement(object): [1e-12] Success - + # stray character 100Z ^ @@ -2313,7 +2497,7 @@ class ParserElement(object): lines, create a test like this:: expr.runTest(r"this is a test\\n of strings that spans \\n 3 lines") - + (Note that this is a raw string literal, you must include the leading 'r'.) """ if isinstance(tests, basestring): @@ -2332,10 +2516,18 @@ class ParserElement(object): out = ['\n'.join(comments), t] comments = [] try: - t = t.replace(r'\n','\n') + # convert newline marks to actual newlines, and strip leading BOM if present + t = t.replace(r'\n','\n').lstrip('\ufeff') result = self.parseString(t, parseAll=parseAll) out.append(result.dump(full=fullDump)) success = success and not failureTests + if postParse is not None: + try: + pp_value = postParse(t, result) + if pp_value is not None: + out.append(str(pp_value)) + except Exception as e: + out.append("{0} failed: {1}: {2}".format(postParse.__name__, type(e).__name__, e)) except ParseBaseException as pe: fatal = "(FATAL)" if isinstance(pe, ParseFatalException) else "" if '\n' in t: @@ -2357,21 +2549,20 @@ class ParserElement(object): print('\n'.join(out)) allResults.append((t, result)) - + return success, allResults - + class Token(ParserElement): - """ - Abstract C{ParserElement} subclass, for defining atomic matching patterns. + """Abstract :class:`ParserElement` subclass, for defining atomic + matching patterns. """ def __init__( self ): super(Token,self).__init__( savelist=False ) class Empty(Token): - """ - An empty token, will always match. + """An empty token, will always match. """ def __init__( self ): super(Empty,self).__init__() @@ -2381,8 +2572,7 @@ class Empty(Token): class NoMatch(Token): - """ - A token that will never match. + """A token that will never match. """ def __init__( self ): super(NoMatch,self).__init__() @@ -2396,18 +2586,18 @@ class NoMatch(Token): class Literal(Token): - """ - Token to exactly match a specified string. - + """Token to exactly match a specified string. + Example:: + Literal('blah').parseString('blah') # -> ['blah'] Literal('blah').parseString('blahfooblah') # -> ['blah'] Literal('blah').parseString('bla') # -> Exception: Expected "blah" - - For case-insensitive matching, use L{CaselessLiteral}. - + + For case-insensitive matching, use :class:`CaselessLiteral`. + For keyword matching (force word break before and after the matched string), - use L{Keyword} or L{CaselessKeyword}. + use :class:`Keyword` or :class:`CaselessKeyword`. """ def __init__( self, matchString ): super(Literal,self).__init__() @@ -2437,21 +2627,29 @@ _L = Literal ParserElement._literalStringClass = Literal class Keyword(Token): - """ - Token to exactly match a specified string as a keyword, that is, it must be - immediately followed by a non-keyword character. Compare with C{L{Literal}}: - - C{Literal("if")} will match the leading C{'if'} in C{'ifAndOnlyIf'}. - - C{Keyword("if")} will not; it will only match the leading C{'if'} in C{'if x=1'}, or C{'if(y==2)'} - Accepts two optional constructor arguments in addition to the keyword string: - - C{identChars} is a string of characters that would be valid identifier characters, - defaulting to all alphanumerics + "_" and "$" - - C{caseless} allows case-insensitive matching, default is C{False}. - + """Token to exactly match a specified string as a keyword, that is, + it must be immediately followed by a non-keyword character. Compare + with :class:`Literal`: + + - ``Literal("if")`` will match the leading ``'if'`` in + ``'ifAndOnlyIf'``. + - ``Keyword("if")`` will not; it will only match the leading + ``'if'`` in ``'if x=1'``, or ``'if(y==2)'`` + + Accepts two optional constructor arguments in addition to the + keyword string: + + - ``identChars`` is a string of characters that would be valid + identifier characters, defaulting to all alphanumerics + "_" and + "$" + - ``caseless`` allows case-insensitive matching, default is ``False``. + Example:: + Keyword("start").parseString("start") # -> ['start'] Keyword("start").parseString("starting") # -> Exception - For case-insensitive matching, use L{CaselessKeyword}. + For case-insensitive matching, use :class:`CaselessKeyword`. """ DEFAULT_KEYWORD_CHARS = alphanums+"_$" @@ -2502,15 +2700,15 @@ class Keyword(Token): Keyword.DEFAULT_KEYWORD_CHARS = chars class CaselessLiteral(Literal): - """ - Token to match a specified string, ignoring case of letters. + """Token to match a specified string, ignoring case of letters. Note: the matched results will always be in the case of the given match string, NOT the case of the input text. Example:: + OneOrMore(CaselessLiteral("CMD")).parseString("cmd CMD Cmd10") # -> ['CMD', 'CMD', 'CMD'] - - (Contrast with example for L{CaselessKeyword}.) + + (Contrast with example for :class:`CaselessKeyword`.) """ def __init__( self, matchString ): super(CaselessLiteral,self).__init__( matchString.upper() ) @@ -2526,36 +2724,39 @@ class CaselessLiteral(Literal): class CaselessKeyword(Keyword): """ - Caseless version of L{Keyword}. + Caseless version of :class:`Keyword`. Example:: + OneOrMore(CaselessKeyword("CMD")).parseString("cmd CMD Cmd10") # -> ['CMD', 'CMD'] - - (Contrast with example for L{CaselessLiteral}.) + + (Contrast with example for :class:`CaselessLiteral`.) """ def __init__( self, matchString, identChars=None ): super(CaselessKeyword,self).__init__( matchString, identChars, caseless=True ) - def parseImpl( self, instring, loc, doActions=True ): - if ( (instring[ loc:loc+self.matchLen ].upper() == self.caselessmatch) and - (loc >= len(instring)-self.matchLen or instring[loc+self.matchLen].upper() not in self.identChars) ): - return loc+self.matchLen, self.match - raise ParseException(instring, loc, self.errmsg, self) - class CloseMatch(Token): - """ - A variation on L{Literal} which matches "close" matches, that is, - strings with at most 'n' mismatching characters. C{CloseMatch} takes parameters: - - C{match_string} - string to be matched - - C{maxMismatches} - (C{default=1}) maximum number of mismatches allowed to count as a match - - The results from a successful parse will contain the matched text from the input string and the following named results: - - C{mismatches} - a list of the positions within the match_string where mismatches were found - - C{original} - the original match_string used to compare against the input string - - If C{mismatches} is an empty list, then the match was an exact match. - + """A variation on :class:`Literal` which matches "close" matches, + that is, strings with at most 'n' mismatching characters. + :class:`CloseMatch` takes parameters: + + - ``match_string`` - string to be matched + - ``maxMismatches`` - (``default=1``) maximum number of + mismatches allowed to count as a match + + The results from a successful parse will contain the matched text + from the input string and the following named results: + + - ``mismatches`` - a list of the positions within the + match_string where mismatches were found + - ``original`` - the original match_string used to compare + against the input string + + If ``mismatches`` is an empty list, then the match was an exact + match. + Example:: + patt = CloseMatch("ATCATCGAATGGA") patt.parseString("ATCATCGAAXGGA") # -> (['ATCATCGAAXGGA'], {'mismatches': [[9]], 'original': ['ATCATCGAATGGA']}) patt.parseString("ATCAXCGAAXGGA") # -> Exception: Expected 'ATCATCGAATGGA' (with up to 1 mismatches) (at char 0), (line:1, col:1) @@ -2604,49 +2805,55 @@ class CloseMatch(Token): class Word(Token): - """ - Token for matching words composed of allowed character sets. - Defined with string containing all allowed initial characters, - an optional string containing allowed body characters (if omitted, + """Token for matching words composed of allowed character sets. + Defined with string containing all allowed initial characters, an + optional string containing allowed body characters (if omitted, defaults to the initial character set), and an optional minimum, - maximum, and/or exact length. The default value for C{min} is 1 (a - minimum value < 1 is not valid); the default values for C{max} and C{exact} - are 0, meaning no maximum or exact length restriction. An optional - C{excludeChars} parameter can list characters that might be found in - the input C{bodyChars} string; useful to define a word of all printables - except for one or two characters, for instance. - - L{srange} is useful for defining custom character set strings for defining - C{Word} expressions, using range notation from regular expression character sets. - - A common mistake is to use C{Word} to match a specific literal string, as in - C{Word("Address")}. Remember that C{Word} uses the string argument to define - I{sets} of matchable characters. This expression would match "Add", "AAA", - "dAred", or any other word made up of the characters 'A', 'd', 'r', 'e', and 's'. - To match an exact literal string, use L{Literal} or L{Keyword}. + maximum, and/or exact length. The default value for ``min`` is + 1 (a minimum value < 1 is not valid); the default values for + ``max`` and ``exact`` are 0, meaning no maximum or exact + length restriction. An optional ``excludeChars`` parameter can + list characters that might be found in the input ``bodyChars`` + string; useful to define a word of all printables except for one or + two characters, for instance. + + :class:`srange` is useful for defining custom character set strings + for defining ``Word`` expressions, using range notation from + regular expression character sets. + + A common mistake is to use :class:`Word` to match a specific literal + string, as in ``Word("Address")``. Remember that :class:`Word` + uses the string argument to define *sets* of matchable characters. + This expression would match "Add", "AAA", "dAred", or any other word + made up of the characters 'A', 'd', 'r', 'e', and 's'. To match an + exact literal string, use :class:`Literal` or :class:`Keyword`. pyparsing includes helper strings for building Words: - - L{alphas} - - L{nums} - - L{alphanums} - - L{hexnums} - - L{alphas8bit} (alphabetic characters in ASCII range 128-255 - accented, tilded, umlauted, etc.) - - L{punc8bit} (non-alphabetic characters in ASCII range 128-255 - currency, symbols, superscripts, diacriticals, etc.) - - L{printables} (any non-whitespace character) + + - :class:`alphas` + - :class:`nums` + - :class:`alphanums` + - :class:`hexnums` + - :class:`alphas8bit` (alphabetic characters in ASCII range 128-255 + - accented, tilded, umlauted, etc.) + - :class:`punc8bit` (non-alphabetic characters in ASCII range + 128-255 - currency, symbols, superscripts, diacriticals, etc.) + - :class:`printables` (any non-whitespace character) Example:: + # a word composed of digits integer = Word(nums) # equivalent to Word("0123456789") or Word(srange("0-9")) - + # a word with a leading capital, and zero or more lowercase capital_word = Word(alphas.upper(), alphas.lower()) # hostnames are alphanumeric, with leading alpha, and '-' hostname = Word(alphas, alphanums+'-') - + # roman numeral (not a strict parser, accepts invalid mix of characters) roman = Word("IVXLCDM") - + # any string of non-whitespace characters, except for ',' csv_value = Word(printables, excludeChars=",") """ @@ -2762,22 +2969,38 @@ class Word(Token): return self.strRepr +class Char(Word): + """A short-cut class for defining ``Word(characters, exact=1)``, + when defining a match of any single character in a string of + characters. + """ + def __init__(self, charset): + super(Char, self).__init__(charset, exact=1) + self.reString = "[%s]" % _escapeRegexRangeChars(self.initCharsOrig) + self.re = re.compile( self.reString ) + + class Regex(Token): - r""" - Token for matching strings that match a given regular expression. - Defined with string specifying the regular expression in a form recognized by the inbuilt Python re module. - If the given regex contains named groups (defined using C{(?P...)}), these will be preserved as - named parse results. + r"""Token for matching strings that match a given regular + expression. Defined with string specifying the regular expression in + a form recognized by the stdlib Python `re module `_. + If the given regex contains named groups (defined using ``(?P...)``), + these will be preserved as named parse results. Example:: + realnum = Regex(r"[+-]?\d+\.\d*") date = Regex(r'(?P\d{4})-(?P\d\d?)-(?P\d\d?)') - # ref: http://stackoverflow.com/questions/267399/how-do-you-match-only-valid-roman-numerals-with-a-regular-expression - roman = Regex(r"M{0,4}(CM|CD|D?C{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3})") + # ref: https://stackoverflow.com/questions/267399/how-do-you-match-only-valid-roman-numerals-with-a-regular-expression + roman = Regex(r"M{0,4}(CM|CD|D?{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3})") """ compiledREtype = type(re.compile("[A-Z]")) - def __init__( self, pattern, flags=0): - """The parameters C{pattern} and C{flags} are passed to the C{re.compile()} function as-is. See the Python C{re} module for an explanation of the acceptable patterns and flags.""" + def __init__( self, pattern, flags=0, asGroupList=False, asMatch=False): + """The parameters ``pattern`` and ``flags`` are passed + to the ``re.compile()`` function as-is. See the Python + `re module `_ module for an + explanation of the acceptable patterns and flags. + """ super(Regex,self).__init__() if isinstance(pattern, basestring): @@ -2801,7 +3024,7 @@ class Regex(Token): self.pattern = \ self.reString = str(pattern) self.flags = flags - + else: raise ValueError("Regex may only be constructed with a string or a compiled RE object") @@ -2809,6 +3032,8 @@ class Regex(Token): self.errmsg = "Expected " + self.name self.mayIndexError = False self.mayReturnEmpty = True + self.asGroupList = asGroupList + self.asMatch = asMatch def parseImpl( self, instring, loc, doActions=True ): result = self.re.match(instring,loc) @@ -2816,11 +3041,16 @@ class Regex(Token): raise ParseException(instring, loc, self.errmsg, self) loc = result.end() - d = result.groupdict() - ret = ParseResults(result.group()) - if d: - for k in d: - ret[k] = d[k] + if self.asMatch: + ret = result + elif self.asGroupList: + ret = result.groups() + else: + ret = ParseResults(result.group()) + d = result.groupdict() + if d: + for k, v in d.items(): + ret[k] = v return loc,ret def __str__( self ): @@ -2834,28 +3064,70 @@ class Regex(Token): return self.strRepr + def sub(self, repl): + """ + Return Regex with an attached parse action to transform the parsed + result as if called using `re.sub(expr, repl, string) `_. + + Example:: + + make_html = Regex(r"(\w+):(.*?):").sub(r"<\1>\2") + print(make_html.transformString("h1:main title:")) + # prints "

main title

" + """ + if self.asGroupList: + warnings.warn("cannot use sub() with Regex(asGroupList=True)", + SyntaxWarning, stacklevel=2) + raise SyntaxError() + + if self.asMatch and callable(repl): + warnings.warn("cannot use sub() with a callable with Regex(asMatch=True)", + SyntaxWarning, stacklevel=2) + raise SyntaxError() + + if self.asMatch: + def pa(tokens): + return tokens[0].expand(repl) + else: + def pa(tokens): + return self.re.sub(repl, tokens[0]) + return self.addParseAction(pa) class QuotedString(Token): r""" Token for matching strings that are delimited by quoting characters. - + Defined with the following parameters: - - quoteChar - string of one or more characters defining the quote delimiting string - - escChar - character to escape quotes, typically backslash (default=C{None}) - - escQuote - special quote sequence to escape an embedded quote string (such as SQL's "" to escape an embedded ") (default=C{None}) - - multiline - boolean indicating whether quotes can span multiple lines (default=C{False}) - - unquoteResults - boolean indicating whether the matched text should be unquoted (default=C{True}) - - endQuoteChar - string of one or more characters defining the end of the quote delimited string (default=C{None} => same as quoteChar) - - convertWhitespaceEscapes - convert escaped whitespace (C{'\t'}, C{'\n'}, etc.) to actual whitespace (default=C{True}) + + - quoteChar - string of one or more characters defining the + quote delimiting string + - escChar - character to escape quotes, typically backslash + (default= ``None`` ) + - escQuote - special quote sequence to escape an embedded quote + string (such as SQL's ``""`` to escape an embedded ``"``) + (default= ``None`` ) + - multiline - boolean indicating whether quotes can span + multiple lines (default= ``False`` ) + - unquoteResults - boolean indicating whether the matched text + should be unquoted (default= ``True`` ) + - endQuoteChar - string of one or more characters defining the + end of the quote delimited string (default= ``None`` => same as + quoteChar) + - convertWhitespaceEscapes - convert escaped whitespace + (``'\t'``, ``'\n'``, etc.) to actual whitespace + (default= ``True`` ) Example:: + qs = QuotedString('"') print(qs.searchString('lsjdf "This is the quote" sldjf')) complex_qs = QuotedString('{{', endQuoteChar='}}') print(complex_qs.searchString('lsjdf {{This is the "quote"}} sldjf')) sql_qs = QuotedString('"', escQuote='""') print(sql_qs.searchString('lsjdf "This is the quote with ""embedded"" quotes" sldjf')) + prints:: + [['This is the quote']] [['This is the "quote"']] [['This is the quote with "embedded" quotes']] @@ -2973,19 +3245,23 @@ class QuotedString(Token): class CharsNotIn(Token): - """ - Token for matching words composed of characters I{not} in a given set (will - include whitespace in matched characters if not listed in the provided exclusion set - see example). - Defined with string containing all disallowed characters, and an optional - minimum, maximum, and/or exact length. The default value for C{min} is 1 (a - minimum value < 1 is not valid); the default values for C{max} and C{exact} - are 0, meaning no maximum or exact length restriction. + """Token for matching words composed of characters *not* in a given + set (will include whitespace in matched characters if not listed in + the provided exclusion set - see example). Defined with string + containing all disallowed characters, and an optional minimum, + maximum, and/or exact length. The default value for ``min`` is + 1 (a minimum value < 1 is not valid); the default values for + ``max`` and ``exact`` are 0, meaning no maximum or exact + length restriction. Example:: + # define a comma-separated-value as anything that is not a ',' csv_value = CharsNotIn(',') print(delimitedList(csv_value).parseString("dkls,lsdkjf,s12 34,@!#,213")) + prints:: + ['dkls', 'lsdkjf', 's12 34', '@!#', '213'] """ def __init__( self, notChars, min=1, max=0, exact=0 ): @@ -2994,7 +3270,9 @@ class CharsNotIn(Token): self.notChars = notChars if min < 1: - raise ValueError("cannot specify a minimum length < 1; use Optional(CharsNotIn()) if zero-length char group is permitted") + raise ValueError( + "cannot specify a minimum length < 1; use " + + "Optional(CharsNotIn()) if zero-length char group is permitted") self.minLen = min @@ -3044,19 +3322,38 @@ class CharsNotIn(Token): return self.strRepr class White(Token): - """ - Special matching class for matching whitespace. Normally, whitespace is ignored - by pyparsing grammars. This class is included when some whitespace structures - are significant. Define with a string containing the whitespace characters to be - matched; default is C{" \\t\\r\\n"}. Also takes optional C{min}, C{max}, and C{exact} arguments, - as defined for the C{L{Word}} class. + """Special matching class for matching whitespace. Normally, + whitespace is ignored by pyparsing grammars. This class is included + when some whitespace structures are significant. Define with + a string containing the whitespace characters to be matched; default + is ``" \\t\\r\\n"``. Also takes optional ``min``, + ``max``, and ``exact`` arguments, as defined for the + :class:`Word` class. """ whiteStrs = { - " " : "", - "\t": "", - "\n": "", - "\r": "", - "\f": "", + ' ' : '', + '\t': '', + '\n': '', + '\r': '', + '\f': '', + 'u\00A0': '', + 'u\1680': '', + 'u\180E': '', + 'u\2000': '', + 'u\2001': '', + 'u\2002': '', + 'u\2003': '', + 'u\2004': '', + 'u\2005': '', + 'u\2006': '', + 'u\2007': '', + 'u\2008': '', + 'u\2009': '', + 'u\200A': '', + 'u\200B': '', + 'u\202F': '', + 'u\205F': '', + 'u\3000': '', } def __init__(self, ws=" \t\r\n", min=1, max=0, exact=0): super(White,self).__init__() @@ -3102,8 +3399,8 @@ class _PositionToken(Token): self.mayIndexError = False class GoToColumn(_PositionToken): - """ - Token to advance to a specific column of input text; useful for tabular report scraping. + """Token to advance to a specific column of input text; useful for + tabular report scraping. """ def __init__( self, colno ): super(GoToColumn,self).__init__() @@ -3128,11 +3425,11 @@ class GoToColumn(_PositionToken): class LineStart(_PositionToken): - """ - Matches if current position is at the beginning of a line within the parse string - + """Matches if current position is at the beginning of a line within + the parse string + Example:: - + test = '''\ AAA this line AAA and this line @@ -3142,10 +3439,11 @@ class LineStart(_PositionToken): for t in (LineStart() + 'AAA' + restOfLine).searchString(test): print(t) - - Prints:: + + prints:: + ['AAA', ' this line'] - ['AAA', ' and this line'] + ['AAA', ' and this line'] """ def __init__( self ): @@ -3158,8 +3456,8 @@ class LineStart(_PositionToken): raise ParseException(instring, loc, self.errmsg, self) class LineEnd(_PositionToken): - """ - Matches if current position is at the end of a line within the parse string + """Matches if current position is at the end of a line within the + parse string """ def __init__( self ): super(LineEnd,self).__init__() @@ -3178,8 +3476,8 @@ class LineEnd(_PositionToken): raise ParseException(instring, loc, self.errmsg, self) class StringStart(_PositionToken): - """ - Matches if current position is at the beginning of the parse string + """Matches if current position is at the beginning of the parse + string """ def __init__( self ): super(StringStart,self).__init__() @@ -3193,8 +3491,7 @@ class StringStart(_PositionToken): return loc, [] class StringEnd(_PositionToken): - """ - Matches if current position is at the end of the parse string + """Matches if current position is at the end of the parse string """ def __init__( self ): super(StringEnd,self).__init__() @@ -3211,12 +3508,13 @@ class StringEnd(_PositionToken): raise ParseException(instring, loc, self.errmsg, self) class WordStart(_PositionToken): - """ - Matches if the current position is at the beginning of a Word, and - is not preceded by any character in a given set of C{wordChars} - (default=C{printables}). To emulate the C{\b} behavior of regular expressions, - use C{WordStart(alphanums)}. C{WordStart} will also match at the beginning of - the string being parsed, or at the beginning of a line. + """Matches if the current position is at the beginning of a Word, + and is not preceded by any character in a given set of + ``wordChars`` (default= ``printables``). To emulate the + ``\b`` behavior of regular expressions, use + ``WordStart(alphanums)``. ``WordStart`` will also match at + the beginning of the string being parsed, or at the beginning of + a line. """ def __init__(self, wordChars = printables): super(WordStart,self).__init__() @@ -3231,12 +3529,12 @@ class WordStart(_PositionToken): return loc, [] class WordEnd(_PositionToken): - """ - Matches if the current position is at the end of a Word, and - is not followed by any character in a given set of C{wordChars} - (default=C{printables}). To emulate the C{\b} behavior of regular expressions, - use C{WordEnd(alphanums)}. C{WordEnd} will also match at the end of - the string being parsed, or at the end of a line. + """Matches if the current position is at the end of a Word, and is + not followed by any character in a given set of ``wordChars`` + (default= ``printables``). To emulate the ``\b`` behavior of + regular expressions, use ``WordEnd(alphanums)``. ``WordEnd`` + will also match at the end of the string being parsed, or at the end + of a line. """ def __init__(self, wordChars = printables): super(WordEnd,self).__init__() @@ -3254,8 +3552,8 @@ class WordEnd(_PositionToken): class ParseExpression(ParserElement): - """ - Abstract subclass of ParserElement, for combining and post-processing parsed tokens. + """Abstract subclass of ParserElement, for combining and + post-processing parsed tokens. """ def __init__( self, exprs, savelist = False ): super(ParseExpression,self).__init__(savelist) @@ -3286,7 +3584,7 @@ class ParseExpression(ParserElement): return self def leaveWhitespace( self ): - """Extends C{leaveWhitespace} defined in base class, and also invokes C{leaveWhitespace} on + """Extends ``leaveWhitespace`` defined in base class, and also invokes ``leaveWhitespace`` on all contained expressions.""" self.skipWhitespace = False self.exprs = [ e.copy() for e in self.exprs ] @@ -3347,7 +3645,7 @@ class ParseExpression(ParserElement): self.mayIndexError |= other.mayIndexError self.errmsg = "Expected " + _ustr(self) - + return self def setResultsName( self, name, listAllMatches=False ): @@ -3359,7 +3657,7 @@ class ParseExpression(ParserElement): for e in self.exprs: e.validate(tmp) self.checkRecursion( [] ) - + def copy(self): ret = super(ParseExpression,self).copy() ret.exprs = [e.copy() for e in self.exprs] @@ -3367,12 +3665,14 @@ class ParseExpression(ParserElement): class And(ParseExpression): """ - Requires all given C{ParseExpression}s to be found in the given order. + Requires all given :class:`ParseExpression` s to be found in the given order. Expressions may be separated by whitespace. - May be constructed using the C{'+'} operator. - May also be constructed using the C{'-'} operator, which will suppress backtracking. + May be constructed using the ``'+'`` operator. + May also be constructed using the ``'-'`` operator, which will + suppress backtracking. Example:: + integer = Word(nums) name_expr = OneOrMore(Word(alphas)) @@ -3394,6 +3694,11 @@ class And(ParseExpression): self.skipWhitespace = self.exprs[0].skipWhitespace self.callPreparse = True + def streamline(self): + super(And, self).streamline() + self.mayReturnEmpty = all(e.mayReturnEmpty for e in self.exprs) + return self + def parseImpl( self, instring, loc, doActions=True ): # pass False as last arg to _parse for first element, since we already # pre-parsed the string as part of our And pre-parsing @@ -3442,17 +3747,20 @@ class And(ParseExpression): class Or(ParseExpression): - """ - Requires that at least one C{ParseExpression} is found. - If two expressions match, the expression that matches the longest string will be used. - May be constructed using the C{'^'} operator. + """Requires that at least one :class:`ParseExpression` is found. If + two expressions match, the expression that matches the longest + string will be used. May be constructed using the ``'^'`` + operator. Example:: + # construct Or using '^' operator - + number = Word(nums) ^ Combine(Word(nums) + '.' + Word(nums)) print(number.searchString("123 3.1416 789")) + prints:: + [['123'], ['3.1416'], ['789']] """ def __init__( self, exprs, savelist = False ): @@ -3462,6 +3770,11 @@ class Or(ParseExpression): else: self.mayReturnEmpty = True + def streamline(self): + super(Or, self).streamline() + self.saveAsList = any(e.saveAsList for e in self.exprs) + return self + def parseImpl( self, instring, loc, doActions=True ): maxExcLoc = -1 maxException = None @@ -3521,14 +3834,14 @@ class Or(ParseExpression): class MatchFirst(ParseExpression): - """ - Requires that at least one C{ParseExpression} is found. - If two expressions match, the first one listed is the one that will match. - May be constructed using the C{'|'} operator. + """Requires that at least one :class:`ParseExpression` is found. If + two expressions match, the first one listed is the one that will + match. May be constructed using the ``'|'`` operator. Example:: + # construct MatchFirst using '|' operator - + # watch the order of expressions to match number = Word(nums) | Combine(Word(nums) + '.' + Word(nums)) print(number.searchString("123 3.1416 789")) # Fail! -> [['123'], ['3'], ['1416'], ['789']] @@ -3541,9 +3854,15 @@ class MatchFirst(ParseExpression): super(MatchFirst,self).__init__(exprs, savelist) if self.exprs: self.mayReturnEmpty = any(e.mayReturnEmpty for e in self.exprs) + # self.saveAsList = any(e.saveAsList for e in self.exprs) else: self.mayReturnEmpty = True + def streamline(self): + super(MatchFirst, self).streamline() + self.saveAsList = any(e.saveAsList for e in self.exprs) + return self + def parseImpl( self, instring, loc, doActions=True ): maxExcLoc = -1 maxException = None @@ -3589,12 +3908,13 @@ class MatchFirst(ParseExpression): class Each(ParseExpression): - """ - Requires all given C{ParseExpression}s to be found, but in any order. - Expressions may be separated by whitespace. - May be constructed using the C{'&'} operator. + """Requires all given :class:`ParseExpression` s to be found, but in + any order. Expressions may be separated by whitespace. + + May be constructed using the ``'&'`` operator. Example:: + color = oneOf("RED ORANGE YELLOW GREEN BLUE PURPLE BLACK WHITE BROWN") shape_type = oneOf("SQUARE CIRCLE TRIANGLE STAR HEXAGON OCTAGON") integer = Word(nums) @@ -3603,7 +3923,7 @@ class Each(ParseExpression): color_attr = "color:" + color("color") size_attr = "size:" + integer("size") - # use Each (using operator '&') to accept attributes in any order + # use Each (using operator '&') to accept attributes in any order # (shape and posn are required, color and size are optional) shape_spec = shape_attr & posn_attr & Optional(color_attr) & Optional(size_attr) @@ -3613,7 +3933,9 @@ class Each(ParseExpression): color:GREEN size:20 shape:TRIANGLE posn:20,40 ''' ) + prints:: + shape: SQUARE color: BLACK posn: 100, 120 ['shape:', 'SQUARE', 'color:', 'BLACK', 'posn:', ['100', ',', '120']] - color: BLACK @@ -3647,6 +3969,12 @@ class Each(ParseExpression): self.mayReturnEmpty = all(e.mayReturnEmpty for e in self.exprs) self.skipWhitespace = True self.initExprGroups = True + self.saveAsList = True + + def streamline(self): + super(Each, self).streamline() + self.mayReturnEmpty = all(e.mayReturnEmpty for e in self.exprs) + return self def parseImpl( self, instring, loc, doActions=True ): if self.initExprGroups: @@ -3713,8 +4041,8 @@ class Each(ParseExpression): class ParseElementEnhance(ParserElement): - """ - Abstract subclass of C{ParserElement}, for combining and post-processing parsed tokens. + """Abstract subclass of :class:`ParserElement`, for combining and + post-processing parsed tokens. """ def __init__( self, expr, savelist=False ): super(ParseElementEnhance,self).__init__(savelist) @@ -3790,20 +4118,25 @@ class ParseElementEnhance(ParserElement): class FollowedBy(ParseElementEnhance): - """ - Lookahead matching of the given parse expression. C{FollowedBy} - does I{not} advance the parsing position within the input string, it only - verifies that the specified parse expression matches at the current - position. C{FollowedBy} always returns a null token list. + """Lookahead matching of the given parse expression. + ``FollowedBy`` does *not* advance the parsing position within + the input string, it only verifies that the specified parse + expression matches at the current position. ``FollowedBy`` + always returns a null token list. If any results names are defined + in the lookahead expression, those *will* be returned for access by + name. Example:: + # use FollowedBy to match a label only if it is followed by a ':' data_word = Word(alphas) label = data_word + FollowedBy(':') attr_expr = Group(label + Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join)) - + OneOrMore(attr_expr).parseString("shape: SQUARE color: BLACK posn: upper left").pprint() + prints:: + [['shape', 'SQUARE'], ['color', 'BLACK'], ['posn', 'upper left']] """ def __init__( self, expr ): @@ -3811,20 +4144,108 @@ class FollowedBy(ParseElementEnhance): self.mayReturnEmpty = True def parseImpl( self, instring, loc, doActions=True ): - self.expr.tryParse( instring, loc ) - return loc, [] + _, ret = self.expr._parse(instring, loc, doActions=doActions) + del ret[:] + return loc, ret + + +class PrecededBy(ParseElementEnhance): + """Lookbehind matching of the given parse expression. + ``PrecededBy`` does not advance the parsing position within the + input string, it only verifies that the specified parse expression + matches prior to the current position. ``PrecededBy`` always + returns a null token list, but if a results name is defined on the + given expression, it is returned. + + Parameters: + + - expr - expression that must match prior to the current parse + location + - retreat - (default= ``None``) - (int) maximum number of characters + to lookbehind prior to the current parse location + + If the lookbehind expression is a string, Literal, Keyword, or + a Word or CharsNotIn with a specified exact or maximum length, then + the retreat parameter is not required. Otherwise, retreat must be + specified to give a maximum number of characters to look back from + the current parse position for a lookbehind match. + + Example:: + + # VB-style variable names with type prefixes + int_var = PrecededBy("#") + pyparsing_common.identifier + str_var = PrecededBy("$") + pyparsing_common.identifier + + """ + def __init__(self, expr, retreat=None): + super(PrecededBy, self).__init__(expr) + self.expr = self.expr().leaveWhitespace() + self.mayReturnEmpty = True + self.mayIndexError = False + self.exact = False + if isinstance(expr, str): + retreat = len(expr) + self.exact = True + elif isinstance(expr, (Literal, Keyword)): + retreat = expr.matchLen + self.exact = True + elif isinstance(expr, (Word, CharsNotIn)) and expr.maxLen != _MAX_INT: + retreat = expr.maxLen + self.exact = True + elif isinstance(expr, _PositionToken): + retreat = 0 + self.exact = True + self.retreat = retreat + self.errmsg = "not preceded by " + str(expr) + self.skipWhitespace = False + + def parseImpl(self, instring, loc=0, doActions=True): + if self.exact: + if loc < self.retreat: + raise ParseException(instring, loc, self.errmsg) + start = loc - self.retreat + _, ret = self.expr._parse(instring, start) + else: + # retreat specified a maximum lookbehind window, iterate + test_expr = self.expr + StringEnd() + instring_slice = instring[:loc] + last_expr = ParseException(instring, loc, self.errmsg) + for offset in range(1, min(loc, self.retreat+1)): + try: + _, ret = test_expr._parse(instring_slice, loc-offset) + except ParseBaseException as pbe: + last_expr = pbe + else: + break + else: + raise last_expr + # return empty list of tokens, but preserve any defined results names + del ret[:] + return loc, ret class NotAny(ParseElementEnhance): - """ - Lookahead to disallow matching with the given parse expression. C{NotAny} - does I{not} advance the parsing position within the input string, it only - verifies that the specified parse expression does I{not} match at the current - position. Also, C{NotAny} does I{not} skip over leading whitespace. C{NotAny} - always returns a null token list. May be constructed using the '~' operator. + """Lookahead to disallow matching with the given parse expression. + ``NotAny`` does *not* advance the parsing position within the + input string, it only verifies that the specified parse expression + does *not* match at the current position. Also, ``NotAny`` does + *not* skip over leading whitespace. ``NotAny`` always returns + a null token list. May be constructed using the '~' operator. Example:: - + + AND, OR, NOT = map(CaselessKeyword, "AND OR NOT".split()) + + # take care not to mistake keywords for identifiers + ident = ~(AND | OR | NOT) + Word(alphas) + boolean_term = Optional(NOT) + ident + + # very crude boolean expression - to support parenthesis groups and + # operation hierarchy, use infixNotation + boolean_expr = boolean_term + ZeroOrMore((AND | OR) + boolean_term) + + # integers that are followed by "." are actually floats + integer = Word(nums) + ~Char(".") """ def __init__( self, expr ): super(NotAny,self).__init__(expr) @@ -3862,7 +4283,7 @@ class _MultipleMatch(ParseElementEnhance): check_ender = self.not_ender is not None if check_ender: try_not_ender = self.not_ender.tryParse - + # must be at least one (but first see if we are the stopOn sentinel; # if so, fail) if check_ender: @@ -3884,18 +4305,18 @@ class _MultipleMatch(ParseElementEnhance): pass return loc, tokens - + class OneOrMore(_MultipleMatch): - """ - Repetition of one or more of the given expression. - + """Repetition of one or more of the given expression. + Parameters: - expr - expression that must match one or more times - - stopOn - (default=C{None}) - expression for a terminating sentinel - (only required if the sentinel would ordinarily match the repetition - expression) + - stopOn - (default= ``None``) - expression for a terminating sentinel + (only required if the sentinel would ordinarily match the repetition + expression) Example:: + data_word = Word(alphas) label = data_word + FollowedBy(':') attr_expr = Group(label + Suppress(':') + OneOrMore(data_word).setParseAction(' '.join)) @@ -3906,7 +4327,7 @@ class OneOrMore(_MultipleMatch): # use stopOn attribute for OneOrMore to avoid reading label string as part of the data attr_expr = Group(label + Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join)) OneOrMore(attr_expr).parseString(text).pprint() # Better -> [['shape', 'SQUARE'], ['posn', 'upper left'], ['color', 'BLACK']] - + # could also be written as (attr_expr * (1,)).parseString(text).pprint() """ @@ -3921,21 +4342,20 @@ class OneOrMore(_MultipleMatch): return self.strRepr class ZeroOrMore(_MultipleMatch): - """ - Optional repetition of zero or more of the given expression. - + """Optional repetition of zero or more of the given expression. + Parameters: - expr - expression that must match zero or more times - - stopOn - (default=C{None}) - expression for a terminating sentinel - (only required if the sentinel would ordinarily match the repetition - expression) + - stopOn - (default= ``None``) - expression for a terminating sentinel + (only required if the sentinel would ordinarily match the repetition + expression) - Example: similar to L{OneOrMore} + Example: similar to :class:`OneOrMore` """ def __init__( self, expr, stopOn=None): super(ZeroOrMore,self).__init__(expr, stopOn=stopOn) self.mayReturnEmpty = True - + def parseImpl( self, instring, loc, doActions=True ): try: return super(ZeroOrMore, self).parseImpl(instring, loc, doActions) @@ -3960,27 +4380,29 @@ class _NullToken(object): _optionalNotMatched = _NullToken() class Optional(ParseElementEnhance): - """ - Optional matching of the given expression. + """Optional matching of the given expression. Parameters: - expr - expression that must match zero or more times - default (optional) - value to be returned if the optional expression is not found. Example:: + # US postal code can be a 5-digit zip, plus optional 4-digit qualifier zip = Combine(Word(nums, exact=5) + Optional('-' + Word(nums, exact=4))) zip.runTests(''' # traditional ZIP code 12345 - + # ZIP+4 form 12101-0001 - + # invalid ZIP 98765- ''') + prints:: + # traditional ZIP code 12345 ['12345'] @@ -4024,20 +4446,21 @@ class Optional(ParseElementEnhance): return self.strRepr class SkipTo(ParseElementEnhance): - """ - Token for skipping over all undefined text until the matched expression is found. + """Token for skipping over all undefined text until the matched + expression is found. Parameters: - expr - target expression marking the end of the data to be skipped - - include - (default=C{False}) if True, the target expression is also parsed + - include - (default= ``False``) if True, the target expression is also parsed (the skipped text and target expression are returned as a 2-element list). - - ignore - (default=C{None}) used to define grammars (typically quoted strings and + - ignore - (default= ``None``) used to define grammars (typically quoted strings and comments) that might contain false matches to the target expression - - failOn - (default=C{None}) define expressions that are not allowed to be - included in the skipped test; if found before the target expression is found, + - failOn - (default= ``None``) define expressions that are not allowed to be + included in the skipped test; if found before the target expression is found, the SkipTo is not a match Example:: + report = ''' Outstanding Issues Report - 1 Jan 2000 @@ -4054,14 +4477,16 @@ class SkipTo(ParseElementEnhance): # - parse action will call token.strip() for each matched token, i.e., the description body string_data = SkipTo(SEP, ignore=quotedString) string_data.setParseAction(tokenMap(str.strip)) - ticket_expr = (integer("issue_num") + SEP - + string_data("sev") + SEP - + string_data("desc") + SEP + ticket_expr = (integer("issue_num") + SEP + + string_data("sev") + SEP + + string_data("desc") + SEP + integer("days_open")) - + for tkt in ticket_expr.searchString(report): print tkt.dump() + prints:: + ['101', 'Critical', 'Intermittent system crash', '6'] - days_open: 6 - desc: Intermittent system crash @@ -4084,7 +4509,7 @@ class SkipTo(ParseElementEnhance): self.mayReturnEmpty = True self.mayIndexError = False self.includeMatch = include - self.asList = False + self.saveAsList = False if isinstance(failOn, basestring): self.failOn = ParserElement._literalStringClass(failOn) else: @@ -4098,14 +4523,14 @@ class SkipTo(ParseElementEnhance): expr_parse = self.expr._parse self_failOn_canParseNext = self.failOn.canParseNext if self.failOn is not None else None self_ignoreExpr_tryParse = self.ignoreExpr.tryParse if self.ignoreExpr is not None else None - + tmploc = loc while tmploc <= instrlen: if self_failOn_canParseNext is not None: # break if failOn expression matches if self_failOn_canParseNext(instring, tmploc): break - + if self_ignoreExpr_tryParse is not None: # advance past ignore expressions while 1: @@ -4113,7 +4538,7 @@ class SkipTo(ParseElementEnhance): tmploc = self_ignoreExpr_tryParse(instring, tmploc) except ParseBaseException: break - + try: expr_parse(instring, tmploc, doActions=False, callPreParse=False) except (ParseException, IndexError): @@ -4131,7 +4556,7 @@ class SkipTo(ParseElementEnhance): loc = tmploc skiptext = instring[startloc:loc] skipresult = ParseResults(skiptext) - + if self.includeMatch: loc, mat = expr_parse(instring,loc,doActions,callPreParse=False) skipresult += mat @@ -4139,23 +4564,31 @@ class SkipTo(ParseElementEnhance): return loc, skipresult class Forward(ParseElementEnhance): - """ - Forward declaration of an expression to be defined later - + """Forward declaration of an expression to be defined later - used for recursive grammars, such as algebraic infix notation. - When the expression is known, it is assigned to the C{Forward} variable using the '<<' operator. + When the expression is known, it is assigned to the ``Forward`` + variable using the '<<' operator. + + Note: take care when assigning to ``Forward`` not to overlook + precedence of operators. - Note: take care when assigning to C{Forward} not to overlook precedence of operators. Specifically, '|' has a lower precedence than '<<', so that:: + fwdExpr << a | b | c + will actually be evaluated as:: + (fwdExpr << a) | b | c + thereby leaving b and c out as parseable alternatives. It is recommended that you - explicitly group the values inserted into the C{Forward}:: + explicitly group the values inserted into the ``Forward``:: + fwdExpr << (a | b | c) + Converting to use the '<<=' operator instead will avoid this problem. - See L{ParseResults.pprint} for an example of a recursive parser created using - C{Forward}. + See :class:`ParseResults.pprint` for an example of a recursive + parser created using ``Forward``. """ def __init__( self, other=None ): super(Forward,self).__init__( other, savelist=False ) @@ -4172,10 +4605,10 @@ class Forward(ParseElementEnhance): self.saveAsList = self.expr.saveAsList self.ignoreExprs.extend(self.expr.ignoreExprs) return self - + def __ilshift__(self, other): return self << other - + def leaveWhitespace( self ): self.skipWhitespace = False return self @@ -4225,19 +4658,20 @@ class _ForwardNoRecurse(Forward): class TokenConverter(ParseElementEnhance): """ - Abstract subclass of C{ParseExpression}, for converting parsed results. + Abstract subclass of :class:`ParseExpression`, for converting parsed results. """ def __init__( self, expr, savelist=False ): super(TokenConverter,self).__init__( expr )#, savelist ) self.saveAsList = False class Combine(TokenConverter): - """ - Converter to concatenate all matching tokens to a single string. - By default, the matching patterns must also be contiguous in the input string; - this can be disabled by specifying C{'adjacent=False'} in the constructor. + """Converter to concatenate all matching tokens to a single string. + By default, the matching patterns must also be contiguous in the + input string; this can be disabled by specifying + ``'adjacent=False'`` in the constructor. Example:: + real = Word(nums) + '.' + Word(nums) print(real.parseString('3.1416')) # -> ['3', '.', '1416'] # will also erroneously match the following @@ -4276,10 +4710,11 @@ class Combine(TokenConverter): return retToks class Group(TokenConverter): - """ - Converter to return the matched tokens as a list - useful for returning tokens of C{L{ZeroOrMore}} and C{L{OneOrMore}} expressions. + """Converter to return the matched tokens as a list - useful for + returning tokens of :class:`ZeroOrMore` and :class:`OneOrMore` expressions. Example:: + ident = Word(alphas) num = Word(nums) term = ident | num @@ -4291,38 +4726,40 @@ class Group(TokenConverter): """ def __init__( self, expr ): super(Group,self).__init__( expr ) - self.saveAsList = True + self.saveAsList = expr.saveAsList def postParse( self, instring, loc, tokenlist ): return [ tokenlist ] class Dict(TokenConverter): - """ - Converter to return a repetitive expression as a list, but also as a dictionary. - Each element can also be referenced using the first token in the expression as its key. - Useful for tabular report scraping when the first column can be used as a item key. + """Converter to return a repetitive expression as a list, but also + as a dictionary. Each element can also be referenced using the first + token in the expression as its key. Useful for tabular report + scraping when the first column can be used as a item key. Example:: + data_word = Word(alphas) label = data_word + FollowedBy(':') attr_expr = Group(label + Suppress(':') + OneOrMore(data_word).setParseAction(' '.join)) text = "shape: SQUARE posn: upper left color: light blue texture: burlap" attr_expr = (label + Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join)) - + # print attributes as plain groups print(OneOrMore(attr_expr).parseString(text).dump()) - + # instead of OneOrMore(expr), parse using Dict(OneOrMore(Group(expr))) - Dict will auto-assign names result = Dict(OneOrMore(Group(attr_expr))).parseString(text) print(result.dump()) - - # access named fields as dict entries, or output as dict - print(result['shape']) - print(result.asDict()) - prints:: - ['shape', 'SQUARE', 'posn', 'upper left', 'color', 'light blue', 'texture', 'burlap'] + # access named fields as dict entries, or output as dict + print(result['shape']) + print(result.asDict()) + + prints:: + + ['shape', 'SQUARE', 'posn', 'upper left', 'color', 'light blue', 'texture', 'burlap'] [['shape', 'SQUARE'], ['posn', 'upper left'], ['color', 'light blue'], ['texture', 'burlap']] - color: light blue - posn: upper left @@ -4330,7 +4767,8 @@ class Dict(TokenConverter): - texture: burlap SQUARE {'color': 'light blue', 'posn': 'upper left', 'texture': 'burlap', 'shape': 'SQUARE'} - See more examples at L{ParseResults} of accessing fields by results name. + + See more examples at :class:`ParseResults` of accessing fields by results name. """ def __init__( self, expr ): super(Dict,self).__init__( expr ) @@ -4362,10 +4800,10 @@ class Dict(TokenConverter): class Suppress(TokenConverter): - """ - Converter for ignoring the results of a parsed expression. + """Converter for ignoring the results of a parsed expression. Example:: + source = "a, b, c,d" wd = Word(alphas) wd_list1 = wd + ZeroOrMore(',' + wd) @@ -4375,10 +4813,13 @@ class Suppress(TokenConverter): # way afterward - use Suppress to keep them out of the parsed output wd_list2 = wd + ZeroOrMore(Suppress(',') + wd) print(wd_list2.parseString(source)) + prints:: + ['a', ',', 'b', ',', 'c', ',', 'd'] ['a', 'b', 'c', 'd'] - (See also L{delimitedList}.) + + (See also :class:`delimitedList`.) """ def postParse( self, instring, loc, tokenlist ): return [] @@ -4388,8 +4829,7 @@ class Suppress(TokenConverter): class OnlyOnce(object): - """ - Wrapper for parse actions, to ensure they are only called once. + """Wrapper for parse actions, to ensure they are only called once. """ def __init__(self, methodCall): self.callable = _trim_arity(methodCall) @@ -4404,13 +4844,15 @@ class OnlyOnce(object): self.called = False def traceParseAction(f): - """ - Decorator for debugging parse actions. - - When the parse action is called, this decorator will print C{">> entering I{method-name}(line:I{current_source_line}, I{parse_location}, I{matched_tokens})".} - When the parse action completes, the decorator will print C{"<<"} followed by the returned value, or any exception that the parse action raised. + """Decorator for debugging parse actions. + + When the parse action is called, this decorator will print + ``">> entering method-name(line:, , )"``. + When the parse action completes, the decorator will print + ``"<<"`` followed by the returned value, or any exception that the parse action raised. Example:: + wd = Word(alphas) @traceParseAction @@ -4419,7 +4861,9 @@ def traceParseAction(f): wds = OneOrMore(wd).setParseAction(remove_duplicate_chars) print(wds.parseString("slkdjs sld sldd sdlf sdljf")) + prints:: + >>entering remove_duplicate_chars(line: 'slkdjs sld sldd sdlf sdljf', 0, (['slkdjs', 'sld', 'sldd', 'sdlf', 'sdljf'], {})) < ['aa', 'bb', 'cc'] delimitedList(Word(hexnums), delim=':', combine=True).parseString("AA:BB:CC:DD:EE") # -> ['AA:BB:CC:DD:EE'] """ @@ -4467,16 +4913,21 @@ def delimitedList( expr, delim=",", combine=False ): return ( expr + ZeroOrMore( Suppress( delim ) + expr ) ).setName(dlName) def countedArray( expr, intExpr=None ): - """ - Helper to define a counted list of expressions. + """Helper to define a counted list of expressions. + This helper defines a pattern of the form:: + integer expr expr expr... + where the leading integer tells how many expr expressions follow. - The matched tokens returns the array of expr tokens as a list - the leading count token is suppressed. - - If C{intExpr} is specified, it should be a pyparsing expression that produces an integer value. + The matched tokens returns the array of expr tokens as a list - the + leading count token is suppressed. + + If ``intExpr`` is specified, it should be a pyparsing expression + that produces an integer value. Example:: + countedArray(Word(alphas)).parseString('2 ab cd ef') # -> ['ab', 'cd'] # in this parser, the leading integer value is given in binary, @@ -4507,17 +4958,19 @@ def _flatten(L): return ret def matchPreviousLiteral(expr): - """ - Helper to define an expression that is indirectly defined from - the tokens matched in a previous expression, that is, it looks - for a 'repeat' of a previous expression. For example:: + """Helper to define an expression that is indirectly defined from + the tokens matched in a previous expression, that is, it looks for + a 'repeat' of a previous expression. For example:: + first = Word(nums) second = matchPreviousLiteral(first) matchExpr = first + ":" + second - will match C{"1:1"}, but not C{"1:2"}. Because this matches a - previous literal, will also match the leading C{"1:1"} in C{"1:10"}. - If this is not desired, use C{matchPreviousExpr}. - Do I{not} use with packrat parsing enabled. + + will match ``"1:1"``, but not ``"1:2"``. Because this + matches a previous literal, will also match the leading + ``"1:1"`` in ``"1:10"``. If this is not desired, use + :class:`matchPreviousExpr`. Do *not* use with packrat parsing + enabled. """ rep = Forward() def copyTokenToRepeater(s,l,t): @@ -4535,18 +4988,19 @@ def matchPreviousLiteral(expr): return rep def matchPreviousExpr(expr): - """ - Helper to define an expression that is indirectly defined from - the tokens matched in a previous expression, that is, it looks - for a 'repeat' of a previous expression. For example:: + """Helper to define an expression that is indirectly defined from + the tokens matched in a previous expression, that is, it looks for + a 'repeat' of a previous expression. For example:: + first = Word(nums) second = matchPreviousExpr(first) matchExpr = first + ":" + second - will match C{"1:1"}, but not C{"1:2"}. Because this matches by - expressions, will I{not} match the leading C{"1:1"} in C{"1:10"}; - the expressions are evaluated first, and then compared, so - C{"1"} is compared with C{"10"}. - Do I{not} use with packrat parsing enabled. + + will match ``"1:1"``, but not ``"1:2"``. Because this + matches by expressions, will *not* match the leading ``"1:1"`` + in ``"1:10"``; the expressions are evaluated first, and then + compared, so ``"1"`` is compared with ``"10"``. Do *not* use + with packrat parsing enabled. """ rep = Forward() e2 = expr.copy() @@ -4571,26 +5025,33 @@ def _escapeRegexRangeChars(s): return _ustr(s) def oneOf( strs, caseless=False, useRegex=True ): - """ - Helper to quickly define a set of alternative Literals, and makes sure to do - longest-first testing when there is a conflict, regardless of the input order, - but returns a C{L{MatchFirst}} for best performance. + """Helper to quickly define a set of alternative Literals, and makes + sure to do longest-first testing when there is a conflict, + regardless of the input order, but returns + a :class:`MatchFirst` for best performance. Parameters: - - strs - a string of space-delimited literals, or a collection of string literals - - caseless - (default=C{False}) - treat all literals as caseless - - useRegex - (default=C{True}) - as an optimization, will generate a Regex - object; otherwise, will generate a C{MatchFirst} object (if C{caseless=True}, or - if creating a C{Regex} raises an exception) + + - strs - a string of space-delimited literals, or a collection of + string literals + - caseless - (default= ``False``) - treat all literals as + caseless + - useRegex - (default= ``True``) - as an optimization, will + generate a Regex object; otherwise, will generate + a :class:`MatchFirst` object (if ``caseless=True``, or if + creating a :class:`Regex` raises an exception) Example:: + comp_oper = oneOf("< = > <= >= !=") var = Word(alphas) number = Word(nums) term = var | number comparison_expr = term + comp_oper + term print(comparison_expr.searchString("B = 12 AA=23 B<=AA AA>12")) + prints:: + [['B', '=', '12'], ['AA', '=', '23'], ['B', '<=', 'AA'], ['AA', '>', '12']] """ if caseless: @@ -4644,19 +5105,21 @@ def oneOf( strs, caseless=False, useRegex=True ): return MatchFirst(parseElementClass(sym) for sym in symbols).setName(' | '.join(symbols)) def dictOf( key, value ): - """ - Helper to easily and clearly define a dictionary by specifying the respective patterns - for the key and value. Takes care of defining the C{L{Dict}}, C{L{ZeroOrMore}}, and C{L{Group}} tokens - in the proper order. The key pattern can include delimiting markers or punctuation, - as long as they are suppressed, thereby leaving the significant key text. The value - pattern can include named results, so that the C{Dict} results can include named token - fields. + """Helper to easily and clearly define a dictionary by specifying + the respective patterns for the key and value. Takes care of + defining the :class:`Dict`, :class:`ZeroOrMore`, and + :class:`Group` tokens in the proper order. The key pattern + can include delimiting markers or punctuation, as long as they are + suppressed, thereby leaving the significant key text. The value + pattern can include named results, so that the :class:`Dict` results + can include named token fields. Example:: + text = "shape: SQUARE posn: upper left color: light blue texture: burlap" attr_expr = (label + Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join)) print(OneOrMore(attr_expr).parseString(text).dump()) - + attr_label = label attr_value = Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join) @@ -4666,7 +5129,9 @@ def dictOf( key, value ): print(result['shape']) print(result.shape) # object attribute access works too print(result.asDict()) + prints:: + [['shape', 'SQUARE'], ['posn', 'upper left'], ['color', 'light blue'], ['texture', 'burlap']] - color: light blue - posn: upper left @@ -4676,29 +5141,34 @@ def dictOf( key, value ): SQUARE {'color': 'light blue', 'shape': 'SQUARE', 'posn': 'upper left', 'texture': 'burlap'} """ - return Dict( ZeroOrMore( Group ( key + value ) ) ) + return Dict(OneOrMore(Group(key + value))) def originalTextFor(expr, asString=True): - """ - Helper to return the original, untokenized text for a given expression. Useful to - restore the parsed fields of an HTML start tag into the raw tag text itself, or to - revert separate tokens with intervening whitespace back to the original matching - input text. By default, returns astring containing the original parsed text. - - If the optional C{asString} argument is passed as C{False}, then the return value is a - C{L{ParseResults}} containing any results names that were originally matched, and a - single token containing the original matched text from the input string. So if - the expression passed to C{L{originalTextFor}} contains expressions with defined - results names, you must set C{asString} to C{False} if you want to preserve those - results name values. + """Helper to return the original, untokenized text for a given + expression. Useful to restore the parsed fields of an HTML start + tag into the raw tag text itself, or to revert separate tokens with + intervening whitespace back to the original matching input text. By + default, returns astring containing the original parsed text. + + If the optional ``asString`` argument is passed as + ``False``, then the return value is + a :class:`ParseResults` containing any results names that + were originally matched, and a single token containing the original + matched text from the input string. So if the expression passed to + :class:`originalTextFor` contains expressions with defined + results names, you must set ``asString`` to ``False`` if you + want to preserve those results name values. Example:: + src = "this is test bold text normal text " for tag in ("b","i"): opener,closer = makeHTMLTags(tag) patt = originalTextFor(opener + SkipTo(closer) + closer) print(patt.searchString(src)[0]) + prints:: + [' bold text '] ['text'] """ @@ -4715,29 +5185,33 @@ def originalTextFor(expr, asString=True): matchExpr.ignoreExprs = expr.ignoreExprs return matchExpr -def ungroup(expr): - """ - Helper to undo pyparsing's default grouping of And expressions, even - if all but one are non-empty. +def ungroup(expr): + """Helper to undo pyparsing's default grouping of And expressions, + even if all but one are non-empty. """ return TokenConverter(expr).setParseAction(lambda t:t[0]) def locatedExpr(expr): - """ - Helper to decorate a returned token with its starting and ending locations in the input string. + """Helper to decorate a returned token with its starting and ending + locations in the input string. + This helper adds the following results names: + - locn_start = location where matched expression begins - locn_end = location where matched expression ends - value = the actual parsed results - Be careful if the input text contains C{} characters, you may want to call - C{L{ParserElement.parseWithTabs}} + Be careful if the input text contains ```` characters, you + may want to call :class:`ParserElement.parseWithTabs` Example:: + wd = Word(alphas) for match in locatedExpr(wd).searchString("ljsdf123lksdjjf123lkkjj1222"): print(match) + prints:: + [[0, 'ljsdf', 5]] [[8, 'lksdjjf', 15]] [[18, 'lkkjj', 23]] @@ -4761,22 +5235,30 @@ _charRange = Group(_singleChar + Suppress("-") + _singleChar) _reBracketExpr = Literal("[") + Optional("^").setResultsName("negate") + Group( OneOrMore( _charRange | _singleChar ) ).setResultsName("body") + "]" def srange(s): - r""" - Helper to easily define string ranges for use in Word construction. Borrows - syntax from regexp '[]' string range definitions:: + r"""Helper to easily define string ranges for use in Word + construction. Borrows syntax from regexp '[]' string range + definitions:: + srange("[0-9]") -> "0123456789" srange("[a-z]") -> "abcdefghijklmnopqrstuvwxyz" srange("[a-z$_]") -> "abcdefghijklmnopqrstuvwxyz$_" - The input string must be enclosed in []'s, and the returned string is the expanded - character set joined into a single string. - The values enclosed in the []'s may be: + + The input string must be enclosed in []'s, and the returned string + is the expanded character set joined into a single string. The + values enclosed in the []'s may be: + - a single character - - an escaped character with a leading backslash (such as C{\-} or C{\]}) - - an escaped hex character with a leading C{'\x'} (C{\x21}, which is a C{'!'} character) - (C{\0x##} is also supported for backwards compatibility) - - an escaped octal character with a leading C{'\0'} (C{\041}, which is a C{'!'} character) - - a range of any of the above, separated by a dash (C{'a-z'}, etc.) - - any combination of the above (C{'aeiouy'}, C{'a-zA-Z0-9_$'}, etc.) + - an escaped character with a leading backslash (such as ``\-`` + or ``\]``) + - an escaped hex character with a leading ``'\x'`` + (``\x21``, which is a ``'!'`` character) (``\0x##`` + is also supported for backwards compatibility) + - an escaped octal character with a leading ``'\0'`` + (``\041``, which is a ``'!'`` character) + - a range of any of the above, separated by a dash (``'a-z'``, + etc.) + - any combination of the above (``'aeiouy'``, + ``'a-zA-Z0-9_$'``, etc.) """ _expanded = lambda p: p if not isinstance(p,ParseResults) else ''.join(unichr(c) for c in range(ord(p[0]),ord(p[1])+1)) try: @@ -4785,9 +5267,8 @@ def srange(s): return "" def matchOnlyAtCol(n): - """ - Helper method for defining parse actions that require matching at a specific - column in the input text. + """Helper method for defining parse actions that require matching at + a specific column in the input text. """ def verifyCol(strg,locn,toks): if col(locn,strg) != n: @@ -4795,24 +5276,26 @@ def matchOnlyAtCol(n): return verifyCol def replaceWith(replStr): - """ - Helper method for common parse actions that simply return a literal value. Especially - useful when used with C{L{transformString}()}. + """Helper method for common parse actions that simply return + a literal value. Especially useful when used with + :class:`transformString` (). Example:: + num = Word(nums).setParseAction(lambda toks: int(toks[0])) na = oneOf("N/A NA").setParseAction(replaceWith(math.nan)) term = na | num - + OneOrMore(term).parseString("324 234 N/A 234") # -> [324, 234, nan, 234] """ return lambda s,l,t: [replStr] def removeQuotes(s,l,t): - """ - Helper parse action for removing quotation marks from parsed quoted strings. + """Helper parse action for removing quotation marks from parsed + quoted strings. Example:: + # by default, quotation marks are included in parsed results quotedString.parseString("'Now is the Winter of our Discontent'") # -> ["'Now is the Winter of our Discontent'"] @@ -4823,18 +5306,20 @@ def removeQuotes(s,l,t): return t[0][1:-1] def tokenMap(func, *args): - """ - Helper to define a parse action by mapping a function to all elements of a ParseResults list.If any additional - args are passed, they are forwarded to the given function as additional arguments after - the token, as in C{hex_integer = Word(hexnums).setParseAction(tokenMap(int, 16))}, which will convert the - parsed data to an integer using base 16. + """Helper to define a parse action by mapping a function to all + elements of a ParseResults list. If any additional args are passed, + they are forwarded to the given function as additional arguments + after the token, as in + ``hex_integer = Word(hexnums).setParseAction(tokenMap(int, 16))``, + which will convert the parsed data to an integer using base 16. + + Example (compare the last to example in :class:`ParserElement.transformString`:: - Example (compare the last to example in L{ParserElement.transformString}:: hex_ints = OneOrMore(Word(hexnums)).setParseAction(tokenMap(int, 16)) hex_ints.runTests(''' 00 11 22 aa FF 0a 0d 1a ''') - + upperword = Word(alphas).setParseAction(tokenMap(str.upper)) OneOrMore(upperword).runTests(''' my kingdom for a horse @@ -4844,7 +5329,9 @@ def tokenMap(func, *args): OneOrMore(wd).setParseAction(' '.join).runTests(''' now is the winter of our discontent made glorious summer by this sun of york ''') + prints:: + 00 11 22 aa FF 0a 0d 1a [0, 17, 34, 170, 255, 10, 13, 26] @@ -4858,7 +5345,7 @@ def tokenMap(func, *args): return [func(tokn, *args) for tokn in t] try: - func_name = getattr(func, '__name__', + func_name = getattr(func, '__name__', getattr(func, '__class__').__name__) except Exception: func_name = str(func) @@ -4867,11 +5354,13 @@ def tokenMap(func, *args): return pa upcaseTokens = tokenMap(lambda t: _ustr(t).upper()) -"""(Deprecated) Helper parse action to convert tokens to upper case. Deprecated in favor of L{pyparsing_common.upcaseTokens}""" +"""(Deprecated) Helper parse action to convert tokens to upper case. +Deprecated in favor of :class:`pyparsing_common.upcaseTokens`""" downcaseTokens = tokenMap(lambda t: _ustr(t).lower()) -"""(Deprecated) Helper parse action to convert tokens to lower case. Deprecated in favor of L{pyparsing_common.downcaseTokens}""" - +"""(Deprecated) Helper parse action to convert tokens to lower case. +Deprecated in favor of :class:`pyparsing_common.downcaseTokens`""" + def _makeTags(tagStr, xml): """Internal helper to construct opening and closing tag expressions, given a tag name""" if isinstance(tagStr,basestring): @@ -4902,55 +5391,63 @@ def _makeTags(tagStr, xml): return openTag, closeTag def makeHTMLTags(tagStr): - """ - Helper to construct opening and closing tag expressions for HTML, given a tag name. Matches - tags in either upper or lower case, attributes with namespaces and with quoted or unquoted values. + """Helper to construct opening and closing tag expressions for HTML, + given a tag name. Matches tags in either upper or lower case, + attributes with namespaces and with quoted or unquoted values. Example:: - text = 'More info at the pyparsing wiki page' - # makeHTMLTags returns pyparsing expressions for the opening and closing tags as a 2-tuple + + text = 'More info at the pyparsing wiki page' + # makeHTMLTags returns pyparsing expressions for the opening and + # closing tags as a 2-tuple a,a_end = makeHTMLTags("A") link_expr = a + SkipTo(a_end)("link_text") + a_end - + for link in link_expr.searchString(text): - # attributes in the tag (like "href" shown here) are also accessible as named results + # attributes in the tag (like "href" shown here) are + # also accessible as named results print(link.link_text, '->', link.href) + prints:: - pyparsing -> http://pyparsing.wikispaces.com + + pyparsing -> https://github.com/pyparsing/pyparsing/wiki """ return _makeTags( tagStr, False ) def makeXMLTags(tagStr): - """ - Helper to construct opening and closing tag expressions for XML, given a tag name. Matches - tags only in the given upper/lower case. + """Helper to construct opening and closing tag expressions for XML, + given a tag name. Matches tags only in the given upper/lower case. - Example: similar to L{makeHTMLTags} + Example: similar to :class:`makeHTMLTags` """ return _makeTags( tagStr, True ) def withAttribute(*args,**attrDict): - """ - Helper to create a validating parse action to be used with start tags created - with C{L{makeXMLTags}} or C{L{makeHTMLTags}}. Use C{withAttribute} to qualify a starting tag - with a required attribute value, to avoid false matches on common tags such as - C{} or C{
}. + """Helper to create a validating parse action to be used with start + tags created with :class:`makeXMLTags` or + :class:`makeHTMLTags`. Use ``withAttribute`` to qualify + a starting tag with a required attribute value, to avoid false + matches on common tags such as ```` or ``
``. - Call C{withAttribute} with a series of attribute names and values. Specify the list - of filter attributes names and values as: - - keyword arguments, as in C{(align="right")}, or - - as an explicit dict with C{**} operator, when an attribute name is also a Python - reserved word, as in C{**{"class":"Customer", "align":"right"}} - - a list of name-value tuples, as in ( ("ns1:class", "Customer"), ("ns2:align","right") ) - For attribute names with a namespace prefix, you must use the second form. Attribute - names are matched insensitive to upper/lower case. - - If just testing for C{class} (with or without a namespace), use C{L{withClass}}. + Call ``withAttribute`` with a series of attribute names and + values. Specify the list of filter attributes names and values as: - To verify that the attribute exists, but without specifying a value, pass - C{withAttribute.ANY_VALUE} as the value. + - keyword arguments, as in ``(align="right")``, or + - as an explicit dict with ``**`` operator, when an attribute + name is also a Python reserved word, as in ``**{"class":"Customer", "align":"right"}`` + - a list of name-value tuples, as in ``(("ns1:class", "Customer"), ("ns2:align","right"))`` + + For attribute names with a namespace prefix, you must use the second + form. Attribute names are matched insensitive to upper/lower case. + + If just testing for ``class`` (with or without a namespace), use + :class:`withClass`. + + To verify that the attribute exists, but without specifying a value, + pass ``withAttribute.ANY_VALUE`` as the value. Example:: + html = '''
Some text @@ -4958,7 +5455,7 @@ def withAttribute(*args,**attrDict):
1,3 2,3 1,1
this has no type
- + ''' div,div_end = makeHTMLTags("div") @@ -4967,13 +5464,15 @@ def withAttribute(*args,**attrDict): grid_expr = div_grid + SkipTo(div | div_end)("body") for grid_header in grid_expr.searchString(html): print(grid_header.body) - + # construct a match with any div tag having a type attribute, regardless of the value div_any_type = div().setParseAction(withAttribute(type=withAttribute.ANY_VALUE)) div_expr = div_any_type + SkipTo(div | div_end)("body") for div_header in div_expr.searchString(html): print(div_header.body) + prints:: + 1 4 0 1 0 1 4 0 1 0 @@ -4995,11 +5494,12 @@ def withAttribute(*args,**attrDict): withAttribute.ANY_VALUE = object() def withClass(classname, namespace=''): - """ - Simplified version of C{L{withAttribute}} when matching on a div class - made - difficult because C{class} is a reserved word in Python. + """Simplified version of :class:`withAttribute` when + matching on a div class - made difficult because ``class`` is + a reserved word in Python. Example:: + html = '''
Some text @@ -5007,84 +5507,96 @@ def withClass(classname, namespace=''):
1,3 2,3 1,1
this <div> has no class
- + ''' div,div_end = makeHTMLTags("div") div_grid = div().setParseAction(withClass("grid")) - + grid_expr = div_grid + SkipTo(div | div_end)("body") for grid_header in grid_expr.searchString(html): print(grid_header.body) - + div_any_type = div().setParseAction(withClass(withAttribute.ANY_VALUE)) div_expr = div_any_type + SkipTo(div | div_end)("body") for div_header in div_expr.searchString(html): print(div_header.body) + prints:: + 1 4 0 1 0 1 4 0 1 0 1,3 2,3 1,1 """ classattr = "%s:class" % namespace if namespace else "class" - return withAttribute(**{classattr : classname}) + return withAttribute(**{classattr : classname}) -opAssoc = _Constants() +opAssoc = SimpleNamespace() opAssoc.LEFT = object() opAssoc.RIGHT = object() def infixNotation( baseExpr, opList, lpar=Suppress('('), rpar=Suppress(')') ): - """ - Helper method for constructing grammars of expressions made up of - operators working in a precedence hierarchy. Operators may be unary or - binary, left- or right-associative. Parse actions can also be attached - to operator expressions. The generated parser will also recognize the use - of parentheses to override operator precedences (see example below). - - Note: if you define a deep operator list, you may see performance issues - when using infixNotation. See L{ParserElement.enablePackrat} for a - mechanism to potentially improve your parser performance. + """Helper method for constructing grammars of expressions made up of + operators working in a precedence hierarchy. Operators may be unary + or binary, left- or right-associative. Parse actions can also be + attached to operator expressions. The generated parser will also + recognize the use of parentheses to override operator precedences + (see example below). + + Note: if you define a deep operator list, you may see performance + issues when using infixNotation. See + :class:`ParserElement.enablePackrat` for a mechanism to potentially + improve your parser performance. Parameters: - - baseExpr - expression representing the most basic element for the nested - - opList - list of tuples, one for each operator precedence level in the - expression grammar; each tuple is of the form - (opExpr, numTerms, rightLeftAssoc, parseAction), where: - - opExpr is the pyparsing expression for the operator; - may also be a string, which will be converted to a Literal; - if numTerms is 3, opExpr is a tuple of two expressions, for the - two operators separating the 3 terms - - numTerms is the number of terms for this operator (must - be 1, 2, or 3) - - rightLeftAssoc is the indicator whether the operator is - right or left associative, using the pyparsing-defined - constants C{opAssoc.RIGHT} and C{opAssoc.LEFT}. + - baseExpr - expression representing the most basic element for the + nested + - opList - list of tuples, one for each operator precedence level + in the expression grammar; each tuple is of the form ``(opExpr, + numTerms, rightLeftAssoc, parseAction)``, where: + + - opExpr is the pyparsing expression for the operator; may also + be a string, which will be converted to a Literal; if numTerms + is 3, opExpr is a tuple of two expressions, for the two + operators separating the 3 terms + - numTerms is the number of terms for this operator (must be 1, + 2, or 3) + - rightLeftAssoc is the indicator whether the operator is right + or left associative, using the pyparsing-defined constants + ``opAssoc.RIGHT`` and ``opAssoc.LEFT``. - parseAction is the parse action to be associated with - expressions matching this operator expression (the - parse action tuple member may be omitted); if the parse action - is passed a tuple or list of functions, this is equivalent to - calling C{setParseAction(*fn)} (L{ParserElement.setParseAction}) - - lpar - expression for matching left-parentheses (default=C{Suppress('(')}) - - rpar - expression for matching right-parentheses (default=C{Suppress(')')}) + expressions matching this operator expression (the parse action + tuple member may be omitted); if the parse action is passed + a tuple or list of functions, this is equivalent to calling + ``setParseAction(*fn)`` + (:class:`ParserElement.setParseAction`) + - lpar - expression for matching left-parentheses + (default= ``Suppress('(')``) + - rpar - expression for matching right-parentheses + (default= ``Suppress(')')``) Example:: - # simple example of four-function arithmetic with ints and variable names + + # simple example of four-function arithmetic with ints and + # variable names integer = pyparsing_common.signed_integer - varname = pyparsing_common.identifier - + varname = pyparsing_common.identifier + arith_expr = infixNotation(integer | varname, [ ('-', 1, opAssoc.RIGHT), (oneOf('* /'), 2, opAssoc.LEFT), (oneOf('+ -'), 2, opAssoc.LEFT), ]) - + arith_expr.runTests(''' 5+3*6 (5+3)*6 -2--11 ''', fullDump=False) + prints:: + 5+3*6 [[5, '+', [3, '*', 6]]] @@ -5094,6 +5606,12 @@ def infixNotation( baseExpr, opList, lpar=Suppress('('), rpar=Suppress(')') ): -2--11 [[['-', 2], '-', ['-', 11]]] """ + # captive version of FollowedBy that does not do parse actions or capture results names + class _FB(FollowedBy): + def parseImpl(self, instring, loc, doActions=True): + self.expr.tryParse(instring, loc) + return loc, [] + ret = Forward() lastExpr = baseExpr | ( lpar + ret + rpar ) for i,operDef in enumerate(opList): @@ -5101,19 +5619,20 @@ def infixNotation( baseExpr, opList, lpar=Suppress('('), rpar=Suppress(')') ): termName = "%s term" % opExpr if arity < 3 else "%s%s term" % opExpr if arity == 3: if opExpr is None or len(opExpr) != 2: - raise ValueError("if numterms=3, opExpr must be a tuple or list of two expressions") + raise ValueError( + "if numterms=3, opExpr must be a tuple or list of two expressions") opExpr1, opExpr2 = opExpr thisExpr = Forward().setName(termName) if rightLeftAssoc == opAssoc.LEFT: if arity == 1: - matchExpr = FollowedBy(lastExpr + opExpr) + Group( lastExpr + OneOrMore( opExpr ) ) + matchExpr = _FB(lastExpr + opExpr) + Group( lastExpr + OneOrMore( opExpr ) ) elif arity == 2: if opExpr is not None: - matchExpr = FollowedBy(lastExpr + opExpr + lastExpr) + Group( lastExpr + OneOrMore( opExpr + lastExpr ) ) + matchExpr = _FB(lastExpr + opExpr + lastExpr) + Group( lastExpr + OneOrMore( opExpr + lastExpr ) ) else: - matchExpr = FollowedBy(lastExpr+lastExpr) + Group( lastExpr + OneOrMore(lastExpr) ) + matchExpr = _FB(lastExpr+lastExpr) + Group( lastExpr + OneOrMore(lastExpr) ) elif arity == 3: - matchExpr = FollowedBy(lastExpr + opExpr1 + lastExpr + opExpr2 + lastExpr) + \ + matchExpr = _FB(lastExpr + opExpr1 + lastExpr + opExpr2 + lastExpr) + \ Group( lastExpr + opExpr1 + lastExpr + opExpr2 + lastExpr ) else: raise ValueError("operator must be unary (1), binary (2), or ternary (3)") @@ -5122,14 +5641,14 @@ def infixNotation( baseExpr, opList, lpar=Suppress('('), rpar=Suppress(')') ): # try to avoid LR with this extra test if not isinstance(opExpr, Optional): opExpr = Optional(opExpr) - matchExpr = FollowedBy(opExpr.expr + thisExpr) + Group( opExpr + thisExpr ) + matchExpr = _FB(opExpr.expr + thisExpr) + Group( opExpr + thisExpr ) elif arity == 2: if opExpr is not None: - matchExpr = FollowedBy(lastExpr + opExpr + thisExpr) + Group( lastExpr + OneOrMore( opExpr + thisExpr ) ) + matchExpr = _FB(lastExpr + opExpr + thisExpr) + Group( lastExpr + OneOrMore( opExpr + thisExpr ) ) else: - matchExpr = FollowedBy(lastExpr + thisExpr) + Group( lastExpr + OneOrMore( thisExpr ) ) + matchExpr = _FB(lastExpr + thisExpr) + Group( lastExpr + OneOrMore( thisExpr ) ) elif arity == 3: - matchExpr = FollowedBy(lastExpr + opExpr1 + thisExpr + opExpr2 + thisExpr) + \ + matchExpr = _FB(lastExpr + opExpr1 + thisExpr + opExpr2 + thisExpr) + \ Group( lastExpr + opExpr1 + thisExpr + opExpr2 + thisExpr ) else: raise ValueError("operator must be unary (1), binary (2), or ternary (3)") @@ -5146,7 +5665,8 @@ def infixNotation( baseExpr, opList, lpar=Suppress('('), rpar=Suppress(')') ): return ret operatorPrecedence = infixNotation -"""(Deprecated) Former name of C{L{infixNotation}}, will be dropped in a future release.""" +"""(Deprecated) Former name of :class:`infixNotation`, will be +dropped in a future release.""" dblQuotedString = Combine(Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*')+'"').setName("string enclosed in double quotes") sglQuotedString = Combine(Regex(r"'(?:[^'\n\r\\]|(?:'')|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*")+"'").setName("string enclosed in single quotes") @@ -5155,28 +5675,33 @@ quotedString = Combine(Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\(?:[^x]|x[0-9a-fA-F]+) unicodeString = Combine(_L('u') + quotedString.copy()).setName("unicode string literal") def nestedExpr(opener="(", closer=")", content=None, ignoreExpr=quotedString.copy()): - """ - Helper method for defining nested lists enclosed in opening and closing - delimiters ("(" and ")" are the default). + """Helper method for defining nested lists enclosed in opening and + closing delimiters ("(" and ")" are the default). Parameters: - - opener - opening character for a nested list (default=C{"("}); can also be a pyparsing expression - - closer - closing character for a nested list (default=C{")"}); can also be a pyparsing expression - - content - expression for items within the nested lists (default=C{None}) - - ignoreExpr - expression for ignoring opening and closing delimiters (default=C{quotedString}) + - opener - opening character for a nested list + (default= ``"("``); can also be a pyparsing expression + - closer - closing character for a nested list + (default= ``")"``); can also be a pyparsing expression + - content - expression for items within the nested lists + (default= ``None``) + - ignoreExpr - expression for ignoring opening and closing + delimiters (default= :class:`quotedString`) - If an expression is not provided for the content argument, the nested - expression will capture all whitespace-delimited content between delimiters - as a list of separate values. + If an expression is not provided for the content argument, the + nested expression will capture all whitespace-delimited content + between delimiters as a list of separate values. - Use the C{ignoreExpr} argument to define expressions that may contain - opening or closing characters that should not be treated as opening - or closing characters for nesting, such as quotedString or a comment - expression. Specify multiple expressions using an C{L{Or}} or C{L{MatchFirst}}. - The default is L{quotedString}, but if no expressions are to be ignored, - then pass C{None} for this argument. + Use the ``ignoreExpr`` argument to define expressions that may + contain opening or closing characters that should not be treated as + opening or closing characters for nesting, such as quotedString or + a comment expression. Specify multiple expressions using an + :class:`Or` or :class:`MatchFirst`. The default is + :class:`quotedString`, but if no expressions are to be ignored, then + pass ``None`` for this argument. Example:: + data_type = oneOf("void int short long char float double") decl_data_type = Combine(data_type + Optional(Word('*'))) ident = Word(alphas+'_', alphanums+'_') @@ -5186,29 +5711,31 @@ def nestedExpr(opener="(", closer=")", content=None, ignoreExpr=quotedString.cop code_body = nestedExpr('{', '}', ignoreExpr=(quotedString | cStyleComment)) - c_function = (decl_data_type("type") + c_function = (decl_data_type("type") + ident("name") - + LPAR + Optional(delimitedList(arg), [])("args") + RPAR + + LPAR + Optional(delimitedList(arg), [])("args") + RPAR + code_body("body")) c_function.ignore(cStyleComment) - + source_code = ''' - int is_odd(int x) { - return (x%2); + int is_odd(int x) { + return (x%2); } - - int dec_to_hex(char hchar) { - if (hchar >= '0' && hchar <= '9') { - return (ord(hchar)-ord('0')); - } else { + + int dec_to_hex(char hchar) { + if (hchar >= '0' && hchar <= '9') { + return (ord(hchar)-ord('0')); + } else { return (10+ord(hchar)-ord('A')); - } + } } ''' for func in c_function.searchString(source_code): print("%(name)s (%(type)s) args: %(args)s" % func) + prints:: + is_odd (int) args: [['int', 'x']] dec_to_hex (int) args: [['char', 'hchar']] """ @@ -5226,7 +5753,7 @@ def nestedExpr(opener="(", closer=")", content=None, ignoreExpr=quotedString.cop ).setParseAction(lambda t:t[0].strip())) else: if ignoreExpr is not None: - content = (Combine(OneOrMore(~ignoreExpr + + content = (Combine(OneOrMore(~ignoreExpr + ~Literal(opener) + ~Literal(closer) + CharsNotIn(ParserElement.DEFAULT_WHITE_CHARS,exact=1)) ).setParseAction(lambda t:t[0].strip())) @@ -5245,23 +5772,24 @@ def nestedExpr(opener="(", closer=")", content=None, ignoreExpr=quotedString.cop return ret def indentedBlock(blockStatementExpr, indentStack, indent=True): - """ - Helper method for defining space-delimited indentation blocks, such as - those used to define block statements in Python source code. + """Helper method for defining space-delimited indentation blocks, + such as those used to define block statements in Python source code. Parameters: - - blockStatementExpr - expression defining syntax of statement that - is repeated within the indented block - - indentStack - list created by caller to manage indentation stack - (multiple statementWithIndentedBlock expressions within a single grammar - should share a common indentStack) - - indent - boolean indicating whether block must be indented beyond the - the current level; set to False for block of left-most statements - (default=C{True}) - A valid block must contain at least one C{blockStatement}. + - blockStatementExpr - expression defining syntax of statement that + is repeated within the indented block + - indentStack - list created by caller to manage indentation stack + (multiple statementWithIndentedBlock expressions within a single + grammar should share a common indentStack) + - indent - boolean indicating whether block must be indented beyond + the the current level; set to False for block of left-most + statements (default= ``True``) + + A valid block must contain at least one ``blockStatement``. Example:: + data = ''' def A(z): A1 @@ -5302,7 +5830,9 @@ def indentedBlock(blockStatementExpr, indentStack, indent=True): parseTree = module_body.parseString(data) parseTree.pprint() + prints:: + [['def', 'A', ['(', 'z', ')'], @@ -5320,7 +5850,7 @@ def indentedBlock(blockStatementExpr, indentStack, indent=True): 'spam', ['(', 'x', 'y', ')'], ':', - [[['def', 'eggs', ['(', 'z', ')'], ':', [['pass']]]]]]] + [[['def', 'eggs', ['(', 'z', ')'], ':', [['pass']]]]]]] """ def checkPeerIndent(s,l,t): if l >= len(s): return @@ -5370,51 +5900,61 @@ def replaceHTMLEntity(t): # it's easy to get these comment structures wrong - they're very common, so may as well make them available cStyleComment = Combine(Regex(r"/\*(?:[^*]|\*(?!/))*") + '*/').setName("C style comment") -"Comment of the form C{/* ... */}" +"Comment of the form ``/* ... */``" htmlComment = Regex(r"").setName("HTML comment") -"Comment of the form C{}" +"Comment of the form ````" restOfLine = Regex(r".*").leaveWhitespace().setName("rest of line") dblSlashComment = Regex(r"//(?:\\\n|[^\n])*").setName("// comment") -"Comment of the form C{// ... (to end of line)}" +"Comment of the form ``// ... (to end of line)``" cppStyleComment = Combine(Regex(r"/\*(?:[^*]|\*(?!/))*") + '*/'| dblSlashComment).setName("C++ style comment") -"Comment of either form C{L{cStyleComment}} or C{L{dblSlashComment}}" +"Comment of either form :class:`cStyleComment` or :class:`dblSlashComment`" javaStyleComment = cppStyleComment -"Same as C{L{cppStyleComment}}" +"Same as :class:`cppStyleComment`" pythonStyleComment = Regex(r"#.*").setName("Python style comment") -"Comment of the form C{# ... (to end of line)}" +"Comment of the form ``# ... (to end of line)``" _commasepitem = Combine(OneOrMore(Word(printables, excludeChars=',') + Optional( Word(" \t") + ~Literal(",") + ~LineEnd() ) ) ).streamline().setName("commaItem") commaSeparatedList = delimitedList( Optional( quotedString.copy() | _commasepitem, default="") ).setName("commaSeparatedList") -"""(Deprecated) Predefined expression of 1 or more printable words or quoted strings, separated by commas. - This expression is deprecated in favor of L{pyparsing_common.comma_separated_list}.""" +"""(Deprecated) Predefined expression of 1 or more printable words or +quoted strings, separated by commas. + +This expression is deprecated in favor of :class:`pyparsing_common.comma_separated_list`. +""" # some other useful expressions - using lower-case class name since we are really using this as a namespace class pyparsing_common: - """ - Here are some common low-level expressions that may be useful in jump-starting parser development: - - numeric forms (L{integers}, L{reals}, L{scientific notation}) - - common L{programming identifiers} - - network addresses (L{MAC}, L{IPv4}, L{IPv6}) - - ISO8601 L{dates} and L{datetime} - - L{UUID} - - L{comma-separated list} + """Here are some common low-level expressions that may be useful in + jump-starting parser development: + + - numeric forms (:class:`integers`, :class:`reals`, + :class:`scientific notation`) + - common :class:`programming identifiers` + - network addresses (:class:`MAC`, + :class:`IPv4`, :class:`IPv6`) + - ISO8601 :class:`dates` and + :class:`datetime` + - :class:`UUID` + - :class:`comma-separated list` + Parse actions: - - C{L{convertToInteger}} - - C{L{convertToFloat}} - - C{L{convertToDate}} - - C{L{convertToDatetime}} - - C{L{stripHTMLTags}} - - C{L{upcaseTokens}} - - C{L{downcaseTokens}} + + - :class:`convertToInteger` + - :class:`convertToFloat` + - :class:`convertToDate` + - :class:`convertToDatetime` + - :class:`stripHTMLTags` + - :class:`upcaseTokens` + - :class:`downcaseTokens` Example:: + pyparsing_common.number.runTests(''' # any int or real number, returned as the appropriate type 100 @@ -5461,7 +6001,9 @@ class pyparsing_common: # uuid 12345678-1234-5678-1234-567812345678 ''') + prints:: + # any int or real number, returned as the appropriate type 100 [100] @@ -5563,7 +6105,8 @@ class pyparsing_common: """expression that parses a floating point number and returns a float""" sci_real = Regex(r'[+-]?\d+([eE][+-]?\d+|\.\d*([eE][+-]?\d+)?)').setName("real number with scientific notation").setParseAction(convertToFloat) - """expression that parses a floating point number with optional scientific notation and returns a float""" + """expression that parses a floating point number with optional + scientific notation and returns a float""" # streamlining this expression makes the docs nicer-looking number = (sci_real | real | signed_integer).streamline() @@ -5571,12 +6114,12 @@ class pyparsing_common: fnumber = Regex(r'[+-]?\d+\.?\d*([eE][+-]?\d+)?').setName("fnumber").setParseAction(convertToFloat) """any int or real number, returned as float""" - + identifier = Word(alphas+'_', alphanums+'_').setName("identifier") """typical code identifier (leading alpha or '_', followed by 0 or more alphas, nums, or '_')""" - + ipv4_address = Regex(r'(25[0-5]|2[0-4][0-9]|1?[0-9]{1,2})(\.(25[0-5]|2[0-4][0-9]|1?[0-9]{1,2})){3}').setName("IPv4 address") - "IPv4 address (C{0.0.0.0 - 255.255.255.255})" + "IPv4 address (``0.0.0.0 - 255.255.255.255``)" _ipv6_part = Regex(r'[0-9a-fA-F]{1,4}').setName("hex_integer") _full_ipv6_address = (_ipv6_part + (':' + _ipv6_part)*7).setName("full IPv6 address") @@ -5585,7 +6128,7 @@ class pyparsing_common: _mixed_ipv6_address = ("::ffff:" + ipv4_address).setName("mixed IPv6 address") ipv6_address = Combine((_full_ipv6_address | _mixed_ipv6_address | _short_ipv6_address).setName("IPv6 address")).setName("IPv6 address") "IPv6 address (long, short, or mixed form)" - + mac_address = Regex(r'[0-9a-fA-F]{2}([:.-])[0-9a-fA-F]{2}(?:\1[0-9a-fA-F]{2}){4}').setName("MAC address") "MAC address xx:xx:xx:xx:xx (may also have '-' or '.' delimiters)" @@ -5595,13 +6138,16 @@ class pyparsing_common: Helper to create a parse action for converting parsed date string to Python datetime.date Params - - - fmt - format to be passed to datetime.strptime (default=C{"%Y-%m-%d"}) + - fmt - format to be passed to datetime.strptime (default= ``"%Y-%m-%d"``) Example:: + date_expr = pyparsing_common.iso8601_date.copy() date_expr.setParseAction(pyparsing_common.convertToDate()) print(date_expr.parseString("1999-12-31")) + prints:: + [datetime.date(1999, 12, 31)] """ def cvt_fn(s,l,t): @@ -5613,17 +6159,20 @@ class pyparsing_common: @staticmethod def convertToDatetime(fmt="%Y-%m-%dT%H:%M:%S.%f"): - """ - Helper to create a parse action for converting parsed datetime string to Python datetime.datetime + """Helper to create a parse action for converting parsed + datetime string to Python datetime.datetime Params - - - fmt - format to be passed to datetime.strptime (default=C{"%Y-%m-%dT%H:%M:%S.%f"}) + - fmt - format to be passed to datetime.strptime (default= ``"%Y-%m-%dT%H:%M:%S.%f"``) Example:: + dt_expr = pyparsing_common.iso8601_datetime.copy() dt_expr.setParseAction(pyparsing_common.convertToDatetime()) print(dt_expr.parseString("1999-12-31T23:59:59.999")) + prints:: + [datetime.datetime(1999, 12, 31, 23, 59, 59, 999000)] """ def cvt_fn(s,l,t): @@ -5634,31 +6183,34 @@ class pyparsing_common: return cvt_fn iso8601_date = Regex(r'(?P\d{4})(?:-(?P\d\d)(?:-(?P\d\d))?)?').setName("ISO8601 date") - "ISO8601 date (C{yyyy-mm-dd})" + "ISO8601 date (``yyyy-mm-dd``)" iso8601_datetime = Regex(r'(?P\d{4})-(?P\d\d)-(?P\d\d)[T ](?P\d\d):(?P\d\d)(:(?P\d\d(\.\d*)?)?)?(?PZ|[+-]\d\d:?\d\d)?').setName("ISO8601 datetime") - "ISO8601 datetime (C{yyyy-mm-ddThh:mm:ss.s(Z|+-00:00)}) - trailing seconds, milliseconds, and timezone optional; accepts separating C{'T'} or C{' '}" + "ISO8601 datetime (``yyyy-mm-ddThh:mm:ss.s(Z|+-00:00)``) - trailing seconds, milliseconds, and timezone optional; accepts separating ``'T'`` or ``' '``" uuid = Regex(r'[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}').setName("UUID") - "UUID (C{xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx})" + "UUID (``xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx``)" _html_stripper = anyOpenTag.suppress() | anyCloseTag.suppress() @staticmethod def stripHTMLTags(s, l, tokens): - """ - Parse action to remove HTML tags from web page HTML source + """Parse action to remove HTML tags from web page HTML source Example:: - # strip HTML links from normal text - text = 'More info at the pyparsing wiki page' + + # strip HTML links from normal text + text = 'More info at the pyparsing wiki page' td,td_end = makeHTMLTags("TD") table_text = td + SkipTo(td_end).setParseAction(pyparsing_common.stripHTMLTags)("body") + td_end - - print(table_text.parseString(text).body) # -> 'More info at the pyparsing wiki page' + print(table_text.parseString(text).body) + + Prints:: + + More info at the pyparsing wiki page """ return pyparsing_common._html_stripper.transformString(tokens[0]) - _commasepitem = Combine(OneOrMore(~Literal(",") + ~LineEnd() + Word(printables, excludeChars=',') + _commasepitem = Combine(OneOrMore(~Literal(",") + ~LineEnd() + Word(printables, excludeChars=',') + Optional( White(" \t") ) ) ).streamline().setName("commaItem") comma_separated_list = delimitedList( Optional( quotedString.copy() | _commasepitem, default="") ).setName("comma separated list") """Predefined expression of 1 or more printable words or quoted strings, separated by commas.""" @@ -5670,6 +6222,164 @@ class pyparsing_common: """Parse action to convert tokens to lower case.""" +class _lazyclassproperty(object): + def __init__(self, fn): + self.fn = fn + self.__doc__ = fn.__doc__ + self.__name__ = fn.__name__ + + def __get__(self, obj, cls): + if cls is None: + cls = type(obj) + if not hasattr(cls, '_intern') or any(cls._intern is getattr(superclass, '_intern', []) for superclass in cls.__mro__[1:]): + cls._intern = {} + attrname = self.fn.__name__ + if attrname not in cls._intern: + cls._intern[attrname] = self.fn(cls) + return cls._intern[attrname] + + +class unicode_set(object): + """ + A set of Unicode characters, for language-specific strings for + ``alphas``, ``nums``, ``alphanums``, and ``printables``. + A unicode_set is defined by a list of ranges in the Unicode character + set, in a class attribute ``_ranges``, such as:: + + _ranges = [(0x0020, 0x007e), (0x00a0, 0x00ff),] + + A unicode set can also be defined using multiple inheritance of other unicode sets:: + + class CJK(Chinese, Japanese, Korean): + pass + """ + _ranges = [] + + @classmethod + def _get_chars_for_ranges(cls): + ret = [] + for cc in cls.__mro__: + if cc is unicode_set: + break + for rr in cc._ranges: + ret.extend(range(rr[0], rr[-1]+1)) + return [unichr(c) for c in sorted(set(ret))] + + @_lazyclassproperty + def printables(cls): + "all non-whitespace characters in this range" + return u''.join(filterfalse(unicode.isspace, cls._get_chars_for_ranges())) + + @_lazyclassproperty + def alphas(cls): + "all alphabetic characters in this range" + return u''.join(filter(unicode.isalpha, cls._get_chars_for_ranges())) + + @_lazyclassproperty + def nums(cls): + "all numeric digit characters in this range" + return u''.join(filter(unicode.isdigit, cls._get_chars_for_ranges())) + + @_lazyclassproperty + def alphanums(cls): + "all alphanumeric characters in this range" + return cls.alphas + cls.nums + + +class pyparsing_unicode(unicode_set): + """ + A namespace class for defining common language unicode_sets. + """ + _ranges = [(32, sys.maxunicode)] + + class Latin1(unicode_set): + "Unicode set for Latin-1 Unicode Character Range" + _ranges = [(0x0020, 0x007e), (0x00a0, 0x00ff),] + + class LatinA(unicode_set): + "Unicode set for Latin-A Unicode Character Range" + _ranges = [(0x0100, 0x017f),] + + class LatinB(unicode_set): + "Unicode set for Latin-B Unicode Character Range" + _ranges = [(0x0180, 0x024f),] + + class Greek(unicode_set): + "Unicode set for Greek Unicode Character Ranges" + _ranges = [ + (0x0370, 0x03ff), (0x1f00, 0x1f15), (0x1f18, 0x1f1d), (0x1f20, 0x1f45), (0x1f48, 0x1f4d), + (0x1f50, 0x1f57), (0x1f59,), (0x1f5b,), (0x1f5d,), (0x1f5f, 0x1f7d), (0x1f80, 0x1fb4), (0x1fb6, 0x1fc4), + (0x1fc6, 0x1fd3), (0x1fd6, 0x1fdb), (0x1fdd, 0x1fef), (0x1ff2, 0x1ff4), (0x1ff6, 0x1ffe), + ] + + class Cyrillic(unicode_set): + "Unicode set for Cyrillic Unicode Character Range" + _ranges = [(0x0400, 0x04ff)] + + class Chinese(unicode_set): + "Unicode set for Chinese Unicode Character Range" + _ranges = [(0x4e00, 0x9fff), (0x3000, 0x303f), ] + + class Japanese(unicode_set): + "Unicode set for Japanese Unicode Character Range, combining Kanji, Hiragana, and Katakana ranges" + _ranges = [ ] + + class Kanji(unicode_set): + "Unicode set for Kanji Unicode Character Range" + _ranges = [(0x4E00, 0x9Fbf), (0x3000, 0x303f), ] + + class Hiragana(unicode_set): + "Unicode set for Hiragana Unicode Character Range" + _ranges = [(0x3040, 0x309f), ] + + class Katakana(unicode_set): + "Unicode set for Katakana Unicode Character Range" + _ranges = [(0x30a0, 0x30ff), ] + + class Korean(unicode_set): + "Unicode set for Korean Unicode Character Range" + _ranges = [(0xac00, 0xd7af), (0x1100, 0x11ff), (0x3130, 0x318f), (0xa960, 0xa97f), (0xd7b0, 0xd7ff), (0x3000, 0x303f), ] + + class CJK(Chinese, Japanese, Korean): + "Unicode set for combined Chinese, Japanese, and Korean (CJK) Unicode Character Range" + pass + + class Thai(unicode_set): + "Unicode set for Thai Unicode Character Range" + _ranges = [(0x0e01, 0x0e3a), (0x0e3f, 0x0e5b), ] + + class Arabic(unicode_set): + "Unicode set for Arabic Unicode Character Range" + _ranges = [(0x0600, 0x061b), (0x061e, 0x06ff), (0x0700, 0x077f), ] + + class Hebrew(unicode_set): + "Unicode set for Hebrew Unicode Character Range" + _ranges = [(0x0590, 0x05ff), ] + + class Devanagari(unicode_set): + "Unicode set for Devanagari Unicode Character Range" + _ranges = [(0x0900, 0x097f), (0xa8e0, 0xa8ff)] + +pyparsing_unicode.Japanese._ranges = (pyparsing_unicode.Japanese.Kanji._ranges + + pyparsing_unicode.Japanese.Hiragana._ranges + + pyparsing_unicode.Japanese.Katakana._ranges) + +# define ranges in language character sets +if PY_3: + setattr(pyparsing_unicode, "العربية", pyparsing_unicode.Arabic) + setattr(pyparsing_unicode, "中文", pyparsing_unicode.Chinese) + setattr(pyparsing_unicode, "кириллица", pyparsing_unicode.Cyrillic) + setattr(pyparsing_unicode, "Ελληνικά", pyparsing_unicode.Greek) + setattr(pyparsing_unicode, "עִברִית", pyparsing_unicode.Hebrew) + setattr(pyparsing_unicode, "日本語", pyparsing_unicode.Japanese) + setattr(pyparsing_unicode.Japanese, "漢字", pyparsing_unicode.Japanese.Kanji) + setattr(pyparsing_unicode.Japanese, "カタカナ", pyparsing_unicode.Japanese.Katakana) + setattr(pyparsing_unicode.Japanese, "ひらがな", pyparsing_unicode.Japanese.Hiragana) + setattr(pyparsing_unicode, "한국어", pyparsing_unicode.Korean) + setattr(pyparsing_unicode, "ไทย", pyparsing_unicode.Thai) + setattr(pyparsing_unicode, "देवनागरी", pyparsing_unicode.Devanagari) + + if __name__ == "__main__": selectToken = CaselessLiteral("select") @@ -5683,7 +6393,7 @@ if __name__ == "__main__": tableName = delimitedList(ident, ".", combine=True).setParseAction(upcaseTokens) tableNameList = Group(delimitedList(tableName)).setName("tables") - + simpleSQL = selectToken("command") + columnSpec("columns") + fromToken + tableNameList("tables") # demo runTests method, including embedded comments in test string diff --git a/pipenv/patched/notpip/_vendor/pytoml/__init__.py b/pipenv/patched/notpip/_vendor/pytoml/__init__.py index 8dc73155..8ed060ff 100644 --- a/pipenv/patched/notpip/_vendor/pytoml/__init__.py +++ b/pipenv/patched/notpip/_vendor/pytoml/__init__.py @@ -1,3 +1,4 @@ from .core import TomlError from .parser import load, loads -from .writer import dump, dumps +from .test import translate_to_test +from .writer import dump, dumps \ No newline at end of file diff --git a/pipenv/patched/notpip/_vendor/pytoml/parser.py b/pipenv/patched/notpip/_vendor/pytoml/parser.py index 9f94e923..3493aa64 100644 --- a/pipenv/patched/notpip/_vendor/pytoml/parser.py +++ b/pipenv/patched/notpip/_vendor/pytoml/parser.py @@ -1,5 +1,6 @@ import string, re, sys, datetime from .core import TomlError +from .utils import rfc3339_re, parse_rfc3339_re if sys.version_info[0] == 2: _chr = unichr @@ -179,13 +180,13 @@ _ws_re = re.compile(r'[ \t]*') def _p_ws(s): s.expect_re(_ws_re) -_escapes = { 'b': '\b', 'n': '\n', 'r': '\r', 't': '\t', '"': '"', '\'': '\'', - '\\': '\\', '/': '/', 'f': '\f' } +_escapes = { 'b': '\b', 'n': '\n', 'r': '\r', 't': '\t', '"': '"', + '\\': '\\', 'f': '\f' } _basicstr_re = re.compile(r'[^"\\\000-\037]*') _short_uni_re = re.compile(r'u([0-9a-fA-F]{4})') _long_uni_re = re.compile(r'U([0-9a-fA-F]{8})') -_escapes_re = re.compile('[bnrt"\'\\\\/f]') +_escapes_re = re.compile(r'[btnfr\"\\]') _newline_esc_re = re.compile('\n[ \t\n]*') def _p_basicstr_content(s, content=_basicstr_re): res = [] @@ -196,7 +197,10 @@ def _p_basicstr_content(s, content=_basicstr_re): if s.consume_re(_newline_esc_re): pass elif s.consume_re(_short_uni_re) or s.consume_re(_long_uni_re): - res.append(_chr(int(s.last().group(1), 16))) + v = int(s.last().group(1), 16) + if 0xd800 <= v < 0xe000: + s.fail() + res.append(_chr(v)) else: s.expect_re(_escapes_re) res.append(_escapes[s.last().group(0)]) @@ -220,9 +224,8 @@ def _p_key(s): return s.expect_re(_key_re).group(0) _float_re = re.compile(r'[+-]?(?:0|[1-9](?:_?\d)*)(?:\.\d(?:_?\d)*)?(?:[eE][+-]?(?:\d(?:_?\d)*))?') -_datetime_re = re.compile(r'(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(\.\d+)?(?:Z|([+-]\d{2}):(\d{2}))') -_basicstr_ml_re = re.compile(r'(?:(?:|"|"")[^"\\\000-\011\013-\037])*') +_basicstr_ml_re = re.compile(r'(?:""?(?!")|[^"\\\000-\011\013-\037])*') _litstr_re = re.compile(r"[^'\000\010\012-\037]*") _litstr_ml_re = re.compile(r"(?:(?:|'|'')(?:[^'\000-\010\013-\037]))*") def _p_value(s, object_pairs_hook): @@ -251,24 +254,9 @@ def _p_value(s, object_pairs_hook): s.expect('\'') return 'str', r, r, pos - if s.consume_re(_datetime_re): + if s.consume_re(rfc3339_re): m = s.last() - s0 = m.group(0) - r = map(int, m.groups()[:6]) - if m.group(7): - micro = float(m.group(7)) - else: - micro = 0 - - if m.group(8): - g = int(m.group(8), 10) * 60 + int(m.group(9), 10) - tz = _TimeZone(datetime.timedelta(0, g * 60)) - else: - tz = _TimeZone(datetime.timedelta(0, 0)) - - y, m, d, H, M, S = r - dt = datetime.datetime(y, m, d, H, M, S, int(micro * 1000000), tz) - return 'datetime', s0, dt, pos + return 'datetime', m.group(0), parse_rfc3339_re(m), pos if s.consume_re(_float_re): m = s.last().group(0) @@ -351,24 +339,3 @@ def _p_toml(s, object_pairs_hook): _p_ews(s) s.expect_eof() return stmts - -class _TimeZone(datetime.tzinfo): - def __init__(self, offset): - self._offset = offset - - def utcoffset(self, dt): - return self._offset - - def dst(self, dt): - return None - - def tzname(self, dt): - m = self._offset.total_seconds() // 60 - if m < 0: - res = '-' - m = -m - else: - res = '+' - h = m // 60 - m = m - h * 60 - return '{}{:.02}{:.02}'.format(res, h, m) diff --git a/pipenv/patched/notpip/_vendor/pytoml/test.py b/pipenv/patched/notpip/_vendor/pytoml/test.py new file mode 100644 index 00000000..ec8abfc6 --- /dev/null +++ b/pipenv/patched/notpip/_vendor/pytoml/test.py @@ -0,0 +1,30 @@ +import datetime +from .utils import format_rfc3339 + +try: + _string_types = (str, unicode) + _int_types = (int, long) +except NameError: + _string_types = str + _int_types = int + +def translate_to_test(v): + if isinstance(v, dict): + return { k: translate_to_test(v) for k, v in v.items() } + if isinstance(v, list): + a = [translate_to_test(x) for x in v] + if v and isinstance(v[0], dict): + return a + else: + return {'type': 'array', 'value': a} + if isinstance(v, datetime.datetime): + return {'type': 'datetime', 'value': format_rfc3339(v)} + if isinstance(v, bool): + return {'type': 'bool', 'value': 'true' if v else 'false'} + if isinstance(v, _int_types): + return {'type': 'integer', 'value': str(v)} + if isinstance(v, float): + return {'type': 'float', 'value': '{:.17}'.format(v)} + if isinstance(v, _string_types): + return {'type': 'string', 'value': v} + raise RuntimeError('unexpected value: {!r}'.format(v)) diff --git a/pipenv/patched/notpip/_vendor/pytoml/utils.py b/pipenv/patched/notpip/_vendor/pytoml/utils.py new file mode 100644 index 00000000..636a680b --- /dev/null +++ b/pipenv/patched/notpip/_vendor/pytoml/utils.py @@ -0,0 +1,67 @@ +import datetime +import re + +rfc3339_re = re.compile(r'(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(\.\d+)?(?:Z|([+-]\d{2}):(\d{2}))') + +def parse_rfc3339(v): + m = rfc3339_re.match(v) + if not m or m.group(0) != v: + return None + return parse_rfc3339_re(m) + +def parse_rfc3339_re(m): + r = map(int, m.groups()[:6]) + if m.group(7): + micro = float(m.group(7)) + else: + micro = 0 + + if m.group(8): + g = int(m.group(8), 10) * 60 + int(m.group(9), 10) + tz = _TimeZone(datetime.timedelta(0, g * 60)) + else: + tz = _TimeZone(datetime.timedelta(0, 0)) + + y, m, d, H, M, S = r + return datetime.datetime(y, m, d, H, M, S, int(micro * 1000000), tz) + + +def format_rfc3339(v): + offs = v.utcoffset() + offs = int(offs.total_seconds()) // 60 if offs is not None else 0 + + if offs == 0: + suffix = 'Z' + else: + if offs > 0: + suffix = '+' + else: + suffix = '-' + offs = -offs + suffix = '{0}{1:02}:{2:02}'.format(suffix, offs // 60, offs % 60) + + if v.microsecond: + return v.strftime('%Y-%m-%dT%H:%M:%S.%f') + suffix + else: + return v.strftime('%Y-%m-%dT%H:%M:%S') + suffix + +class _TimeZone(datetime.tzinfo): + def __init__(self, offset): + self._offset = offset + + def utcoffset(self, dt): + return self._offset + + def dst(self, dt): + return None + + def tzname(self, dt): + m = self._offset.total_seconds() // 60 + if m < 0: + res = '-' + m = -m + else: + res = '+' + h = m // 60 + m = m - h * 60 + return '{}{:.02}{:.02}'.format(res, h, m) diff --git a/pipenv/patched/notpip/_vendor/pytoml/writer.py b/pipenv/patched/notpip/_vendor/pytoml/writer.py index 6eaf5d76..73b5089c 100644 --- a/pipenv/patched/notpip/_vendor/pytoml/writer.py +++ b/pipenv/patched/notpip/_vendor/pytoml/writer.py @@ -1,5 +1,7 @@ from __future__ import unicode_literals -import io, datetime, math, sys +import io, datetime, math, string, sys + +from .utils import format_rfc3339 if sys.version_info[0] == 3: long = int @@ -39,22 +41,13 @@ def _escape_string(s): return '"' + ''.join(res) + '"' +_key_chars = string.digits + string.ascii_letters + '-_' def _escape_id(s): - if any(not c.isalnum() and c not in '-_' for c in s): + if any(c not in _key_chars for c in s): return _escape_string(s) return s -def _format_list(v): - return '[{0}]'.format(', '.join(_format_value(obj) for obj in v)) - -# Formula from: -# https://docs.python.org/2/library/datetime.html#datetime.timedelta.total_seconds -# Once support for py26 is dropped, this can be replaced by td.total_seconds() -def _total_seconds(td): - return ((td.microseconds - + (td.seconds + td.days * 24 * 3600) * 10**6) / 10.0**6) - def _format_value(v): if isinstance(v, bool): return 'true' if v else 'false' @@ -68,25 +61,11 @@ def _format_value(v): elif isinstance(v, unicode) or isinstance(v, bytes): return _escape_string(v) elif isinstance(v, datetime.datetime): - offs = v.utcoffset() - offs = _total_seconds(offs) // 60 if offs is not None else 0 - - if offs == 0: - suffix = 'Z' - else: - if offs > 0: - suffix = '+' - else: - suffix = '-' - offs = -offs - suffix = '{0}{1:.02}{2:.02}'.format(suffix, offs // 60, offs % 60) - - if v.microsecond: - return v.strftime('%Y-%m-%dT%H:%M:%S.%f') + suffix - else: - return v.strftime('%Y-%m-%dT%H:%M:%S') + suffix + return format_rfc3339(v) elif isinstance(v, list): - return _format_list(v) + return '[{0}]'.format(', '.join(_format_value(obj) for obj in v)) + elif isinstance(v, dict): + return '{{{0}}}'.format(', '.join('{} = {}'.format(_escape_id(k), _format_value(obj)) for k, obj in v.items())) else: raise RuntimeError(v) diff --git a/pipenv/patched/notpip/_vendor/requests/LICENSE b/pipenv/patched/notpip/_vendor/requests/LICENSE index 2e68b82e..841c6023 100644 --- a/pipenv/patched/notpip/_vendor/requests/LICENSE +++ b/pipenv/patched/notpip/_vendor/requests/LICENSE @@ -4,7 +4,7 @@ Copyright 2018 Kenneth Reitz you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, diff --git a/pipenv/patched/notpip/_vendor/requests/__init__.py b/pipenv/patched/notpip/_vendor/requests/__init__.py index af3d4b1d..1544bc89 100644 --- a/pipenv/patched/notpip/_vendor/requests/__init__.py +++ b/pipenv/patched/notpip/_vendor/requests/__init__.py @@ -22,7 +22,7 @@ usage: ... or POST: >>> payload = dict(key1='value1', key2='value2') - >>> r = requests.post('http://httpbin.org/post', data=payload) + >>> r = requests.post('https://httpbin.org/post', data=payload) >>> print(r.text) { ... @@ -57,10 +57,10 @@ def check_compatibility(urllib3_version, chardet_version): # Check urllib3 for compatibility. major, minor, patch = urllib3_version # noqa: F811 major, minor, patch = int(major), int(minor), int(patch) - # urllib3 >= 1.21.1, <= 1.23 + # urllib3 >= 1.21.1, <= 1.24 assert major == 1 assert minor >= 21 - assert minor <= 23 + assert minor <= 24 # Check chardet for compatibility. major, minor, patch = chardet_version.split('.')[:3] @@ -79,14 +79,14 @@ def _check_cryptography(cryptography_version): return if cryptography_version < [1, 3, 4]: - warning = 'Old version of cryptography ({0}) may cause slowdown.'.format(cryptography_version) + warning = 'Old version of cryptography ({}) may cause slowdown.'.format(cryptography_version) warnings.warn(warning, RequestsDependencyWarning) # Check imported dependencies for compatibility. try: check_compatibility(urllib3.__version__, chardet.__version__) except (AssertionError, ValueError): - warnings.warn("urllib3 ({0}) or chardet ({1}) doesn't match a supported " + warnings.warn("urllib3 ({}) or chardet ({}) doesn't match a supported " "version!".format(urllib3.__version__, chardet.__version__), RequestsDependencyWarning) @@ -125,12 +125,7 @@ from .exceptions import ( # Set default logging handler to avoid "No handler found" warnings. import logging -try: # Python 2.7+ - from logging import NullHandler -except ImportError: - class NullHandler(logging.Handler): - def emit(self, record): - pass +from logging import NullHandler logging.getLogger(__name__).addHandler(NullHandler()) diff --git a/pipenv/patched/notpip/_vendor/requests/__version__.py b/pipenv/patched/notpip/_vendor/requests/__version__.py index ef61ec0f..f5b5d036 100644 --- a/pipenv/patched/notpip/_vendor/requests/__version__.py +++ b/pipenv/patched/notpip/_vendor/requests/__version__.py @@ -5,8 +5,8 @@ __title__ = 'requests' __description__ = 'Python HTTP for Humans.' __url__ = 'http://python-requests.org' -__version__ = '2.19.1' -__build__ = 0x021901 +__version__ = '2.21.0' +__build__ = 0x022100 __author__ = 'Kenneth Reitz' __author_email__ = 'me@kennethreitz.org' __license__ = 'Apache 2.0' diff --git a/pipenv/patched/notpip/_vendor/requests/adapters.py b/pipenv/patched/notpip/_vendor/requests/adapters.py index 014c2675..abae6d6d 100644 --- a/pipenv/patched/notpip/_vendor/requests/adapters.py +++ b/pipenv/patched/notpip/_vendor/requests/adapters.py @@ -26,6 +26,7 @@ from pipenv.patched.notpip._vendor.urllib3.exceptions import ProtocolError from pipenv.patched.notpip._vendor.urllib3.exceptions import ReadTimeoutError from pipenv.patched.notpip._vendor.urllib3.exceptions import SSLError as _SSLError from pipenv.patched.notpip._vendor.urllib3.exceptions import ResponseError +from pipenv.patched.notpip._vendor.urllib3.exceptions import LocationValueError from .models import Response from .compat import urlparse, basestring @@ -35,7 +36,8 @@ from .utils import (DEFAULT_CA_BUNDLE_PATH, extract_zipped_paths, from .structures import CaseInsensitiveDict from .cookies import extract_cookies_to_jar from .exceptions import (ConnectionError, ConnectTimeout, ReadTimeout, SSLError, - ProxyError, RetryError, InvalidSchema, InvalidProxyURL) + ProxyError, RetryError, InvalidSchema, InvalidProxyURL, + InvalidURL) from .auth import _basic_auth_str try: @@ -127,8 +129,7 @@ class HTTPAdapter(BaseAdapter): self.init_poolmanager(pool_connections, pool_maxsize, block=pool_block) def __getstate__(self): - return dict((attr, getattr(self, attr, None)) for attr in - self.__attrs__) + return {attr: getattr(self, attr, None) for attr in self.__attrs__} def __setstate__(self, state): # Can't handle by adding 'proxy_manager' to self.__attrs__ because @@ -224,7 +225,7 @@ class HTTPAdapter(BaseAdapter): if not cert_loc or not os.path.exists(cert_loc): raise IOError("Could not find a suitable TLS CA certificate bundle, " - "invalid path: {0}".format(cert_loc)) + "invalid path: {}".format(cert_loc)) conn.cert_reqs = 'CERT_REQUIRED' @@ -246,10 +247,10 @@ class HTTPAdapter(BaseAdapter): conn.key_file = None if conn.cert_file and not os.path.exists(conn.cert_file): raise IOError("Could not find the TLS certificate file, " - "invalid path: {0}".format(conn.cert_file)) + "invalid path: {}".format(conn.cert_file)) if conn.key_file and not os.path.exists(conn.key_file): raise IOError("Could not find the TLS key file, " - "invalid path: {0}".format(conn.key_file)) + "invalid path: {}".format(conn.key_file)) def build_response(self, req, resp): """Builds a :class:`Response ` object from a urllib3 @@ -378,7 +379,7 @@ class HTTPAdapter(BaseAdapter): when subclassing the :class:`HTTPAdapter `. - :param proxies: The url of the proxy being used for this request. + :param proxy: The url of the proxy being used for this request. :rtype: dict """ headers = {} @@ -407,7 +408,10 @@ class HTTPAdapter(BaseAdapter): :rtype: requests.Response """ - conn = self.get_connection(request.url, proxies) + try: + conn = self.get_connection(request.url, proxies) + except LocationValueError as e: + raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) @@ -421,7 +425,7 @@ class HTTPAdapter(BaseAdapter): timeout = TimeoutSauce(connect=connect, read=read) except ValueError as e: # this may raise a string formatting error. - err = ("Invalid timeout {0}. Pass a (connect, read) " + err = ("Invalid timeout {}. Pass a (connect, read) " "timeout tuple, or a single float to set " "both timeouts to the same value".format(timeout)) raise ValueError(err) @@ -471,11 +475,10 @@ class HTTPAdapter(BaseAdapter): # Receive the response from the server try: - # For Python 2.7+ versions, use buffering of HTTP - # responses + # For Python 2.7, use buffering of HTTP responses r = low_conn.getresponse(buffering=True) except TypeError: - # For compatibility with Python 2.6 versions and back + # For compatibility with Python 3.3+ r = low_conn.getresponse() resp = HTTPResponse.from_httplib( diff --git a/pipenv/patched/notpip/_vendor/requests/api.py b/pipenv/patched/notpip/_vendor/requests/api.py index a2cc84d7..abada96d 100644 --- a/pipenv/patched/notpip/_vendor/requests/api.py +++ b/pipenv/patched/notpip/_vendor/requests/api.py @@ -18,8 +18,10 @@ def request(method, url, **kwargs): :param method: method for the new :class:`Request` object. :param url: URL for the new :class:`Request` object. - :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. - :param data: (optional) Dictionary or list of tuples ``[(key, value)]`` (will be form-encoded), bytes, or file-like object to send in the body of the :class:`Request`. + :param params: (optional) Dictionary, list of tuples or bytes to send + in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) A JSON serializable Python object to send in the body of the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. @@ -47,7 +49,7 @@ def request(method, url, **kwargs): Usage:: >>> import requests - >>> req = requests.request('GET', 'http://httpbin.org/get') + >>> req = requests.request('GET', 'https://httpbin.org/get') """ @@ -62,7 +64,8 @@ def get(url, params=None, **kwargs): r"""Sends a GET request. :param url: URL for the new :class:`Request` object. - :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. + :param params: (optional) Dictionary, list of tuples or bytes to send + in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :return: :class:`Response ` object :rtype: requests.Response @@ -102,7 +105,8 @@ def post(url, data=None, json=None, **kwargs): r"""Sends a POST request. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary (will be form-encoded), bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) json data to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :return: :class:`Response ` object @@ -116,7 +120,8 @@ def put(url, data=None, **kwargs): r"""Sends a PUT request. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary (will be form-encoded), bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) json data to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :return: :class:`Response ` object @@ -130,7 +135,8 @@ def patch(url, data=None, **kwargs): r"""Sends a PATCH request. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary (will be form-encoded), bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) json data to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :return: :class:`Response ` object diff --git a/pipenv/patched/notpip/_vendor/requests/auth.py b/pipenv/patched/notpip/_vendor/requests/auth.py index 4ae45947..bdde51c7 100644 --- a/pipenv/patched/notpip/_vendor/requests/auth.py +++ b/pipenv/patched/notpip/_vendor/requests/auth.py @@ -38,7 +38,7 @@ def _basic_auth_str(username, password): if not isinstance(username, basestring): warnings.warn( "Non-string usernames will no longer be supported in Requests " - "3.0.0. Please convert the object you've passed in ({0!r}) to " + "3.0.0. Please convert the object you've passed in ({!r}) to " "a string or bytes object in the near future to avoid " "problems.".format(username), category=DeprecationWarning, @@ -48,7 +48,7 @@ def _basic_auth_str(username, password): if not isinstance(password, basestring): warnings.warn( "Non-string passwords will no longer be supported in Requests " - "3.0.0. Please convert the object you've passed in ({0!r}) to " + "3.0.0. Please convert the object you've passed in ({!r}) to " "a string or bytes object in the near future to avoid " "problems.".format(password), category=DeprecationWarning, diff --git a/pipenv/patched/notpip/_vendor/requests/compat.py b/pipenv/patched/notpip/_vendor/requests/compat.py index 4fbd6231..7c143940 100644 --- a/pipenv/patched/notpip/_vendor/requests/compat.py +++ b/pipenv/patched/notpip/_vendor/requests/compat.py @@ -47,9 +47,8 @@ if is_py2: import cookielib from Cookie import Morsel from StringIO import StringIO - from collections import Callable, Mapping, MutableMapping + from collections import Callable, Mapping, MutableMapping, OrderedDict - from pipenv.patched.notpip._vendor.urllib3.packages.ordered_dict import OrderedDict builtin_str = str bytes = str diff --git a/pipenv/patched/notpip/_vendor/requests/cookies.py b/pipenv/patched/notpip/_vendor/requests/cookies.py index 50883a84..56fccd9c 100644 --- a/pipenv/patched/notpip/_vendor/requests/cookies.py +++ b/pipenv/patched/notpip/_vendor/requests/cookies.py @@ -444,20 +444,21 @@ def create_cookie(name, value, **kwargs): By default, the pair of `name` and `value` will be set for the domain '' and sent on every request (this is sometimes called a "supercookie"). """ - result = dict( - version=0, - name=name, - value=value, - port=None, - domain='', - path='/', - secure=False, - expires=None, - discard=True, - comment=None, - comment_url=None, - rest={'HttpOnly': None}, - rfc2109=False,) + result = { + 'version': 0, + 'name': name, + 'value': value, + 'port': None, + 'domain': '', + 'path': '/', + 'secure': False, + 'expires': None, + 'discard': True, + 'comment': None, + 'comment_url': None, + 'rest': {'HttpOnly': None}, + 'rfc2109': False, + } badargs = set(kwargs) - set(result) if badargs: @@ -511,6 +512,7 @@ def cookiejar_from_dict(cookie_dict, cookiejar=None, overwrite=True): :param cookiejar: (optional) A cookiejar to add the cookies to. :param overwrite: (optional) If False, will not replace cookies already in the jar with new ones. + :rtype: CookieJar """ if cookiejar is None: cookiejar = RequestsCookieJar() @@ -529,6 +531,7 @@ def merge_cookies(cookiejar, cookies): :param cookiejar: CookieJar object to add the cookies to. :param cookies: Dictionary or CookieJar object to be added. + :rtype: CookieJar """ if not isinstance(cookiejar, cookielib.CookieJar): raise ValueError('You can only merge into CookieJar') diff --git a/pipenv/patched/notpip/_vendor/requests/help.py b/pipenv/patched/notpip/_vendor/requests/help.py index eba69edb..72d72160 100644 --- a/pipenv/patched/notpip/_vendor/requests/help.py +++ b/pipenv/patched/notpip/_vendor/requests/help.py @@ -89,8 +89,7 @@ def info(): 'version': getattr(idna, '__version__', ''), } - # OPENSSL_VERSION_NUMBER doesn't exist in the Python 2.6 ssl module. - system_ssl = getattr(ssl, 'OPENSSL_VERSION_NUMBER', None) + system_ssl = ssl.OPENSSL_VERSION_NUMBER system_ssl_info = { 'version': '%x' % system_ssl if system_ssl is not None else '' } diff --git a/pipenv/patched/notpip/_vendor/requests/hooks.py b/pipenv/patched/notpip/_vendor/requests/hooks.py index 32b32de7..7a51f212 100644 --- a/pipenv/patched/notpip/_vendor/requests/hooks.py +++ b/pipenv/patched/notpip/_vendor/requests/hooks.py @@ -15,14 +15,14 @@ HOOKS = ['response'] def default_hooks(): - return dict((event, []) for event in HOOKS) + return {event: [] for event in HOOKS} # TODO: response is the only one def dispatch_hook(key, hooks, hook_data, **kwargs): """Dispatches a hook dictionary on a given piece of data.""" - hooks = hooks or dict() + hooks = hooks or {} hooks = hooks.get(key) if hooks: if hasattr(hooks, '__call__'): diff --git a/pipenv/patched/notpip/_vendor/requests/models.py b/pipenv/patched/notpip/_vendor/requests/models.py index 6708f09b..5f899c42 100644 --- a/pipenv/patched/notpip/_vendor/requests/models.py +++ b/pipenv/patched/notpip/_vendor/requests/models.py @@ -204,9 +204,13 @@ class Request(RequestHooksMixin): :param url: URL to send. :param headers: dictionary of headers to send. :param files: dictionary of {filename: fileobject} files to multipart upload. - :param data: the body to attach to the request. If a dictionary is provided, form-encoding will take place. + :param data: the body to attach to the request. If a dictionary or + list of tuples ``[(key, value)]`` is provided, form-encoding will + take place. :param json: json for the body to attach to the request (if files or data is not specified). - :param params: dictionary of URL parameters to append to the URL. + :param params: URL parameters to append to the URL. If a dictionary or + list of tuples ``[(key, value)]`` is provided, form-encoding will + take place. :param auth: Auth handler or (user, pass) tuple. :param cookies: dictionary or CookieJar of cookies to attach to this request. :param hooks: dictionary of callback hooks, for internal usage. @@ -214,7 +218,7 @@ class Request(RequestHooksMixin): Usage:: >>> import requests - >>> req = requests.Request('GET', 'http://httpbin.org/get') + >>> req = requests.Request('GET', 'https://httpbin.org/get') >>> req.prepare() """ @@ -274,7 +278,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): Usage:: >>> import requests - >>> req = requests.Request('GET', 'http://httpbin.org/get') + >>> req = requests.Request('GET', 'https://httpbin.org/get') >>> r = req.prepare() @@ -648,10 +652,7 @@ class Response(object): if not self._content_consumed: self.content - return dict( - (attr, getattr(self, attr, None)) - for attr in self.__attrs__ - ) + return {attr: getattr(self, attr, None) for attr in self.__attrs__} def __setstate__(self, state): for name, value in state.items(): @@ -780,7 +781,7 @@ class Response(object): return chunks - def iter_lines(self, chunk_size=ITER_CHUNK_SIZE, decode_unicode=None, delimiter=None): + def iter_lines(self, chunk_size=ITER_CHUNK_SIZE, decode_unicode=False, delimiter=None): """Iterates over the response data, one line at a time. When stream=True is set on the request, this avoids reading the content at once into memory for large responses. diff --git a/pipenv/patched/notpip/_vendor/requests/sessions.py b/pipenv/patched/notpip/_vendor/requests/sessions.py index ba135268..d73d700f 100644 --- a/pipenv/patched/notpip/_vendor/requests/sessions.py +++ b/pipenv/patched/notpip/_vendor/requests/sessions.py @@ -19,7 +19,7 @@ from .cookies import ( from .models import Request, PreparedRequest, DEFAULT_REDIRECT_LIMIT from .hooks import default_hooks, dispatch_hook from ._internal_utils import to_native_string -from .utils import to_key_val_list, default_headers +from .utils import to_key_val_list, default_headers, DEFAULT_PORTS from .exceptions import ( TooManyRedirects, InvalidSchema, ChunkedEncodingError, ContentDecodingError) @@ -115,6 +115,31 @@ class SessionRedirectMixin(object): return to_native_string(location, 'utf8') return None + def should_strip_auth(self, old_url, new_url): + """Decide whether Authorization header should be removed when redirecting""" + old_parsed = urlparse(old_url) + new_parsed = urlparse(new_url) + if old_parsed.hostname != new_parsed.hostname: + return True + # Special case: allow http -> https redirect when using the standard + # ports. This isn't specified by RFC 7235, but is kept to avoid + # breaking backwards compatibility with older versions of requests + # that allowed any redirects on the same host. + if (old_parsed.scheme == 'http' and old_parsed.port in (80, None) + and new_parsed.scheme == 'https' and new_parsed.port in (443, None)): + return False + + # Handle default port usage corresponding to scheme. + changed_port = old_parsed.port != new_parsed.port + changed_scheme = old_parsed.scheme != new_parsed.scheme + default_port = (DEFAULT_PORTS.get(old_parsed.scheme, None), None) + if (not changed_scheme and old_parsed.port in default_port + and new_parsed.port in default_port): + return False + + # Standard case: root URI must match + return changed_port or changed_scheme + def resolve_redirects(self, resp, req, stream=False, timeout=None, verify=True, cert=None, proxies=None, yield_requests=False, **adapter_kwargs): """Receives a Response. Returns a generator of Responses or Requests.""" @@ -236,14 +261,10 @@ class SessionRedirectMixin(object): headers = prepared_request.headers url = prepared_request.url - if 'Authorization' in headers: + if 'Authorization' in headers and self.should_strip_auth(response.request.url, url): # If we get redirected to a new host, we should strip out any # authentication headers. - original_parsed = urlparse(response.request.url) - redirect_parsed = urlparse(url) - - if (original_parsed.hostname != redirect_parsed.hostname): - del headers['Authorization'] + del headers['Authorization'] # .netrc might have more auth for us on our new host. new_auth = get_netrc_auth(url) if self.trust_env else None @@ -299,7 +320,7 @@ class SessionRedirectMixin(object): """ method = prepared_request.method - # http://tools.ietf.org/html/rfc7231#section-6.4.4 + # https://tools.ietf.org/html/rfc7231#section-6.4.4 if response.status_code == codes.see_other and method != 'HEAD': method = 'GET' @@ -325,13 +346,13 @@ class Session(SessionRedirectMixin): >>> import requests >>> s = requests.Session() - >>> s.get('http://httpbin.org/get') + >>> s.get('https://httpbin.org/get') Or as a context manager:: >>> with requests.Session() as s: - >>> s.get('http://httpbin.org/get') + >>> s.get('https://httpbin.org/get') """ @@ -453,8 +474,8 @@ class Session(SessionRedirectMixin): :param url: URL for the new :class:`Request` object. :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. - :param data: (optional) Dictionary, bytes, or file-like object to send - in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) json to send in the body of the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to send with the @@ -550,7 +571,8 @@ class Session(SessionRedirectMixin): r"""Sends a POST request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) json to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :rtype: requests.Response @@ -562,7 +584,8 @@ class Session(SessionRedirectMixin): r"""Sends a PUT request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :rtype: requests.Response """ @@ -573,7 +596,8 @@ class Session(SessionRedirectMixin): r"""Sends a PATCH request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :rtype: requests.Response """ @@ -723,7 +747,7 @@ class Session(SessionRedirectMixin): self.adapters[key] = self.adapters.pop(key) def __getstate__(self): - state = dict((attr, getattr(self, attr, None)) for attr in self.__attrs__) + state = {attr: getattr(self, attr, None) for attr in self.__attrs__} return state def __setstate__(self, state): @@ -735,7 +759,12 @@ def session(): """ Returns a :class:`Session` for context-management. + .. deprecated:: 1.0.0 + + This method has been deprecated since version 1.0.0 and is only kept for + backwards compatibility. New code should use :class:`~requests.sessions.Session` + to create a session. This may be removed at a future date. + :rtype: Session """ - return Session() diff --git a/pipenv/patched/notpip/_vendor/requests/status_codes.py b/pipenv/patched/notpip/_vendor/requests/status_codes.py index ff462c6c..813e8c4e 100644 --- a/pipenv/patched/notpip/_vendor/requests/status_codes.py +++ b/pipenv/patched/notpip/_vendor/requests/status_codes.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -""" +r""" The ``codes`` object defines a mapping from common names for HTTP statuses to their numerical codes, accessible either as attributes or as dictionary items. diff --git a/pipenv/patched/notpip/_vendor/requests/utils.py b/pipenv/patched/notpip/_vendor/requests/utils.py index 431f6be0..8170a8d2 100644 --- a/pipenv/patched/notpip/_vendor/requests/utils.py +++ b/pipenv/patched/notpip/_vendor/requests/utils.py @@ -38,6 +38,8 @@ NETRC_FILES = ('.netrc', '_netrc') DEFAULT_CA_BUNDLE_PATH = certs.where() +DEFAULT_PORTS = {'http': 80, 'https': 443} + if sys.platform == 'win32': # provide a proxy_bypass version on Windows without DNS lookups @@ -173,10 +175,10 @@ def get_netrc_auth(url, raise_errors=False): for f in NETRC_FILES: try: - loc = os.path.expanduser('~/{0}'.format(f)) + loc = os.path.expanduser('~/{}'.format(f)) except KeyError: # os.path.expanduser can fail when $HOME is undefined and - # getpwuid fails. See http://bugs.python.org/issue20164 & + # getpwuid fails. See https://bugs.python.org/issue20164 & # https://github.com/requests/requests/issues/1846 return @@ -264,7 +266,7 @@ def from_key_val_list(value): >>> from_key_val_list([('key', 'val')]) OrderedDict([('key', 'val')]) >>> from_key_val_list('string') - ValueError: need more than 1 value to unpack + ValueError: cannot encode objects that are not 2-tuples >>> from_key_val_list({'key': 'val'}) OrderedDict([('key', 'val')]) @@ -466,7 +468,7 @@ def _parse_content_type_header(header): if index_of_equals != -1: key = param[:index_of_equals].strip(items_to_strip) value = param[index_of_equals + 1:].strip(items_to_strip) - params_dict[key] = value + params_dict[key.lower()] = value return content_type, params_dict @@ -706,6 +708,10 @@ def should_bypass_proxies(url, no_proxy): no_proxy = get_proxy('no_proxy') parsed = urlparse(url) + if parsed.hostname is None: + # URLs don't always have hostnames, e.g. file:/// urls. + return True + if no_proxy: # We need to check whether we match here. We need to see if we match # the end of the hostname, both with and without the port. @@ -725,7 +731,7 @@ def should_bypass_proxies(url, no_proxy): else: host_with_port = parsed.hostname if parsed.port: - host_with_port += ':{0}'.format(parsed.port) + host_with_port += ':{}'.format(parsed.port) for host in no_proxy: if parsed.hostname.endswith(host) or host_with_port.endswith(host): @@ -733,13 +739,8 @@ def should_bypass_proxies(url, no_proxy): # to apply the proxies on this URL. return True - # If the system proxy settings indicate that this URL should be bypassed, - # don't proxy. - # The proxy_bypass function is incredibly buggy on OS X in early versions - # of Python 2.6, so allow this call to fail. Only catch the specific - # exceptions we've seen, though: this call failing in other ways can reveal - # legitimate problems. with set_environ('no_proxy', no_proxy_arg): + # parsed.hostname can be `None` in cases such as a file URI. try: bypass = proxy_bypass(parsed.hostname) except (TypeError, socket.gaierror): diff --git a/pipenv/patched/notpip/_vendor/six.LICENSE b/pipenv/patched/notpip/_vendor/six.LICENSE index f3068bfd..365d1074 100644 --- a/pipenv/patched/notpip/_vendor/six.LICENSE +++ b/pipenv/patched/notpip/_vendor/six.LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2010-2017 Benjamin Peterson +Copyright (c) 2010-2018 Benjamin Peterson Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in diff --git a/pipenv/patched/notpip/_vendor/six.py b/pipenv/patched/notpip/_vendor/six.py index 6bf4fd38..89b2188f 100644 --- a/pipenv/patched/notpip/_vendor/six.py +++ b/pipenv/patched/notpip/_vendor/six.py @@ -1,4 +1,4 @@ -# Copyright (c) 2010-2017 Benjamin Peterson +# Copyright (c) 2010-2018 Benjamin Peterson # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -29,7 +29,7 @@ import sys import types __author__ = "Benjamin Peterson " -__version__ = "1.11.0" +__version__ = "1.12.0" # Useful for very coarse version differentiation. @@ -844,10 +844,71 @@ def add_metaclass(metaclass): orig_vars.pop(slots_var) orig_vars.pop('__dict__', None) orig_vars.pop('__weakref__', None) + if hasattr(cls, '__qualname__'): + orig_vars['__qualname__'] = cls.__qualname__ return metaclass(cls.__name__, cls.__bases__, orig_vars) return wrapper +def ensure_binary(s, encoding='utf-8', errors='strict'): + """Coerce **s** to six.binary_type. + + For Python 2: + - `unicode` -> encoded to `str` + - `str` -> `str` + + For Python 3: + - `str` -> encoded to `bytes` + - `bytes` -> `bytes` + """ + if isinstance(s, text_type): + return s.encode(encoding, errors) + elif isinstance(s, binary_type): + return s + else: + raise TypeError("not expecting type '%s'" % type(s)) + + +def ensure_str(s, encoding='utf-8', errors='strict'): + """Coerce *s* to `str`. + + For Python 2: + - `unicode` -> encoded to `str` + - `str` -> `str` + + For Python 3: + - `str` -> `str` + - `bytes` -> decoded to `str` + """ + if not isinstance(s, (text_type, binary_type)): + raise TypeError("not expecting type '%s'" % type(s)) + if PY2 and isinstance(s, text_type): + s = s.encode(encoding, errors) + elif PY3 and isinstance(s, binary_type): + s = s.decode(encoding, errors) + return s + + +def ensure_text(s, encoding='utf-8', errors='strict'): + """Coerce *s* to six.text_type. + + For Python 2: + - `unicode` -> `unicode` + - `str` -> `unicode` + + For Python 3: + - `str` -> `str` + - `bytes` -> decoded to `str` + """ + if isinstance(s, binary_type): + return s.decode(encoding, errors) + elif isinstance(s, text_type): + return s + else: + raise TypeError("not expecting type '%s'" % type(s)) + + + def python_2_unicode_compatible(klass): """ A decorator that defines __unicode__ and __str__ methods under Python 2. diff --git a/pipenv/patched/notpip/_vendor/urllib3/LICENSE.txt b/pipenv/patched/notpip/_vendor/urllib3/LICENSE.txt index 1c3283ee..c89cf27b 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/LICENSE.txt +++ b/pipenv/patched/notpip/_vendor/urllib3/LICENSE.txt @@ -1,19 +1,21 @@ -This is the MIT license: http://www.opensource.org/licenses/mit-license.php +MIT License -Copyright 2008-2016 Andrey Petrov and contributors (see CONTRIBUTORS.txt) +Copyright (c) 2008-2019 Andrey Petrov and contributors (see CONTRIBUTORS.txt) -Permission is hereby granted, free of charge, to any person obtaining a copy of this -software and associated documentation files (the "Software"), to deal in the Software -without restriction, including without limitation the rights to use, copy, modify, merge, -publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons -to whom the Software is furnished to do so, subject to the following conditions: +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: -The above copyright notice and this permission notice shall be included in all copies or -substantial portions of the Software. +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, -INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR -PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE -FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/pipenv/patched/notpip/_vendor/urllib3/__init__.py b/pipenv/patched/notpip/_vendor/urllib3/__init__.py index 4bd533b5..148a9c31 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/__init__.py +++ b/pipenv/patched/notpip/_vendor/urllib3/__init__.py @@ -23,16 +23,11 @@ from .util.retry import Retry # Set default logging handler to avoid "No handler found" warnings. import logging -try: # Python 2.7+ - from logging import NullHandler -except ImportError: - class NullHandler(logging.Handler): - def emit(self, record): - pass +from logging import NullHandler __author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' __license__ = 'MIT' -__version__ = '1.23' +__version__ = '1.24.1' __all__ = ( 'HTTPConnectionPool', diff --git a/pipenv/patched/notpip/_vendor/urllib3/_collections.py b/pipenv/patched/notpip/_vendor/urllib3/_collections.py index 6e36b84e..34f23811 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/_collections.py +++ b/pipenv/patched/notpip/_vendor/urllib3/_collections.py @@ -14,10 +14,7 @@ except ImportError: # Platform-specific: No threads available pass -try: # Python 2.7+ - from collections import OrderedDict -except ImportError: - from .packages.ordered_dict import OrderedDict +from collections import OrderedDict from .exceptions import InvalidHeader from .packages.six import iterkeys, itervalues, PY3 diff --git a/pipenv/patched/notpip/_vendor/urllib3/connection.py b/pipenv/patched/notpip/_vendor/urllib3/connection.py index a03b573f..02b36654 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/connection.py +++ b/pipenv/patched/notpip/_vendor/urllib3/connection.py @@ -2,7 +2,6 @@ from __future__ import absolute_import import datetime import logging import os -import sys import socket from socket import error as SocketError, timeout as SocketTimeout import warnings @@ -78,9 +77,6 @@ class HTTPConnection(_HTTPConnection, object): - ``strict``: See the documentation on :class:`urllib3.connectionpool.HTTPConnectionPool` - ``source_address``: Set the source address for the current connection. - - .. note:: This is ignored for Python 2.6. It is only applied for 2.7 and 3.x - - ``socket_options``: Set specific options on the underlying socket. If not specified, then defaults are loaded from ``HTTPConnection.default_socket_options`` which includes disabling Nagle's algorithm (sets TCP_NODELAY to 1) unless the connection is behind a proxy. @@ -108,21 +104,13 @@ class HTTPConnection(_HTTPConnection, object): if six.PY3: # Python 3 kw.pop('strict', None) - # Pre-set source_address in case we have an older Python like 2.6. + # Pre-set source_address. self.source_address = kw.get('source_address') - if sys.version_info < (2, 7): # Python 2.6 - # _HTTPConnection on Python 2.6 will balk at this keyword arg, but - # not newer versions. We can still use it when creating a - # connection though, so we pop it *after* we have saved it as - # self.source_address. - kw.pop('source_address', None) - #: The socket options provided by the user. If no options are #: provided, we use the default options. self.socket_options = kw.pop('socket_options', self.default_socket_options) - # Superclass also sets self.source_address in Python 2.7+. _HTTPConnection.__init__(self, *args, **kw) @property @@ -183,10 +171,7 @@ class HTTPConnection(_HTTPConnection, object): def _prepare_conn(self, conn): self.sock = conn - # the _tunnel_host attribute was added in python 2.6.3 (via - # http://hg.python.org/cpython/rev/0f57b30a152f) so pythons 2.6(0-2) do - # not have them. - if getattr(self, '_tunnel_host', None): + if self._tunnel_host: # TODO: Fix tunnel so it doesn't depend on self.sock state. self._tunnel() # Mark this connection as not reusable @@ -217,13 +202,13 @@ class HTTPConnection(_HTTPConnection, object): self.endheaders() if body is not None: - stringish_types = six.string_types + (six.binary_type,) + stringish_types = six.string_types + (bytes,) if isinstance(body, stringish_types): body = (body,) for chunk in body: if not chunk: continue - if not isinstance(chunk, six.binary_type): + if not isinstance(chunk, bytes): chunk = chunk.encode('utf8') len_str = hex(len(chunk))[2:] self.send(len_str.encode('utf-8')) @@ -242,7 +227,7 @@ class HTTPSConnection(HTTPConnection): def __init__(self, host, port=None, key_file=None, cert_file=None, strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, - ssl_context=None, **kw): + ssl_context=None, server_hostname=None, **kw): HTTPConnection.__init__(self, host, port, strict=strict, timeout=timeout, **kw) @@ -250,6 +235,7 @@ class HTTPSConnection(HTTPConnection): self.key_file = key_file self.cert_file = cert_file self.ssl_context = ssl_context + self.server_hostname = server_hostname # Required property for Google AppEngine 1.9.0 which otherwise causes # HTTPS requests to go out as HTTP. (See Issue #356) @@ -270,6 +256,7 @@ class HTTPSConnection(HTTPConnection): keyfile=self.key_file, certfile=self.cert_file, ssl_context=self.ssl_context, + server_hostname=self.server_hostname ) @@ -312,12 +299,9 @@ class VerifiedHTTPSConnection(HTTPSConnection): def connect(self): # Add certificate verification conn = self._new_conn() - hostname = self.host - if getattr(self, '_tunnel_host', None): - # _tunnel_host was added in Python 2.6.3 - # (See: http://hg.python.org/cpython/rev/0f57b30a152f) + if self._tunnel_host: self.sock = conn # Calls self._set_hostport(), so self.host is # self._tunnel_host below. @@ -328,6 +312,10 @@ class VerifiedHTTPSConnection(HTTPSConnection): # Override the host with the one we're requesting data from. hostname = self._tunnel_host + server_hostname = hostname + if self.server_hostname is not None: + server_hostname = self.server_hostname + is_time_off = datetime.date.today() < RECENT_DATE if is_time_off: warnings.warn(( @@ -352,7 +340,7 @@ class VerifiedHTTPSConnection(HTTPSConnection): certfile=self.cert_file, ca_certs=self.ca_certs, ca_cert_dir=self.ca_cert_dir, - server_hostname=hostname, + server_hostname=server_hostname, ssl_context=context) if self.assert_fingerprint: @@ -373,7 +361,7 @@ class VerifiedHTTPSConnection(HTTPSConnection): 'for details.)'.format(hostname)), SubjectAltNameWarning ) - _match_hostname(cert, self.assert_hostname or hostname) + _match_hostname(cert, self.assert_hostname or server_hostname) self.is_verified = ( context.verify_mode == ssl.CERT_REQUIRED or diff --git a/pipenv/patched/notpip/_vendor/urllib3/connectionpool.py b/pipenv/patched/notpip/_vendor/urllib3/connectionpool.py index 8fcb0bce..f7a8f193 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/connectionpool.py +++ b/pipenv/patched/notpip/_vendor/urllib3/connectionpool.py @@ -89,7 +89,7 @@ class ConnectionPool(object): # This is taken from http://hg.python.org/cpython/file/7aaba721ebc0/Lib/socket.py#l252 -_blocking_errnos = set([errno.EAGAIN, errno.EWOULDBLOCK]) +_blocking_errnos = {errno.EAGAIN, errno.EWOULDBLOCK} class HTTPConnectionPool(ConnectionPool, RequestMethods): @@ -313,7 +313,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # Catch possible read timeouts thrown as SSL errors. If not the # case, rethrow the original. We need to do this because of: # http://bugs.python.org/issue10272 - if 'timed out' in str(err) or 'did not complete (read)' in str(err): # Python 2.6 + if 'timed out' in str(err) or 'did not complete (read)' in str(err): # Python < 2.7.4 raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value) def _make_request(self, conn, method, url, timeout=_Default, chunked=False, @@ -375,7 +375,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): try: try: # Python 2.7, use buffering of HTTP responses httplib_response = conn.getresponse(buffering=True) - except TypeError: # Python 2.6 and older, Python 3 + except TypeError: # Python 3 try: httplib_response = conn.getresponse() except Exception as e: @@ -801,17 +801,7 @@ class HTTPSConnectionPool(HTTPConnectionPool): Establish tunnel connection early, because otherwise httplib would improperly set Host: header to proxy's IP:port. """ - # Python 2.7+ - try: - set_tunnel = conn.set_tunnel - except AttributeError: # Platform-specific: Python 2.6 - set_tunnel = conn._set_tunnel - - if sys.version_info <= (2, 6, 4) and not self.proxy_headers: # Python 2.6.4 and older - set_tunnel(self._proxy_host, self.port) - else: - set_tunnel(self._proxy_host, self.port, self.proxy_headers) - + conn.set_tunnel(self._proxy_host, self.port, self.proxy_headers) conn.connect() def _new_conn(self): diff --git a/pipenv/patched/notpip/_vendor/urllib3/contrib/_appengine_environ.py b/pipenv/patched/notpip/_vendor/urllib3/contrib/_appengine_environ.py new file mode 100644 index 00000000..f3e00942 --- /dev/null +++ b/pipenv/patched/notpip/_vendor/urllib3/contrib/_appengine_environ.py @@ -0,0 +1,30 @@ +""" +This module provides means to detect the App Engine environment. +""" + +import os + + +def is_appengine(): + return (is_local_appengine() or + is_prod_appengine() or + is_prod_appengine_mvms()) + + +def is_appengine_sandbox(): + return is_appengine() and not is_prod_appengine_mvms() + + +def is_local_appengine(): + return ('APPENGINE_RUNTIME' in os.environ and + 'Development/' in os.environ['SERVER_SOFTWARE']) + + +def is_prod_appengine(): + return ('APPENGINE_RUNTIME' in os.environ and + 'Google App Engine/' in os.environ['SERVER_SOFTWARE'] and + not is_prod_appengine_mvms()) + + +def is_prod_appengine_mvms(): + return os.environ.get('GAE_VM', False) == 'true' diff --git a/pipenv/patched/notpip/_vendor/urllib3/contrib/appengine.py b/pipenv/patched/notpip/_vendor/urllib3/contrib/appengine.py index 06586352..1c2332cb 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/contrib/appengine.py +++ b/pipenv/patched/notpip/_vendor/urllib3/contrib/appengine.py @@ -39,8 +39,8 @@ urllib3 on Google App Engine: """ from __future__ import absolute_import +import io import logging -import os import warnings from ..packages.six.moves.urllib.parse import urljoin @@ -53,11 +53,11 @@ from ..exceptions import ( SSLError ) -from ..packages.six import BytesIO from ..request import RequestMethods from ..response import HTTPResponse from ..util.timeout import Timeout from ..util.retry import Retry +from . import _appengine_environ try: from google.appengine.api import urlfetch @@ -239,7 +239,7 @@ class AppEngineManager(RequestMethods): original_response = HTTPResponse( # In order for decoding to work, we must present the content as # a file-like object. - body=BytesIO(urlfetch_resp.content), + body=io.BytesIO(urlfetch_resp.content), msg=urlfetch_resp.header_msg, headers=urlfetch_resp.headers, status=urlfetch_resp.status_code, @@ -247,7 +247,7 @@ class AppEngineManager(RequestMethods): ) return HTTPResponse( - body=BytesIO(urlfetch_resp.content), + body=io.BytesIO(urlfetch_resp.content), headers=urlfetch_resp.headers, status=urlfetch_resp.status_code, original_response=original_response, @@ -280,26 +280,10 @@ class AppEngineManager(RequestMethods): return retries -def is_appengine(): - return (is_local_appengine() or - is_prod_appengine() or - is_prod_appengine_mvms()) +# Alias methods from _appengine_environ to maintain public API interface. - -def is_appengine_sandbox(): - return is_appengine() and not is_prod_appengine_mvms() - - -def is_local_appengine(): - return ('APPENGINE_RUNTIME' in os.environ and - 'Development/' in os.environ['SERVER_SOFTWARE']) - - -def is_prod_appengine(): - return ('APPENGINE_RUNTIME' in os.environ and - 'Google App Engine/' in os.environ['SERVER_SOFTWARE'] and - not is_prod_appengine_mvms()) - - -def is_prod_appengine_mvms(): - return os.environ.get('GAE_VM', False) == 'true' +is_appengine = _appengine_environ.is_appengine +is_appengine_sandbox = _appengine_environ.is_appengine_sandbox +is_local_appengine = _appengine_environ.is_local_appengine +is_prod_appengine = _appengine_environ.is_prod_appengine +is_prod_appengine_mvms = _appengine_environ.is_prod_appengine_mvms diff --git a/pipenv/patched/notpip/_vendor/urllib3/contrib/ntlmpool.py b/pipenv/patched/notpip/_vendor/urllib3/contrib/ntlmpool.py index 642e99ed..8ea127c5 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/contrib/ntlmpool.py +++ b/pipenv/patched/notpip/_vendor/urllib3/contrib/ntlmpool.py @@ -43,8 +43,7 @@ class NTLMConnectionPool(HTTPSConnectionPool): log.debug('Starting NTLM HTTPS connection no. %d: https://%s%s', self.num_connections, self.host, self.authurl) - headers = {} - headers['Connection'] = 'Keep-Alive' + headers = {'Connection': 'Keep-Alive'} req_header = 'Authorization' resp_header = 'www-authenticate' diff --git a/pipenv/patched/notpip/_vendor/urllib3/contrib/pyopenssl.py b/pipenv/patched/notpip/_vendor/urllib3/contrib/pyopenssl.py index 7787d4e4..f5bc7d83 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/contrib/pyopenssl.py +++ b/pipenv/patched/notpip/_vendor/urllib3/contrib/pyopenssl.py @@ -163,6 +163,9 @@ def _dnsname_to_stdlib(name): from ASCII bytes. We need to idna-encode that string to get it back, and then on Python 3 we also need to convert to unicode via UTF-8 (the stdlib uses PyUnicode_FromStringAndSize on it, which decodes via UTF-8). + + If the name cannot be idna-encoded then we return None signalling that + the name given should be skipped. """ def idna_encode(name): """ @@ -172,14 +175,19 @@ def _dnsname_to_stdlib(name): """ from pipenv.patched.notpip._vendor import idna - for prefix in [u'*.', u'.']: - if name.startswith(prefix): - name = name[len(prefix):] - return prefix.encode('ascii') + idna.encode(name) - return idna.encode(name) + try: + for prefix in [u'*.', u'.']: + if name.startswith(prefix): + name = name[len(prefix):] + return prefix.encode('ascii') + idna.encode(name) + return idna.encode(name) + except idna.core.IDNAError: + return None name = idna_encode(name) - if sys.version_info >= (3, 0): + if name is None: + return None + elif sys.version_info >= (3, 0): name = name.decode('utf-8') return name @@ -223,9 +231,10 @@ def get_subj_alt_name(peer_cert): # Sadly the DNS names need to be idna encoded and then, on Python 3, UTF-8 # decoded. This is pretty frustrating, but that's what the standard library # does with certificates, and so we need to attempt to do the same. + # We also want to skip over names which cannot be idna encoded. names = [ - ('DNS', _dnsname_to_stdlib(name)) - for name in ext.get_values_for_type(x509.DNSName) + ('DNS', name) for name in map(_dnsname_to_stdlib, ext.get_values_for_type(x509.DNSName)) + if name is not None ] names.extend( ('IP Address', str(name)) diff --git a/pipenv/patched/notpip/_vendor/urllib3/packages/backports/makefile.py b/pipenv/patched/notpip/_vendor/urllib3/packages/backports/makefile.py index 75b80dcf..740db377 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/packages/backports/makefile.py +++ b/pipenv/patched/notpip/_vendor/urllib3/packages/backports/makefile.py @@ -16,7 +16,7 @@ def backport_makefile(self, mode="r", buffering=None, encoding=None, """ Backport of ``socket.makefile`` from Python 3.5. """ - if not set(mode) <= set(["r", "w", "b"]): + if not set(mode) <= {"r", "w", "b"}: raise ValueError( "invalid mode %r (only r, w, b allowed)" % (mode,) ) diff --git a/pipenv/patched/notpip/_vendor/urllib3/packages/ordered_dict.py b/pipenv/patched/notpip/_vendor/urllib3/packages/ordered_dict.py deleted file mode 100644 index 4479363c..00000000 --- a/pipenv/patched/notpip/_vendor/urllib3/packages/ordered_dict.py +++ /dev/null @@ -1,259 +0,0 @@ -# Backport of OrderedDict() class that runs on Python 2.4, 2.5, 2.6, 2.7 and pypy. -# Passes Python2.7's test suite and incorporates all the latest updates. -# Copyright 2009 Raymond Hettinger, released under the MIT License. -# http://code.activestate.com/recipes/576693/ -try: - from thread import get_ident as _get_ident -except ImportError: - from dummy_thread import get_ident as _get_ident - -try: - from _abcoll import KeysView, ValuesView, ItemsView -except ImportError: - pass - - -class OrderedDict(dict): - 'Dictionary that remembers insertion order' - # An inherited dict maps keys to values. - # The inherited dict provides __getitem__, __len__, __contains__, and get. - # The remaining methods are order-aware. - # Big-O running times for all methods are the same as for regular dictionaries. - - # The internal self.__map dictionary maps keys to links in a doubly linked list. - # The circular doubly linked list starts and ends with a sentinel element. - # The sentinel element never gets deleted (this simplifies the algorithm). - # Each link is stored as a list of length three: [PREV, NEXT, KEY]. - - def __init__(self, *args, **kwds): - '''Initialize an ordered dictionary. Signature is the same as for - regular dictionaries, but keyword arguments are not recommended - because their insertion order is arbitrary. - - ''' - if len(args) > 1: - raise TypeError('expected at most 1 arguments, got %d' % len(args)) - try: - self.__root - except AttributeError: - self.__root = root = [] # sentinel node - root[:] = [root, root, None] - self.__map = {} - self.__update(*args, **kwds) - - def __setitem__(self, key, value, dict_setitem=dict.__setitem__): - 'od.__setitem__(i, y) <==> od[i]=y' - # Setting a new item creates a new link which goes at the end of the linked - # list, and the inherited dictionary is updated with the new key/value pair. - if key not in self: - root = self.__root - last = root[0] - last[1] = root[0] = self.__map[key] = [last, root, key] - dict_setitem(self, key, value) - - def __delitem__(self, key, dict_delitem=dict.__delitem__): - 'od.__delitem__(y) <==> del od[y]' - # Deleting an existing item uses self.__map to find the link which is - # then removed by updating the links in the predecessor and successor nodes. - dict_delitem(self, key) - link_prev, link_next, key = self.__map.pop(key) - link_prev[1] = link_next - link_next[0] = link_prev - - def __iter__(self): - 'od.__iter__() <==> iter(od)' - root = self.__root - curr = root[1] - while curr is not root: - yield curr[2] - curr = curr[1] - - def __reversed__(self): - 'od.__reversed__() <==> reversed(od)' - root = self.__root - curr = root[0] - while curr is not root: - yield curr[2] - curr = curr[0] - - def clear(self): - 'od.clear() -> None. Remove all items from od.' - try: - for node in self.__map.itervalues(): - del node[:] - root = self.__root - root[:] = [root, root, None] - self.__map.clear() - except AttributeError: - pass - dict.clear(self) - - def popitem(self, last=True): - '''od.popitem() -> (k, v), return and remove a (key, value) pair. - Pairs are returned in LIFO order if last is true or FIFO order if false. - - ''' - if not self: - raise KeyError('dictionary is empty') - root = self.__root - if last: - link = root[0] - link_prev = link[0] - link_prev[1] = root - root[0] = link_prev - else: - link = root[1] - link_next = link[1] - root[1] = link_next - link_next[0] = root - key = link[2] - del self.__map[key] - value = dict.pop(self, key) - return key, value - - # -- the following methods do not depend on the internal structure -- - - def keys(self): - 'od.keys() -> list of keys in od' - return list(self) - - def values(self): - 'od.values() -> list of values in od' - return [self[key] for key in self] - - def items(self): - 'od.items() -> list of (key, value) pairs in od' - return [(key, self[key]) for key in self] - - def iterkeys(self): - 'od.iterkeys() -> an iterator over the keys in od' - return iter(self) - - def itervalues(self): - 'od.itervalues -> an iterator over the values in od' - for k in self: - yield self[k] - - def iteritems(self): - 'od.iteritems -> an iterator over the (key, value) items in od' - for k in self: - yield (k, self[k]) - - def update(*args, **kwds): - '''od.update(E, **F) -> None. Update od from dict/iterable E and F. - - If E is a dict instance, does: for k in E: od[k] = E[k] - If E has a .keys() method, does: for k in E.keys(): od[k] = E[k] - Or if E is an iterable of items, does: for k, v in E: od[k] = v - In either case, this is followed by: for k, v in F.items(): od[k] = v - - ''' - if len(args) > 2: - raise TypeError('update() takes at most 2 positional ' - 'arguments (%d given)' % (len(args),)) - elif not args: - raise TypeError('update() takes at least 1 argument (0 given)') - self = args[0] - # Make progressively weaker assumptions about "other" - other = () - if len(args) == 2: - other = args[1] - if isinstance(other, dict): - for key in other: - self[key] = other[key] - elif hasattr(other, 'keys'): - for key in other.keys(): - self[key] = other[key] - else: - for key, value in other: - self[key] = value - for key, value in kwds.items(): - self[key] = value - - __update = update # let subclasses override update without breaking __init__ - - __marker = object() - - def pop(self, key, default=__marker): - '''od.pop(k[,d]) -> v, remove specified key and return the corresponding value. - If key is not found, d is returned if given, otherwise KeyError is raised. - - ''' - if key in self: - result = self[key] - del self[key] - return result - if default is self.__marker: - raise KeyError(key) - return default - - def setdefault(self, key, default=None): - 'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od' - if key in self: - return self[key] - self[key] = default - return default - - def __repr__(self, _repr_running={}): - 'od.__repr__() <==> repr(od)' - call_key = id(self), _get_ident() - if call_key in _repr_running: - return '...' - _repr_running[call_key] = 1 - try: - if not self: - return '%s()' % (self.__class__.__name__,) - return '%s(%r)' % (self.__class__.__name__, self.items()) - finally: - del _repr_running[call_key] - - def __reduce__(self): - 'Return state information for pickling' - items = [[k, self[k]] for k in self] - inst_dict = vars(self).copy() - for k in vars(OrderedDict()): - inst_dict.pop(k, None) - if inst_dict: - return (self.__class__, (items,), inst_dict) - return self.__class__, (items,) - - def copy(self): - 'od.copy() -> a shallow copy of od' - return self.__class__(self) - - @classmethod - def fromkeys(cls, iterable, value=None): - '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S - and values equal to v (which defaults to None). - - ''' - d = cls() - for key in iterable: - d[key] = value - return d - - def __eq__(self, other): - '''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive - while comparison to a regular mapping is order-insensitive. - - ''' - if isinstance(other, OrderedDict): - return len(self)==len(other) and self.items() == other.items() - return dict.__eq__(self, other) - - def __ne__(self, other): - return not self == other - - # -- the following methods are only used in Python 2.7 -- - - def viewkeys(self): - "od.viewkeys() -> a set-like object providing a view on od's keys" - return KeysView(self) - - def viewvalues(self): - "od.viewvalues() -> an object providing a view on od's values" - return ValuesView(self) - - def viewitems(self): - "od.viewitems() -> a set-like object providing a view on od's items" - return ItemsView(self) diff --git a/pipenv/patched/notpip/_vendor/urllib3/packages/ssl_match_hostname/_implementation.py b/pipenv/patched/notpip/_vendor/urllib3/packages/ssl_match_hostname/_implementation.py index 62a177a7..b48752f8 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/packages/ssl_match_hostname/_implementation.py +++ b/pipenv/patched/notpip/_vendor/urllib3/packages/ssl_match_hostname/_implementation.py @@ -9,8 +9,7 @@ import sys # ipaddress has been backported to 2.6+ in pypi. If it is installed on the # system, use it to handle IPAddress ServerAltnames (this was added in # python-3.5) otherwise only do DNS matching. This allows -# backports.ssl_match_hostname to continue to be used all the way back to -# python-2.4. +# backports.ssl_match_hostname to continue to be used in Python 2.7. try: from pipenv.patched.notpip._vendor import ipaddress except ImportError: diff --git a/pipenv/patched/notpip/_vendor/urllib3/poolmanager.py b/pipenv/patched/notpip/_vendor/urllib3/poolmanager.py index 506a3c9b..fe5491cf 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/poolmanager.py +++ b/pipenv/patched/notpip/_vendor/urllib3/poolmanager.py @@ -47,6 +47,7 @@ _key_fields = ( 'key__socks_options', # dict 'key_assert_hostname', # bool or string 'key_assert_fingerprint', # str + 'key_server_hostname', #str ) #: The namedtuple class used to construct keys for the connection pool. diff --git a/pipenv/patched/notpip/_vendor/urllib3/request.py b/pipenv/patched/notpip/_vendor/urllib3/request.py index 1be33341..8f2f44bb 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/request.py +++ b/pipenv/patched/notpip/_vendor/urllib3/request.py @@ -36,7 +36,7 @@ class RequestMethods(object): explicitly. """ - _encode_url_methods = set(['DELETE', 'GET', 'HEAD', 'OPTIONS']) + _encode_url_methods = {'DELETE', 'GET', 'HEAD', 'OPTIONS'} def __init__(self, headers=None): self.headers = headers or {} diff --git a/pipenv/patched/notpip/_vendor/urllib3/response.py b/pipenv/patched/notpip/_vendor/urllib3/response.py index 9873cb94..c112690b 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/response.py +++ b/pipenv/patched/notpip/_vendor/urllib3/response.py @@ -11,7 +11,7 @@ from .exceptions import ( BodyNotHttplibCompatible, ProtocolError, DecodeError, ReadTimeoutError, ResponseNotChunked, IncompleteRead, InvalidHeader ) -from .packages.six import string_types as basestring, binary_type, PY3 +from .packages.six import string_types as basestring, PY3 from .packages.six.moves import http_client as httplib from .connection import HTTPException, BaseSSLError from .util.response import is_fp_closed, is_response_to_head @@ -23,7 +23,7 @@ class DeflateDecoder(object): def __init__(self): self._first_try = True - self._data = binary_type() + self._data = b'' self._obj = zlib.decompressobj() def __getattr__(self, name): @@ -69,9 +69,9 @@ class GzipDecoder(object): return getattr(self._obj, name) def decompress(self, data): - ret = binary_type() + ret = bytearray() if self._state == GzipDecoderState.SWALLOW_DATA or not data: - return ret + return bytes(ret) while True: try: ret += self._obj.decompress(data) @@ -81,16 +81,40 @@ class GzipDecoder(object): self._state = GzipDecoderState.SWALLOW_DATA if previous_state == GzipDecoderState.OTHER_MEMBERS: # Allow trailing garbage acceptable in other gzip clients - return ret + return bytes(ret) raise data = self._obj.unused_data if not data: - return ret + return bytes(ret) self._state = GzipDecoderState.OTHER_MEMBERS self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS) +class MultiDecoder(object): + """ + From RFC7231: + If one or more encodings have been applied to a representation, the + sender that applied the encodings MUST generate a Content-Encoding + header field that lists the content codings in the order in which + they were applied. + """ + + def __init__(self, modes): + self._decoders = [_get_decoder(m.strip()) for m in modes.split(',')] + + def flush(self): + return self._decoders[0].flush() + + def decompress(self, data): + for d in reversed(self._decoders): + data = d.decompress(data) + return data + + def _get_decoder(mode): + if ',' in mode: + return MultiDecoder(mode) + if mode == 'gzip': return GzipDecoder() @@ -159,7 +183,7 @@ class HTTPResponse(io.IOBase): self.msg = msg self._request_url = request_url - if body and isinstance(body, (basestring, binary_type)): + if body and isinstance(body, (basestring, bytes)): self._body = body self._pool = pool @@ -283,8 +307,13 @@ class HTTPResponse(io.IOBase): # Note: content-encoding value should be case-insensitive, per RFC 7230 # Section 3.2 content_encoding = self.headers.get('content-encoding', '').lower() - if self._decoder is None and content_encoding in self.CONTENT_DECODERS: - self._decoder = _get_decoder(content_encoding) + if self._decoder is None: + if content_encoding in self.CONTENT_DECODERS: + self._decoder = _get_decoder(content_encoding) + elif ',' in content_encoding: + encodings = [e.strip() for e in content_encoding.split(',') if e.strip() in self.CONTENT_DECODERS] + if len(encodings): + self._decoder = _get_decoder(content_encoding) def _decode(self, data, decode_content, flush_decoder): """ diff --git a/pipenv/patched/notpip/_vendor/urllib3/util/connection.py b/pipenv/patched/notpip/_vendor/urllib3/util/connection.py index 5cf488f4..5ad70b2f 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/util/connection.py +++ b/pipenv/patched/notpip/_vendor/urllib3/util/connection.py @@ -1,6 +1,7 @@ from __future__ import absolute_import import socket from .wait import NoWayToWaitForSocketError, wait_for_read +from ..contrib import _appengine_environ def is_connection_dropped(conn): # Platform-specific @@ -105,6 +106,13 @@ def _has_ipv6(host): sock = None has_ipv6 = False + # App Engine doesn't support IPV6 sockets and actually has a quota on the + # number of sockets that can be used, so just early out here instead of + # creating a socket needlessly. + # See https://github.com/urllib3/urllib3/issues/1446 + if _appengine_environ.is_appengine_sandbox(): + return False + if socket.has_ipv6: # has_ipv6 returns true if cPython was compiled with IPv6 support. # It does not tell us if the system has IPv6 support enabled. To diff --git a/pipenv/patched/notpip/_vendor/urllib3/util/response.py b/pipenv/patched/notpip/_vendor/urllib3/util/response.py index 67cf730a..3d548648 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/util/response.py +++ b/pipenv/patched/notpip/_vendor/urllib3/util/response.py @@ -59,8 +59,14 @@ def assert_header_parsing(headers): get_payload = getattr(headers, 'get_payload', None) unparsed_data = None - if get_payload: # Platform-specific: Python 3. - unparsed_data = get_payload() + if get_payload: + # get_payload is actually email.message.Message.get_payload; + # we're only interested in the result if it's not a multipart message + if not headers.is_multipart(): + payload = get_payload() + + if isinstance(payload, (bytes, str)): + unparsed_data = payload if defects or unparsed_data: raise HeaderParsingError(defects=defects, unparsed_data=unparsed_data) diff --git a/pipenv/patched/notpip/_vendor/urllib3/util/retry.py b/pipenv/patched/notpip/_vendor/urllib3/util/retry.py index 7ad3dc66..e7d0abd6 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/util/retry.py +++ b/pipenv/patched/notpip/_vendor/urllib3/util/retry.py @@ -115,7 +115,7 @@ class Retry(object): (most errors are resolved immediately by a second try without a delay). urllib3 will sleep for:: - {backoff factor} * (2 ^ ({number of total retries} - 1)) + {backoff factor} * (2 ** ({number of total retries} - 1)) seconds. If the backoff_factor is 0.1, then :func:`.sleep` will sleep for [0.0s, 0.2s, 0.4s, ...] between retries. It will never be longer diff --git a/pipenv/patched/notpip/_vendor/urllib3/util/ssl_.py b/pipenv/patched/notpip/_vendor/urllib3/util/ssl_.py index a0868c22..b16d6523 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/util/ssl_.py +++ b/pipenv/patched/notpip/_vendor/urllib3/util/ssl_.py @@ -56,9 +56,8 @@ except ImportError: OP_NO_COMPRESSION = 0x20000 -# Python 2.7 and earlier didn't have inet_pton on non-Linux -# so we fallback on inet_aton in those cases. This means that -# we can only detect IPv4 addresses in this case. +# Python 2.7 doesn't have inet_pton on non-Linux so we fallback on inet_aton in +# those cases. This means that we can only detect IPv4 addresses in this case. if hasattr(socket, 'inet_pton'): inet_pton = socket.inet_pton else: @@ -67,7 +66,7 @@ else: from pipenv.patched.notpip._vendor import ipaddress def inet_pton(_, host): - if isinstance(host, six.binary_type): + if isinstance(host, bytes): host = host.decode('ascii') return ipaddress.ip_address(host) @@ -115,10 +114,7 @@ try: except ImportError: import sys - class SSLContext(object): # Platform-specific: Python 2 & 3.1 - supports_set_ciphers = ((2, 7) <= sys.version_info < (3,) or - (3, 2) <= sys.version_info) - + class SSLContext(object): # Platform-specific: Python 2 def __init__(self, protocol_version): self.protocol = protocol_version # Use default values from a real SSLContext @@ -141,12 +137,6 @@ except ImportError: raise SSLError("CA directories not supported in older Pythons") def set_ciphers(self, cipher_suite): - if not self.supports_set_ciphers: - raise TypeError( - 'Your version of Python does not support setting ' - 'a custom cipher suite. Please upgrade to Python ' - '2.7, 3.2, or later if you need this functionality.' - ) self.ciphers = cipher_suite def wrap_socket(self, socket, server_hostname=None, server_side=False): @@ -167,10 +157,7 @@ except ImportError: 'ssl_version': self.protocol, 'server_side': server_side, } - if self.supports_set_ciphers: # Platform-specific: Python 2.7+ - return wrap_socket(socket, ciphers=self.ciphers, **kwargs) - else: # Platform-specific: Python 2.6 - return wrap_socket(socket, **kwargs) + return wrap_socket(socket, ciphers=self.ciphers, **kwargs) def assert_fingerprint(cert, fingerprint): @@ -276,6 +263,8 @@ def create_urllib3_context(ssl_version=None, cert_reqs=None, """ context = SSLContext(ssl_version or ssl.PROTOCOL_SSLv23) + context.set_ciphers(ciphers or DEFAULT_CIPHERS) + # Setting the default here, as we may have no ssl module on import cert_reqs = ssl.CERT_REQUIRED if cert_reqs is None else cert_reqs @@ -291,9 +280,6 @@ def create_urllib3_context(ssl_version=None, cert_reqs=None, context.options |= options - if getattr(context, 'supports_set_ciphers', True): # Platform-specific: Python 2.6 - context.set_ciphers(ciphers or DEFAULT_CIPHERS) - context.verify_mode = cert_reqs if getattr(context, 'check_hostname', None) is not None: # Platform-specific: Python 3.2 # We do our own verification, including fingerprints and alternative @@ -316,8 +302,7 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, A pre-made :class:`SSLContext` object. If none is provided, one will be created using :func:`create_urllib3_context`. :param ciphers: - A string of ciphers we wish the client to support. This is not - supported on Python 2.6 as the ssl module does not support it. + A string of ciphers we wish the client to support. :param ca_cert_dir: A directory containing CA certificates in multiple separate files, as supported by OpenSSL's -CApath flag or the capath argument to @@ -334,7 +319,7 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, if ca_certs or ca_cert_dir: try: context.load_verify_locations(ca_certs, ca_cert_dir) - except IOError as e: # Platform-specific: Python 2.6, 2.7, 3.2 + except IOError as e: # Platform-specific: Python 2.7 raise SSLError(e) # Py33 raises FileNotFoundError which subclasses OSError # These are not equivalent unless we check the errno attribute @@ -378,7 +363,7 @@ def is_ipaddress(hostname): :param str hostname: Hostname to examine. :return: True if the hostname is an IP address, False otherwise. """ - if six.PY3 and isinstance(hostname, six.binary_type): + if six.PY3 and isinstance(hostname, bytes): # IDN A-label bytes are ASCII compatible. hostname = hostname.decode('ascii') diff --git a/pipenv/patched/notpip/_vendor/urllib3/util/wait.py b/pipenv/patched/notpip/_vendor/urllib3/util/wait.py index fa686eff..4db71baf 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/util/wait.py +++ b/pipenv/patched/notpip/_vendor/urllib3/util/wait.py @@ -43,9 +43,6 @@ if sys.version_info >= (3, 5): else: # Old and broken Pythons. def _retry_on_intr(fn, timeout): - if timeout is not None and timeout <= 0: - return fn(timeout) - if timeout is None: deadline = float("inf") else: @@ -117,7 +114,7 @@ def _have_working_poll(): # from libraries like eventlet/greenlet. try: poll_obj = select.poll() - poll_obj.poll(0) + _retry_on_intr(poll_obj.poll, 0) except (AttributeError, OSError): return False else: diff --git a/pipenv/patched/notpip/_vendor/vendor.txt b/pipenv/patched/notpip/_vendor/vendor.txt index 9389dd94..7b548255 100644 --- a/pipenv/patched/notpip/_vendor/vendor.txt +++ b/pipenv/patched/notpip/_vendor/vendor.txt @@ -1,23 +1,23 @@ appdirs==1.4.3 -distlib==0.2.7 -distro==1.3.0 -html5lib==1.0.1 -six==1.11.0 -colorama==0.3.9 CacheControl==0.12.5 -msgpack-python==0.5.6 -lockfile==0.12.2 -progress==1.4 +colorama==0.4.1 +distlib==0.2.8 +distro==1.4.0 +html5lib==1.0.1 ipaddress==1.0.22 # Only needed on 2.6 and 2.7 -packaging==18.0 -pep517==0.2 -pyparsing==2.2.1 -pytoml==0.1.19 -retrying==1.3.3 -requests==2.19.1 +lockfile==0.12.2 +msgpack==0.5.6 +packaging==19.0 +pep517==0.5.0 +progress==1.5 +pyparsing==2.4.0 +pytoml==0.1.20 +requests==2.21.0 + certifi==2019.3.9 chardet==3.0.4 - idna==2.7 - urllib3==1.23 - certifi==2018.8.24 -setuptools==40.4.3 + idna==2.8 + urllib3==1.25.2 +retrying==1.3.3 +setuptools==41.0.1 +six==1.12.0 webencodings==0.5.1 diff --git a/pipenv/patched/notpip/appdirs.LICENSE.txt b/pipenv/patched/notpip/appdirs.LICENSE.txt new file mode 100644 index 00000000..107c6140 --- /dev/null +++ b/pipenv/patched/notpip/appdirs.LICENSE.txt @@ -0,0 +1,23 @@ +# This is the MIT license + +Copyright (c) 2010 ActiveState Software Inc. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + diff --git a/pipenv/patched/notpip/distro.LICENSE b/pipenv/patched/notpip/distro.LICENSE new file mode 100644 index 00000000..e06d2081 --- /dev/null +++ b/pipenv/patched/notpip/distro.LICENSE @@ -0,0 +1,202 @@ +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/pipenv/patched/notpip/ipaddress.LICENSE b/pipenv/patched/notpip/ipaddress.LICENSE new file mode 100644 index 00000000..41bd16ba --- /dev/null +++ b/pipenv/patched/notpip/ipaddress.LICENSE @@ -0,0 +1,50 @@ +This package is a modified version of cpython's ipaddress module. +It is therefore distributed under the PSF license, as follows: + +PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 +-------------------------------------------- + +1. This LICENSE AGREEMENT is between the Python Software Foundation +("PSF"), and the Individual or Organization ("Licensee") accessing and +otherwise using this software ("Python") in source or binary form and +its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, PSF hereby +grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, +analyze, test, perform and/or display publicly, prepare derivative works, +distribute, and otherwise use Python alone or in any derivative version, +provided, however, that PSF's License Agreement and PSF's notice of copyright, +i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, +2011, 2012, 2013, 2014 Python Software Foundation; All Rights Reserved" are +retained in Python alone or in any derivative version prepared by Licensee. + +3. In the event Licensee prepares a derivative work that is based on +or incorporates Python or any part thereof, and wants to make +the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to Python. + +4. PSF is making Python available to Licensee on an "AS IS" +basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON +FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS +A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, +OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. Nothing in this License Agreement shall be deemed to create any +relationship of agency, partnership, or joint venture between PSF and +Licensee. This License Agreement does not grant permission to use PSF +trademarks or trade name in a trademark sense to endorse or promote +products or services of Licensee, or any third party. + +8. By copying, installing or otherwise using Python, Licensee +agrees to be bound by the terms and conditions of this License +Agreement. diff --git a/pipenv/patched/notpip/packaging.LICENSE.APACHE b/pipenv/patched/notpip/packaging.LICENSE.APACHE new file mode 100644 index 00000000..4947287f --- /dev/null +++ b/pipenv/patched/notpip/packaging.LICENSE.APACHE @@ -0,0 +1,177 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS \ No newline at end of file diff --git a/pipenv/patched/notpip/pyparsing.LICENSE b/pipenv/patched/notpip/pyparsing.LICENSE new file mode 100644 index 00000000..1bf98523 --- /dev/null +++ b/pipenv/patched/notpip/pyparsing.LICENSE @@ -0,0 +1,18 @@ +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/pipenv/patched/notpip/retrying.LICENSE b/pipenv/patched/notpip/retrying.LICENSE new file mode 100644 index 00000000..7a4a3ea2 --- /dev/null +++ b/pipenv/patched/notpip/retrying.LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/pipenv/patched/notpip/six.LICENSE b/pipenv/patched/notpip/six.LICENSE new file mode 100644 index 00000000..365d1074 --- /dev/null +++ b/pipenv/patched/notpip/six.LICENSE @@ -0,0 +1,18 @@ +Copyright (c) 2010-2018 Benjamin Peterson + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/pipenv/patched/notpip/urllib3.LICENSE b/pipenv/patched/notpip/urllib3.LICENSE new file mode 100644 index 00000000..1c3283ee --- /dev/null +++ b/pipenv/patched/notpip/urllib3.LICENSE @@ -0,0 +1,19 @@ +This is the MIT license: http://www.opensource.org/licenses/mit-license.php + +Copyright 2008-2016 Andrey Petrov and contributors (see CONTRIBUTORS.txt) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this +software and associated documentation files (the "Software"), to deal in the Software +without restriction, including without limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons +to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or +substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, +INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR +PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE +FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/pipenv/patched/notpip/webencodings.LICENSE b/pipenv/patched/notpip/webencodings.LICENSE new file mode 100644 index 00000000..3d0d3e70 --- /dev/null +++ b/pipenv/patched/notpip/webencodings.LICENSE @@ -0,0 +1,31 @@ +Copyright (c) 2012 by Simon Sapin. + +Some rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * The names of the contributors may not be used to endorse or + promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/pipenv/patched/patched.txt b/pipenv/patched/patched.txt index e7dadd8e..e34df9fd 100644 --- a/pipenv/patched/patched.txt +++ b/pipenv/patched/patched.txt @@ -1,5 +1,5 @@ safety crayons==0.1.2 pipfile==0.0.2 -pip-tools==3.1.0 -pip==18.1 +pip-tools==3.5.0 +pip==19.0.3 diff --git a/pipenv/patched/piptools/__main__.py b/pipenv/patched/piptools/__main__.py index 22bd7879..b08b8494 100644 --- a/pipenv/patched/piptools/__main__.py +++ b/pipenv/patched/piptools/__main__.py @@ -12,5 +12,5 @@ cli.add_command(sync.cli, 'sync') # Enable ``python -m piptools ...``. -if __name__ == '__main__': # pragma: no cover +if __name__ == '__main__': # pragma: no branch cli() diff --git a/pipenv/patched/piptools/_compat/__init__.py b/pipenv/patched/piptools/_compat/__init__.py index c0ecec8a..19adcbc5 100644 --- a/pipenv/patched/piptools/_compat/__init__.py +++ b/pipenv/patched/piptools/_compat/__init__.py @@ -27,8 +27,10 @@ from .pip_compat import ( cmdoptions, get_installed_distributions, PyPI, - SafeFileCache, - InstallationError, install_req_from_line, install_req_from_editable, + stdlib_pkgs, + DEV_PKGS, + SafeFileCache, + InstallationError ) diff --git a/pipenv/patched/piptools/_compat/pip_compat.py b/pipenv/patched/piptools/_compat/pip_compat.py index c466ef04..715144a3 100644 --- a/pipenv/patched/piptools/_compat/pip_compat.py +++ b/pipenv/patched/piptools/_compat/pip_compat.py @@ -1,12 +1,11 @@ # -*- coding=utf-8 -*- - __all__ = [ "InstallRequirement", "parse_requirements", "RequirementSet", - "user_cache_dir", "FAVORITE_HASH", "is_file_url", + "path_to_url", "url_to_path", "PackageFinder", "FormatControl", @@ -15,22 +14,25 @@ __all__ = [ "cmdoptions", "get_installed_distributions", "PyPI", - "SafeFileCache", - "InstallationError", - "parse_version", - "pip_version", - "install_req_from_editable", + "stdlib_pkgs", + "DEV_PKGS", "install_req_from_line", - "user_cache_dir" + "install_req_from_editable", + "user_cache_dir", + "SafeFileCache", + "InstallationError" ] -from pipenv.vendor.appdirs import user_cache_dir +import os +os.environ["PIP_SHIMS_BASE_MODULE"] = str("pipenv.patched.notpip") + from pip_shims.shims import ( InstallRequirement, parse_requirements, RequirementSet, FAVORITE_HASH, is_file_url, + path_to_url, url_to_path, PackageFinder, FormatControl, @@ -39,17 +41,11 @@ from pip_shims.shims import ( cmdoptions, get_installed_distributions, PyPI, + stdlib_pkgs, + DEV_PKGS, + install_req_from_line, + install_req_from_editable, + USER_CACHE_DIR as user_cache_dir, SafeFileCache, - InstallationError, - parse_version, - pip_version, + InstallationError ) - -# pip 18.1 has refactored InstallRequirement constructors use by pip-tools. -if parse_version(pip_version) < parse_version('18.1'): - install_req_from_line = InstallRequirement.from_line - install_req_from_editable = InstallRequirement.from_editable -else: - from pip_shims.shims import ( - install_req_from_editable, install_req_from_line - ) diff --git a/pipenv/patched/piptools/locations.py b/pipenv/patched/piptools/locations.py index 0d460f64..9fcea0af 100644 --- a/pipenv/patched/piptools/locations.py +++ b/pipenv/patched/piptools/locations.py @@ -5,7 +5,11 @@ from .click import secho from ._compat import user_cache_dir # The user_cache_dir helper comes straight from pipenv.patched.notpip itself -CACHE_DIR = user_cache_dir('pip-tools') +try: + from pipenv.environments import PIPENV_CACHE_DIR + CACHE_DIR = PIPENV_CACHE_DIR +except ImportError: + CACHE_DIR = user_cache_dir('pipenv') # NOTE # We used to store the cache dir under ~/.pip-tools, which is not the diff --git a/pipenv/patched/piptools/logging.py b/pipenv/patched/piptools/logging.py index 98f05287..f0bd1784 100644 --- a/pipenv/patched/piptools/logging.py +++ b/pipenv/patched/piptools/logging.py @@ -8,18 +8,19 @@ from . import click class LogContext(object): - def __init__(self, verbose=False): - self.verbose = verbose + def __init__(self, verbosity=0): + self.verbosity = verbosity def log(self, *args, **kwargs): click.secho(*args, **kwargs) def debug(self, *args, **kwargs): - if self.verbose: + if self.verbosity >= 1: self.log(*args, **kwargs) def info(self, *args, **kwargs): - self.log(*args, **kwargs) + if self.verbosity >= 0: + self.log(*args, **kwargs) def warning(self, *args, **kwargs): kwargs.setdefault('fg', 'yellow') diff --git a/pipenv/patched/piptools/pip.py b/pipenv/patched/piptools/pip.py new file mode 100644 index 00000000..0419a8ab --- /dev/null +++ b/pipenv/patched/piptools/pip.py @@ -0,0 +1,30 @@ +import optparse + +from ._compat import Command, cmdoptions + + +class PipCommand(Command): + name = 'PipCommand' + + +def get_pip_command(): + # Use pip's parser for pip.conf management and defaults. + # General options (find_links, index_url, extra_index_url, trusted_host, + # and pre) are defered to pip. + pip_command = PipCommand() + pip_command.parser.add_option(cmdoptions.no_binary()) + pip_command.parser.add_option(cmdoptions.only_binary()) + index_opts = cmdoptions.make_option_group( + cmdoptions.index_group, + pip_command.parser, + ) + pip_command.parser.insert_option_group(0, index_opts) + pip_command.parser.add_option(optparse.Option('--pre', action='store_true', default=False)) + + return pip_command + + +pip_command = get_pip_command() + +# Get default values of the pip's options (including options from pipenv.patched.notpip.conf). +pip_defaults = pip_command.parser.get_default_values() diff --git a/pipenv/patched/piptools/repositories/local.py b/pipenv/patched/piptools/repositories/local.py index 480ad1ed..36bafdb9 100644 --- a/pipenv/patched/piptools/repositories/local.py +++ b/pipenv/patched/piptools/repositories/local.py @@ -56,7 +56,8 @@ class LocalRequirementsRepository(BaseRepository): if existing_pin and ireq_satisfied_by_existing_pin(ireq, existing_pin): project, version, _ = as_tuple(existing_pin) return make_install_requirement( - project, version, ireq.extras, constraint=ireq.constraint, markers=ireq.markers + project, version, ireq.extras, constraint=ireq.constraint, + markers=ireq.markers ) else: return self.repository.find_best_match(ireq, prereleases) diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py index 2a0743a3..10a0e469 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -1,12 +1,23 @@ # coding: utf-8 from __future__ import (absolute_import, division, print_function, unicode_literals) + import copy import hashlib import os from contextlib import contextmanager from shutil import rmtree +import pkg_resources + +from packaging.requirements import Requirement +from packaging.specifiers import SpecifierSet, Specifier + +os.environ["PIP_SHIMS_BASE_MODULE"] = str("pipenv.patched.notpip") +import pip_shims +from pip_shims.shims import VcsSupport, WheelCache, InstallationError + + from .._compat import ( is_file_url, url_to_path, @@ -19,18 +30,11 @@ from .._compat import ( InstallRequirement, SafeFileCache ) -os.environ["PIP_SHIMS_BASE_MODULE"] = str("pipenv.patched.notpip") -from pip_shims.shims import do_import, VcsSupport, WheelCache -from packaging.requirements import Requirement -from packaging.specifiers import SpecifierSet, Specifier -InstallationError = do_import(("exceptions.InstallationError", "7.0", "9999")) -from pipenv.patched.notpip._internal.resolve import Resolver as PipResolver - -from pipenv.environments import PIPENV_CACHE_DIR as CACHE_DIR +from ..cache import CACHE_DIR from ..exceptions import NoCandidateFound from ..utils import (fs_str, is_pinned_requirement, lookup_table, dedup, - make_install_requirement, clean_requires_python) + make_install_requirement, clean_requires_python) from .base import BaseRepository try: @@ -89,23 +93,29 @@ class PyPIRepository(BaseRepository): config), but any other PyPI mirror can be used if index_urls is changed/configured on the Finder. """ - def __init__(self, pip_options, session, use_json=False): + def __init__(self, pip_options, session, build_isolation=False, use_json=False): self.session = session - self.use_json = use_json self.pip_options = pip_options + self.build_isolation = build_isolation + self.use_json = use_json index_urls = [pip_options.index_url] + pip_options.extra_index_urls if pip_options.no_index: index_urls = [] - self.finder = PackageFinder( - find_links=pip_options.find_links, - index_urls=index_urls, - trusted_hosts=pip_options.trusted_hosts, - allow_all_prereleases=pip_options.pre, - process_dependency_links=pip_options.process_dependency_links, - session=self.session, - ) + finder_kwargs = { + "find_links": pip_options.find_links, + "index_urls": index_urls, + "trusted_hosts": pip_options.trusted_hosts, + "allow_all_prereleases": pip_options.pre, + "session": self.session, + } + + # pip 19.0 has removed process_dependency_links from the PackageFinder constructor + if pkg_resources.parse_version(pip_shims.shims.pip_version) < pkg_resources.parse_version('19.0'): + finder_kwargs["process_dependency_links"] = pip_options.process_dependency_links + + self.finder = PackageFinder(**finder_kwargs) # Caches # stores project_name => InstallationCandidate mappings for all @@ -165,7 +175,7 @@ class PyPIRepository(BaseRepository): candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version, unique=True) try: matching_versions = ireq.specifier.filter((candidate.version for candidate in all_candidates), - prereleases=prereleases) + prereleases=prereleases) except TypeError: matching_versions = [candidate.version for candidate in all_candidates] @@ -177,8 +187,22 @@ class PyPIRepository(BaseRepository): # Turn the candidate into a pinned InstallRequirement return make_install_requirement( - best_candidate.project, best_candidate.version, ireq.extras, ireq.markers, constraint=ireq.constraint - ) + best_candidate.project, best_candidate.version, ireq.extras, ireq.markers, constraint=ireq.constraint + ) + + def get_dependencies(self, ireq): + json_results = set() + + if self.use_json: + try: + json_results = self.get_json_dependencies(ireq) + except TypeError: + json_results = set() + + legacy_results = self.get_legacy_dependencies(ireq) + json_results.update(legacy_results) + + return json_results def get_json_dependencies(self, ireq): @@ -222,31 +246,18 @@ class PyPIRepository(BaseRepository): except Exception: return set() - def get_dependencies(self, ireq): - json_results = set() - - if self.use_json: - try: - json_results = self.get_json_dependencies(ireq) - except TypeError: - json_results = set() - - legacy_results = self.get_legacy_dependencies(ireq) - json_results.update(legacy_results) - - return json_results - - def resolve_reqs(self, download_dir, ireq, wheel_cache, setup_requires={}, dist=None): + def resolve_reqs(self, download_dir, ireq, wheel_cache): results = None - setup_requires = {} - dist = None - ireq.isolated = False + ireq.isolated = self.build_isolation ireq._wheel_cache = wheel_cache - + if ireq and not ireq.link: + ireq.populate_link(self.finder, False, False) + if ireq.link and not ireq.link.is_wheel: + ireq.ensure_has_source_dir(self.source_dir) try: from pipenv.patched.notpip._internal.operations.prepare import RequirementPreparer except ImportError: - # Pip 9 and below + # Pip 9 and below reqset = RequirementSet( self.build_dir, self.source_dir, @@ -266,20 +277,21 @@ class PyPIRepository(BaseRepository): 'download_dir': download_dir, 'wheel_download_dir': self._wheel_download_dir, 'progress_bar': 'off', - 'build_isolation': False + 'build_isolation': self.build_isolation, } resolver_kwargs = { 'finder': self.finder, 'session': self.session, 'upgrade_strategy': "to-satisfy-only", - 'force_reinstall': True, + 'force_reinstall': False, 'ignore_dependencies': False, 'ignore_requires_python': True, 'ignore_installed': True, - 'isolated': False, + 'ignore_compatibility': False, + 'isolated': True, 'wheel_cache': wheel_cache, 'use_user_site': False, - 'ignore_compatibility': False + 'use_pep517': True } resolver = None preparer = None @@ -292,7 +304,7 @@ class PyPIRepository(BaseRepository): reqset = RequirementSet() ireq.is_direct = True # reqset.add_requirement(ireq) - resolver = PipResolver(**resolver_kwargs) + resolver = pip_shims.shims.Resolver(**resolver_kwargs) resolver.require_hashes = False results = resolver._resolve_one(reqset, ireq) @@ -303,93 +315,6 @@ class PyPIRepository(BaseRepository): except OSError: pass - if ireq.editable and (not ireq.source_dir or not os.path.exists(ireq.source_dir)): - if ireq.editable: - self._source_dir = TemporaryDirectory(fs_str("source")) - ireq.ensure_has_source_dir(self.source_dir) - - if ireq.editable and (ireq.source_dir and os.path.exists(ireq.source_dir)): - # Collect setup_requires info from local eggs. - # Do this after we call the preparer on these reqs to make sure their - # egg info has been created - from pipenv.utils import chdir - with chdir(ireq.setup_py_dir): - try: - from setuptools.dist import distutils - dist = distutils.core.run_setup(ireq.setup_py) - except InstallationError: - ireq.run_egg_info() - except (TypeError, ValueError, AttributeError): - pass - if not dist: - try: - dist = ireq.get_dist() - except (ImportError, ValueError, TypeError, AttributeError): - pass - if ireq.editable and dist: - setup_requires = getattr(dist, "extras_require", None) - if not setup_requires: - setup_requires = {"setup_requires": getattr(dist, "setup_requires", None)} - if not getattr(ireq, 'req', None): - try: - ireq.req = dist.as_requirement() if dist else None - except (ValueError, TypeError) as e: - pass - - # Convert setup_requires dict into a somewhat usable form. - if setup_requires: - for section in setup_requires: - python_version = section - not_python = not (section.startswith('[') and ':' in section) - - # This is for cleaning up :extras: formatted markers - # by adding them to the results of the resolver - # since any such extra would have been returned as a result anyway - for value in setup_requires[section]: - # This is a marker. - if value.startswith('[') and ':' in value: - python_version = value[1:-1] - not_python = False - # Strip out other extras. - if value.startswith('[') and ':' not in value: - not_python = True - - if ':' not in value: - try: - if not not_python: - results.add(InstallRequirement.from_line("{0}{1}".format(value, python_version).replace(':', ';'))) - # Anything could go wrong here -- can't be too careful. - except Exception: - pass - - # this section properly creates 'python_version' markers for cross-python - # virtualenv creation and for multi-python compatibility. - requires_python = reqset.requires_python if hasattr(reqset, 'requires_python') else resolver.requires_python - if requires_python: - marker_str = '' - # This corrects a logic error from the previous code which said that if - # we Encountered any 'requires_python' attributes, basically only create a - # single result no matter how many we resolved. This should fix - # a majority of the remaining non-deterministic resolution issues. - if any(requires_python.startswith(op) for op in Specifier._operators.keys()): - # We are checking first if we have leading specifier operator - # if not, we can assume we should be doing a == comparison - specifierset = SpecifierSet(requires_python) - # for multiple specifiers, the correct way to represent that in - # a specifierset is `Requirement('fakepkg; python_version<"3.0,>=2.6"')` - from passa.internals.specifiers import cleanup_pyspecs - marker_str = str(Marker(" and ".join(dedup([ - "python_version {0[0]} '{0[1]}'".format(spec) - for spec in cleanup_pyspecs(specifierset) - ])))) - # The best way to add markers to a requirement is to make a separate requirement - # with only markers on it, and then to transfer the object istelf - marker_to_add = Requirement('fakepkg; {0}'.format(marker_str)).marker - if ireq in results: - results.remove(ireq) - print(marker_to_add) - ireq.req.marker = marker_to_add - results = set(results) if results else set() return results, ireq @@ -408,7 +333,6 @@ class PyPIRepository(BaseRepository): # If a download_dir is passed, pip will unnecessarely # archive the entire source directory download_dir = None - elif ireq.link and not ireq.link.is_artifact: # No download_dir for VCS sources. This also works around pip # using git-checkout-index, which gets rid of the .git dir. @@ -457,20 +381,14 @@ class PyPIRepository(BaseRepository): # We need to get all of the candidates that match our current version # pin, these will represent all of the files that could possibly # satisfy this constraint. - ### Modification -- this is much more efficient.... - ### modification again -- still more efficient matching_candidates = ( c for c in clean_requires_python(self.find_all_candidates(ireq.name)) if c.version in ireq.specifier ) - # candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version) - # matching_versions = list( - # ireq.specifier.filter((candidate.version for candidate in all_candidates))) - # matching_candidates = candidates_by_version[matching_versions[0]] return { h for h in map(lambda c: self._hash_cache.get_hash(c.location), - matching_candidates) if h is not None + matching_candidates) if h is not None } @contextmanager diff --git a/pipenv/patched/piptools/resolver.py b/pipenv/patched/piptools/resolver.py index 1f3e18c1..b642bc9c 100644 --- a/pipenv/patched/piptools/resolver.py +++ b/pipenv/patched/piptools/resolver.py @@ -13,7 +13,7 @@ from . import click from .cache import DependencyCache from .exceptions import UnsupportedConstraint from .logging import log -from .utils import (format_requirement, format_specifier, full_groupby, dedup, simplify_markers, +from .utils import (format_requirement, format_specifier, full_groupby, is_pinned_requirement, key_from_ireq, key_from_req, UNSAFE_PACKAGES) green = partial(click.style, fg='green') @@ -27,8 +27,8 @@ class RequirementSummary(object): def __init__(self, ireq): self.req = ireq.req self.key = key_from_req(ireq.req) - self.markers = ireq.markers self.extras = str(sorted(ireq.extras)) + self.markers = ireq.markers self.specifier = str(ireq.specifier) def __eq__(self, other): @@ -162,7 +162,6 @@ class Resolver(object): _markers = combined_ireq.markers._markers if not isinstance(_markers[0], (tuple, list)): combined_ireq.markers._markers = [_markers, 'and', ireq.markers._markers] - # Return a sorted, de-duped tuple of extras combined_ireq.extras = tuple(sorted(set(tuple(combined_ireq.extras) + tuple(ireq.extras)))) yield combined_ireq diff --git a/pipenv/patched/piptools/scripts/compile.py b/pipenv/patched/piptools/scripts/compile.py index 4625618a..2eaea9b3 100644 --- a/pipenv/patched/piptools/scripts/compile.py +++ b/pipenv/patched/piptools/scripts/compile.py @@ -2,7 +2,6 @@ from __future__ import (absolute_import, division, print_function, unicode_literals) -import optparse import os import sys import tempfile @@ -10,33 +9,30 @@ import tempfile from .._compat import ( install_req_from_line, parse_requirements, - cmdoptions, - Command, ) from .. import click from ..exceptions import PipToolsError from ..logging import log +from ..pip import get_pip_command, pip_defaults from ..repositories import LocalRequirementsRepository, PyPIRepository from ..resolver import Resolver from ..utils import (dedup, is_pinned_requirement, key_from_req, UNSAFE_PACKAGES) from ..writer import OutputWriter DEFAULT_REQUIREMENTS_FILE = 'requirements.in' - - -class PipCommand(Command): - name = 'PipCommand' +DEFAULT_REQUIREMENTS_OUTPUT_FILE = 'requirements.txt' @click.command() @click.version_option() -@click.option('-v', '--verbose', is_flag=True, help="Show more output") +@click.option('-v', '--verbose', count=True, help="Show more output") +@click.option('-q', '--quiet', count=True, help="Give less output") @click.option('-n', '--dry-run', is_flag=True, help="Only show what would happen, don't change anything") @click.option('-p', '--pre', is_flag=True, default=None, help="Allow resolving to prereleases (default is not)") @click.option('-r', '--rebuild', is_flag=True, help="Clear any caches upfront, rebuild from scratch") @click.option('-f', '--find-links', multiple=True, help="Look for archives in this directory or on this HTML page", envvar='PIP_FIND_LINKS') # noqa -@click.option('-i', '--index-url', help="Change index URL (defaults to PyPI)", envvar='PIP_INDEX_URL') +@click.option('-i', '--index-url', help="Change index URL (defaults to {})".format(pip_defaults.index_url), envvar='PIP_INDEX_URL') # noqa @click.option('--extra-index-url', multiple=True, help="Add additional index URL to search", envvar='PIP_EXTRA_INDEX_URL') # noqa @click.option('--cert', help="Path to alternate CA bundle.") @click.option('--client-cert', help="Path to SSL client certificate, a single file containing the private key and the certificate in PEM format.") # noqa @@ -65,27 +61,30 @@ class PipCommand(Command): @click.option('--max-rounds', default=10, help="Maximum number of rounds before resolving the requirements aborts.") @click.argument('src_files', nargs=-1, type=click.Path(exists=True, allow_dash=True)) -def cli(verbose, dry_run, pre, rebuild, find_links, index_url, extra_index_url, +@click.option('--build-isolation/--no-build-isolation', is_flag=True, default=False, + help="Enable isolation when building a modern source distribution. " + "Build dependencies specified by PEP 518 must be already installed " + "if build isolation is disabled.") +def cli(verbose, quiet, dry_run, pre, rebuild, find_links, index_url, extra_index_url, cert, client_cert, trusted_host, header, index, emit_trusted_host, annotate, upgrade, upgrade_packages, output_file, allow_unsafe, generate_hashes, - src_files, max_rounds): + src_files, max_rounds, build_isolation): """Compiles requirements.txt from requirements.in specs.""" - log.verbose = verbose + log.verbosity = verbose - quiet if len(src_files) == 0: if os.path.exists(DEFAULT_REQUIREMENTS_FILE): src_files = (DEFAULT_REQUIREMENTS_FILE,) elif os.path.exists('setup.py'): src_files = ('setup.py',) - if not output_file: - output_file = 'requirements.txt' else: raise click.BadParameter(("If you do not specify an input file, " "the default is {} or setup.py").format(DEFAULT_REQUIREMENTS_FILE)) - if len(src_files) == 1 and src_files[0] == '-': - if not output_file: - raise click.BadParameter('--output-file is required if input is from stdin') + if src_files == ('-',) and not output_file: + raise click.BadParameter('--output-file is required if input is from stdin') + elif src_files == ('setup.py',) and not output_file: + output_file = DEFAULT_REQUIREMENTS_OUTPUT_FILE if len(src_files) > 1 and not output_file: raise click.BadParameter('--output-file is required if two or more input files are given.') @@ -127,17 +126,20 @@ def cli(verbose, dry_run, pre, rebuild, find_links, index_url, extra_index_url, pip_options, _ = pip_command.parse_args(pip_args) session = pip_command._build_session(pip_options) - repository = PyPIRepository(pip_options, session) + repository = PyPIRepository(pip_options, session, build_isolation) + upgrade_install_reqs = {} # Proxy with a LocalRequirementsRepository if --upgrade is not specified # (= default invocation) if not upgrade and os.path.exists(dst_file): ireqs = parse_requirements(dst_file, finder=repository.finder, session=repository.session, options=pip_options) # Exclude packages from --upgrade-package/-P from the existing pins: We want to upgrade. - upgrade_pkgs_key = {key_from_req(install_req_from_line(pkg).req) for pkg in upgrade_packages} + upgrade_reqs_gen = (install_req_from_line(pkg) for pkg in upgrade_packages) + upgrade_install_reqs = {key_from_req(install_req.req): install_req for install_req in upgrade_reqs_gen} + existing_pins = {key_from_req(ireq.req): ireq for ireq in ireqs - if is_pinned_requirement(ireq) and key_from_req(ireq.req) not in upgrade_pkgs_key} + if is_pinned_requirement(ireq) and key_from_req(ireq.req) not in upgrade_install_reqs} repository = LocalRequirementsRepository(existing_pins, repository) log.debug('Using indexes:') @@ -178,6 +180,8 @@ def cli(verbose, dry_run, pre, rebuild, find_links, index_url, extra_index_url, constraints.extend(parse_requirements( src_file, finder=repository.finder, session=repository.session, options=pip_options)) + constraints.extend(upgrade_install_reqs.values()) + # Filter out pip environment markers which do not match (PEP496) constraints = [req for req in constraints if req.markers is None or req.markers.evaluate()] @@ -236,32 +240,15 @@ def cli(verbose, dry_run, pre, rebuild, find_links, index_url, extra_index_url, default_index_url=repository.DEFAULT_INDEX_URL, index_urls=repository.finder.index_urls, trusted_hosts=pip_options.trusted_hosts, - format_control=repository.finder.format_control) + format_control=repository.finder.format_control, + allow_unsafe=allow_unsafe) writer.write(results=results, unsafe_requirements=resolver.unsafe_constraints, reverse_dependencies=reverse_dependencies, primary_packages={key_from_req(ireq.req) for ireq in constraints if not ireq.constraint}, markers={key_from_req(ireq.req): ireq.markers for ireq in constraints if ireq.markers}, - hashes=hashes, - allow_unsafe=allow_unsafe) + hashes=hashes) if dry_run: log.warning('Dry-run, so nothing updated.') - - -def get_pip_command(): - # Use pip's parser for pip.conf management and defaults. - # General options (find_links, index_url, extra_index_url, trusted_host, - # and pre) are defered to pip. - pip_command = PipCommand() - pip_command.parser.add_option(cmdoptions.no_binary()) - pip_command.parser.add_option(cmdoptions.only_binary()) - index_opts = cmdoptions.make_option_group( - cmdoptions.index_group, - pip_command.parser, - ) - pip_command.parser.insert_option_group(0, index_opts) - pip_command.parser.add_option(optparse.Option('--pre', action='store_true', default=False)) - - return pip_command diff --git a/pipenv/patched/piptools/sync.py b/pipenv/patched/piptools/sync.py index 5c473916..f111764e 100644 --- a/pipenv/patched/piptools/sync.py +++ b/pipenv/patched/piptools/sync.py @@ -1,11 +1,13 @@ import collections import os import sys +import tempfile from subprocess import check_call +from piptools._compat import stdlib_pkgs, DEV_PKGS from . import click from .exceptions import IncompatibleRequirements, UnsupportedConstraint -from .utils import flat_map, format_requirement, key_from_ireq, key_from_req +from .utils import flat_map, format_requirement, key_from_ireq, key_from_req, get_hashes_from_ireq PACKAGES_TO_IGNORE = [ '-markerlib', @@ -13,9 +15,7 @@ PACKAGES_TO_IGNORE = [ 'pip-tools', 'pip-review', 'pkg-resources', - 'setuptools', - 'wheel', -] +] + list(stdlib_pkgs) + list(DEV_PKGS) def dependency_tree(installed_keys, root_key): @@ -120,33 +120,24 @@ def diff(compiled_requirements, installed_dists): return (to_install, to_uninstall) -def sync(to_install, to_uninstall, verbose=False, dry_run=False, pip_flags=None, install_flags=None): +def sync(to_install, to_uninstall, verbose=False, dry_run=False, install_flags=None): """ Install and uninstalls the given sets of modules. """ if not to_uninstall and not to_install: click.echo("Everything up-to-date") - if pip_flags is None: - pip_flags = [] - + pip_flags = [] if not verbose: pip_flags += ['-q'] - if os.environ.get('VIRTUAL_ENV'): - # find pip via PATH - pip = 'pip' - else: - # find pip in same directory as pip-sync entry-point script - pip = os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), 'pip') - if to_uninstall: if dry_run: click.echo("Would uninstall:") for pkg in to_uninstall: click.echo(" {}".format(pkg)) else: - check_call([pip, 'uninstall', '-y'] + pip_flags + sorted(to_uninstall)) + check_call([sys.executable, '-m', 'pip', 'uninstall', '-y'] + pip_flags + sorted(to_uninstall)) if to_install: if install_flags is None: @@ -156,11 +147,22 @@ def sync(to_install, to_uninstall, verbose=False, dry_run=False, pip_flags=None, for ireq in to_install: click.echo(" {}".format(format_requirement(ireq))) else: - package_args = [] + # prepare requirement lines + req_lines = [] for ireq in sorted(to_install, key=key_from_ireq): - if ireq.editable: - package_args.extend(['-e', str(ireq.link or ireq.req)]) - else: - package_args.append(str(ireq.req)) - check_call([pip, 'install'] + pip_flags + install_flags + package_args) + ireq_hashes = get_hashes_from_ireq(ireq) + req_lines.append(format_requirement(ireq, hashes=ireq_hashes)) + + # save requirement lines to a temporary file + tmp_req_file = tempfile.NamedTemporaryFile(mode='wt', delete=False) + tmp_req_file.write('\n'.join(req_lines)) + tmp_req_file.close() + + try: + check_call( + [sys.executable, '-m', 'pip', 'install', '-r', tmp_req_file.name] + pip_flags + install_flags + ) + finally: + os.unlink(tmp_req_file.name) + return 0 diff --git a/pipenv/patched/piptools/utils.py b/pipenv/patched/piptools/utils.py index 6f62eb9d..fb846cc4 100644 --- a/pipenv/patched/piptools/utils.py +++ b/pipenv/patched/piptools/utils.py @@ -4,22 +4,24 @@ from __future__ import (absolute_import, division, print_function, import os import sys -import six from itertools import chain, groupby from collections import OrderedDict -from contextlib import contextmanager + +import six + +from pipenv.vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier +from pipenv.vendor.packaging.version import Version, InvalidVersion, parse as parse_version +from pipenv.vendor.packaging.markers import Marker, Op, Value, Variable from ._compat import install_req_from_line from .click import style -from pipenv.patched.notpip._vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier -from pipenv.patched.notpip._vendor.packaging.version import Version, InvalidVersion, parse as parse_version -from pipenv.patched.notpip._vendor.packaging.markers import Marker, Op, Value, Variable UNSAFE_PACKAGES = {'setuptools', 'distribute', 'pip'} + def simplify_markers(ireq): """simplify_markers "This code cleans up markers for a specific :class:`~InstallRequirement`" @@ -68,7 +70,7 @@ def clean_requires_python(candidates): all_candidates = [] py_version = parse_version(os.environ.get('PIP_PYTHON_VERSION', '.'.join(map(str, sys.version_info[:3])))) for c in candidates: - if c.requires_python: + if getattr(c, "requires_python", None): # Old specifications had people setting this to single digits # which is effectively the same as '>=digit,>> assert lookup_table( + ... ['foo', 'bar', 'baz', 'qux', 'quux'], lambda s: s[0], + ... use_lists=True) == { + ... 'b': ['bar', 'baz'], + ... 'f': ['foo'], + ... 'q': ['qux', 'quux'] + ... } + The values of the resulting lookup table will be values, not sets. For extra power, you can even change the values while building up the LUT. @@ -336,15 +356,14 @@ def fs_str(string): _fs_encoding = sys.getfilesystemencoding() or sys.getdefaultencoding() -# Borrowed from pew to avoid importing pew which imports psutil -# See https://github.com/berdario/pew/blob/master/pew/_utils.py#L82 -@contextmanager -def temp_environ(): - """Allow the ability to set os.environ temporarily""" - environ = dict(os.environ) - try: - yield - - finally: - os.environ.clear() - os.environ.update(environ) +def get_hashes_from_ireq(ireq): + """ + Given an InstallRequirement, return a list of string hashes in the format "{algorithm}:{hash}". + Return an empty list if there are no hashes in the requirement options. + """ + result = [] + ireq_hashes = ireq.options.get('hashes', {}) + for algorithm, hexdigests in ireq_hashes.items(): + for hash_ in hexdigests: + result.append("{}:{}".format(algorithm, hash_)) + return result diff --git a/pipenv/patched/piptools/writer.py b/pipenv/patched/piptools/writer.py index 97b6df94..9ac97792 100644 --- a/pipenv/patched/piptools/writer.py +++ b/pipenv/patched/piptools/writer.py @@ -1,8 +1,9 @@ import os +import sys from itertools import chain from ._compat import ExitStack -from .click import unstyle +from .click import unstyle, get_os_args from .io import AtomicSaver from .logging import log from .utils import comment, dedup, format_requirement, key_from_req, UNSAFE_PACKAGES @@ -11,7 +12,8 @@ from .utils import comment, dedup, format_requirement, key_from_req, UNSAFE_PACK class OutputWriter(object): def __init__(self, src_files, dst_file, dry_run, emit_header, emit_index, emit_trusted_host, annotate, generate_hashes, - default_index_url, index_urls, trusted_hosts, format_control): + default_index_url, index_urls, trusted_hosts, format_control, + allow_unsafe): self.src_files = src_files self.dst_file = dst_file self.dry_run = dry_run @@ -24,6 +26,7 @@ class OutputWriter(object): self.index_urls = index_urls self.trusted_hosts = trusted_hosts self.format_control = format_control + self.allow_unsafe = allow_unsafe def _sort_key(self, ireq): return (not ireq.editable, str(ireq.req).lower()) @@ -38,18 +41,9 @@ class OutputWriter(object): if custom_cmd: yield comment('# {}'.format(custom_cmd)) else: - params = [] - if not self.emit_index: - params += ['--no-index'] - if not self.emit_trusted_host: - params += ['--no-emit-trusted-host'] - if not self.annotate: - params += ['--no-annotate'] - if self.generate_hashes: - params += ["--generate-hashes"] - params += ['--output-file', self.dst_file] - params += self.src_files - yield comment('# pip-compile {}'.format(' '.join(params))) + prog = os.path.basename(sys.argv[0]) + args = ' '.join(get_os_args()) + yield comment('# {prog} {args}'.format(prog=prog, args=args)) yield comment('#') def write_index_options(self): @@ -82,7 +76,7 @@ class OutputWriter(object): yield '' def _iter_lines(self, results, unsafe_requirements, reverse_dependencies, - primary_packages, markers, hashes, allow_unsafe=False): + primary_packages, markers, hashes): for line in self.write_header(): yield line for line in self.write_flags(): @@ -110,32 +104,29 @@ class OutputWriter(object): primary_packages, marker=markers.get(key_from_req(ireq.req)), hashes=hashes) - if not allow_unsafe: + if not self.allow_unsafe: yield comment('# {}'.format(req)) else: yield req def write(self, results, unsafe_requirements, reverse_dependencies, - primary_packages, markers, hashes, allow_unsafe=False): + primary_packages, markers, hashes): with ExitStack() as stack: f = None if not self.dry_run: f = stack.enter_context(AtomicSaver(self.dst_file)) for line in self._iter_lines(results, unsafe_requirements, reverse_dependencies, - primary_packages, markers, hashes, allow_unsafe=allow_unsafe): + primary_packages, markers, hashes): log.info(line) if f: f.write(unstyle(line).encode('utf-8')) f.write(os.linesep.encode('utf-8')) def _format_requirement(self, ireq, reverse_dependencies, primary_packages, marker=None, hashes=None): - line = format_requirement(ireq, marker=marker) - ireq_hashes = (hashes if hashes is not None else {}).get(ireq) - if ireq_hashes: - for hash_ in sorted(ireq_hashes): - line += " \\\n --hash={}".format(hash_) + + line = format_requirement(ireq, marker=marker, hashes=ireq_hashes) if not self.annotate or key_from_req(ireq.req) in primary_packages: return line diff --git a/pipenv/patched/safety/__init__.py b/pipenv/patched/safety/__init__.py index 56b497d2..69563274 100644 --- a/pipenv/patched/safety/__init__.py +++ b/pipenv/patched/safety/__init__.py @@ -2,4 +2,4 @@ __author__ = """pyup.io""" __email__ = 'support@pyup.io' -__version__ = '1.8.4' +__version__ = '1.8.5' diff --git a/pipenv/patched/safety/cli.py b/pipenv/patched/safety/cli.py index 9ebd9e8a..2e8f88df 100644 --- a/pipenv/patched/safety/cli.py +++ b/pipenv/patched/safety/cli.py @@ -6,17 +6,13 @@ from safety import __version__ from safety import safety from safety.formatter import report import itertools -from safety.util import read_requirements +from safety.util import read_requirements, read_vulnerabilities from safety.errors import DatabaseFetchError, DatabaseFileNotFoundError, InvalidKeyError - try: - # pip 9 - from pipenv.patched.notpip import get_installed_distributions + from json.decoder import JSONDecodeError except ImportError: - # pip 10 - from pipenv.patched.notpip._internal.utils.misc import get_installed_distributions - + JSONDecodeError = ValueError @click.group() @click.version_option(version=__version__) @@ -46,10 +42,17 @@ def cli(): help="Read input from one (or multiple) requirement files. Default: empty") @click.option("ignore", "--ignore", "-i", multiple=True, type=str, default=[], help="Ignore one (or multiple) vulnerabilities by ID. Default: empty") -def check(key, db, json, full_report, bare, stdin, files, cache, ignore): - +@click.option("--output", "-o", default="", + help="Path to where output file will be placed. Default: empty") +@click.option("proxyhost", "--proxy-host", "-ph", multiple=False, type=str, default=None, + help="Proxy host IP or DNS --proxy-host") +@click.option("proxyport", "--proxy-port", "-pp", multiple=False, type=int, default=80, + help="Proxy port number --proxy-port") +@click.option("proxyprotocol", "--proxy-protocol", "-pr", multiple=False, type=str, default='http', + help="Proxy protocol (https or http) --proxy-protocol") +def check(key, db, json, full_report, bare, stdin, files, cache, ignore, output, proxyprotocol, proxyhost, proxyport): if files and stdin: - click.secho("Can't read from --stdin and --file at the same time, exiting", fg="red") + click.secho("Can't read from --stdin and --file at the same time, exiting", fg="red", file=sys.stderr) sys.exit(-1) if files: @@ -57,33 +60,72 @@ def check(key, db, json, full_report, bare, stdin, files, cache, ignore): elif stdin: packages = list(read_requirements(sys.stdin)) else: - packages = get_installed_distributions() - + import pkg_resources + packages = [ + d for d in pkg_resources.working_set + if d.key not in {"python", "wsgiref", "argparse"} + ] + proxy_dictionary = {} + if proxyhost is not None: + if proxyprotocol in ["http", "https"]: + proxy_dictionary = {proxyprotocol: "{0}://{1}:{2}".format(proxyprotocol, proxyhost, str(proxyport))} + else: + click.secho("Proxy Protocol should be http or https only.", fg="red") + sys.exit(-1) try: - vulns = safety.check(packages=packages, key=key, db_mirror=db, cached=cache, ignore_ids=ignore) - click.secho(report( - vulns=vulns, - full=full_report, - json_report=json, - bare_report=bare, - checked_packages=len(packages), - db=db, - key=key - ) - ) + vulns = safety.check(packages=packages, key=key, db_mirror=db, cached=cache, ignore_ids=ignore, proxy=proxy_dictionary) + output_report = report(vulns=vulns, + full=full_report, + json_report=json, + bare_report=bare, + checked_packages=len(packages), + db=db, + key=key) + + if output: + with open(output, 'w+') as output_file: + output_file.write(output_report) + else: + click.secho(output_report, nl=False if bare and not vulns else True) sys.exit(-1 if vulns else 0) except InvalidKeyError: click.secho("Your API Key '{key}' is invalid. See {link}".format( key=key, link='https://goo.gl/O7Y1rS'), - fg="red") + fg="red", + file=sys.stderr) sys.exit(-1) except DatabaseFileNotFoundError: - click.secho("Unable to load vulnerability database from {db}".format(db=db), fg="red") + click.secho("Unable to load vulnerability database from {db}".format(db=db), fg="red", file=sys.stderr) sys.exit(-1) except DatabaseFetchError: - click.secho("Unable to load vulnerability database", fg="red") + click.secho("Unable to load vulnerability database", fg="red", file=sys.stderr) sys.exit(-1) +@cli.command() +@click.option("--full-report/--short-report", default=False, + help='Full reports include a security advisory (if available). Default: ' + '--short-report') +@click.option("--bare/--not-bare", default=False, + help='Output vulnerable packages only. Useful in combination with other tools.' + 'Default: --not-bare') +@click.option("file", "--file", "-f", type=click.File(), required=True, + help="Read input from an insecure report file. Default: empty") +def review(full_report, bare, file): + if full_report and bare: + click.secho("Can't choose both --bare and --full-report/--short-report", fg="red") + sys.exit(-1) + + try: + input_vulns = read_vulnerabilities(file) + except JSONDecodeError: + click.secho("Not a valid JSON file", fg="red") + sys.exit(-1) + + vulns = safety.review(input_vulns) + output_report = report(vulns=vulns, full=full_report, bare_report=bare) + click.secho(output_report, nl=False if bare and not vulns else True) + + if __name__ == "__main__": cli() diff --git a/pipenv/patched/safety/formatter.py b/pipenv/patched/safety/formatter.py index 8bc57ec1..c19bff1b 100644 --- a/pipenv/patched/safety/formatter.py +++ b/pipenv/patched/safety/formatter.py @@ -3,6 +3,7 @@ import platform import sys import json import os +import textwrap # python 2.7 compat try: @@ -110,9 +111,10 @@ class SheetReport(object): descr = get_advisory(vuln) - for chunk in [descr[i:i + 76] for i in range(0, len(descr), 76)]: - - for line in chunk.splitlines(): + for pn, paragraph in enumerate(descr.replace('\r', '').split('\n\n')): + if pn: + table.append("│ {:76} │".format('')) + for line in textwrap.wrap(paragraph, width=76): try: table.append("│ {:76} │".format(line.encode('utf-8'))) except TypeError: diff --git a/pipenv/patched/safety/safety.py b/pipenv/patched/safety/safety.py index 99b99125..871bd775 100644 --- a/pipenv/patched/safety/safety.py +++ b/pipenv/patched/safety/safety.py @@ -9,6 +9,7 @@ import json import time import errno + class Vulnerability(namedtuple("Vulnerability", ["name", "spec", "version", "advisory", "vuln_id"])): pass @@ -64,7 +65,7 @@ def write_to_cache(db_name, data): f.write(json.dumps(cache)) -def fetch_database_url(mirror, db_name, key, cached): +def fetch_database_url(mirror, db_name, key, cached, proxy): headers = {} if key: @@ -74,9 +75,8 @@ def fetch_database_url(mirror, db_name, key, cached): cached_data = get_from_cache(db_name=db_name) if cached_data: return cached_data - url = mirror + db_name - r = requests.get(url=url, timeout=REQUEST_TIMEOUT, headers=headers) + r = requests.get(url=url, timeout=REQUEST_TIMEOUT, headers=headers, proxies=proxy) if r.status_code == 200: data = r.json() if cached: @@ -94,7 +94,7 @@ def fetch_database_file(path, db_name): return json.loads(f.read()) -def fetch_database(full=False, key=False, db=False, cached=False): +def fetch_database(full=False, key=False, db=False, cached=False, proxy={}): if db: mirrors = [db] @@ -105,7 +105,7 @@ def fetch_database(full=False, key=False, db=False, cached=False): for mirror in mirrors: # mirror can either be a local path or a URL if mirror.startswith("http://") or mirror.startswith("https://"): - data = fetch_database_url(mirror, db_name=db_name, key=key, cached=cached) + data = fetch_database_url(mirror, db_name=db_name, key=key, cached=cached, proxy=proxy) else: data = fetch_database_file(mirror, db_name=db_name) if data: @@ -120,10 +120,9 @@ def get_vulnerabilities(pkg, spec, db): yield entry -def check(packages, key, db_mirror, cached, ignore_ids): - +def check(packages, key, db_mirror, cached, ignore_ids, proxy): key = key if key else os.environ.get("SAFETY_API_KEY", False) - db = fetch_database(key=key, db=db_mirror, cached=cached) + db = fetch_database(key=key, db=db_mirror, cached=cached, proxy=proxy) db_full = None vulnerable_packages = frozenset(db.keys()) vulnerable = [] @@ -152,3 +151,19 @@ def check(packages, key, db_mirror, cached, ignore_ids): ) ) return vulnerable + + +def review(vulnerabilities): + vulnerable = [] + for vuln in vulnerabilities: + current_vuln = { + "name": vuln[0], + "spec": vuln[1], + "version": vuln[2], + "advisory": vuln[3], + "vuln_id": vuln[4], + } + vulnerable.append( + Vulnerability(**current_vuln) + ) + return vulnerable diff --git a/pipenv/patched/safety/util.py b/pipenv/patched/safety/util.py index 2e6efaec..16062f41 100644 --- a/pipenv/patched/safety/util.py +++ b/pipenv/patched/safety/util.py @@ -1,11 +1,17 @@ from dparse.parser import setuptools_parse_requirements_backport as _parse_requirements from collections import namedtuple import click +import sys +import json import os Package = namedtuple("Package", ["key", "version"]) RequirementFile = namedtuple("RequirementFile", ["path"]) +def read_vulnerabilities(fh): + return json.load(fh) + + def iter_lines(fh, lineno=0): for line in fh.readlines()[lineno:]: yield line @@ -85,7 +91,8 @@ def read_requirements(fh, resolve=False): "Warning: unpinned requirement '{req}' found in {fname}, " "unable to check.".format(req=req.name, fname=fname), - fg="yellow" + fg="yellow", + file=sys.stderr ) except ValueError: continue diff --git a/pipenv/pep508checker.py b/pipenv/pep508checker.py index 724be7cc..e875a6d1 100644 --- a/pipenv/pep508checker.py +++ b/pipenv/pep508checker.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- -import sys +import json import os import platform -import json +import sys def format_full_version(info): diff --git a/pipenv/pipenv.1 b/pipenv/pipenv.1 index 0876d34f..6cd64a4c 100644 --- a/pipenv/pipenv.1 +++ b/pipenv/pipenv.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "PIPENV" "1" "Nov 18, 2018" "2018.11.15.dev0" "pipenv" +.TH "PIPENV" "1" "Jul 14, 2019" "2018.11.27.dev0" "pipenv" .SH NAME pipenv \- pipenv Documentation . @@ -43,6 +43,8 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] It automatically creates and manages a virtualenv for your projects, as well as adds/removes packages from your \fBPipfile\fP as you install/uninstall packages. It also generates the ever\-important \fBPipfile.lock\fP, which is used to produce deterministic builds. .sp Pipenv is primarily meant to provide users and developers of applications with an easy method to setup a working environment. For the distinction between libraries and applications and the usage of \fBsetup.py\fP vs \fBPipfile\fP to define dependencies, see pipfile\-vs\-setuppy\&. +[image: a short animation of pipenv at work] +[image] .sp The problems that Pipenv seeks to solve are multi\-faceted: .INDENT 0.0 @@ -63,7 +65,7 @@ Streamline development workflow by loading \fB\&.env\fP files. You can quickly play with Pipenv right in your browser: \fI\%Try in browser\fP.SH INSTALL PIPENV TODAY! .sp -If you\(aqre on MacOS, you can install Pipenv easily with Homebrew: +If you\(aqre on MacOS, you can install Pipenv easily with Homebrew. You can also use Linuxbrew on Linux using the same command: .INDENT 0.0 .INDENT 3.5 .sp @@ -168,12 +170,11 @@ pip 9.0.1 .UNINDENT .UNINDENT .sp -If you installed Python from source, with an installer from \fI\%python.org\fP, or -via \fI\%Homebrew\fP you should already have pip. If you\(aqre on Linux and installed +If you installed Python from source, with an installer from \fI\%python.org\fP, via \fI\%Homebrew\fP or via \fI\%Linuxbrew\fP you should already have pip. If you\(aqre on Linux and installed using your OS package manager, you may have to \fI\%install pip\fP separately. .sp -If you plan to install Pipenv using Homebrew you can skip this step. The -Homebrew installer takes care of pip for you. +If you plan to install Pipenv using Homebrew or Linuxbrew you can skip this step. The +Homebrew/Linuxbrew installer takes care of pip for you. .SS ☤ Installing Pipenv .sp Pipenv is a dependency manager for Python projects. If you\(aqre familiar @@ -183,13 +184,13 @@ it\(aqs a higher\-level tool that simplifies dependency management for common us cases. .SS ☤ Homebrew Installation of Pipenv .sp -Homebrew is a popular open\-source package management system for macOS. +\fI\%Homebrew\fP is a popular open\-source package management system for macOS. For Linux users, \fI\%Linuxbrew\fP is a Linux port of that. .sp -Installing pipenv via Homebrew will keep pipenv and all of its dependencies in +Installing pipenv via Homebrew or Linuxbrew will keep pipenv and all of its dependencies in an isolated virtual environment so it doesn\(aqt interfere with the rest of your Python installation. .sp -Once you have installed \fI\%Homebrew\fP simply run: +Once you have installed Homebrew or Linuxbrew simply run: .INDENT 0.0 .INDENT 3.5 .sp @@ -395,6 +396,66 @@ You might want to set \fBexport PIPENV_VENV_IN_PROJECT=1\fP in your .bashrc/.zsh .sp Congratulations, you now know how to install and use Python packages! ✨ 🍰 ✨ .SS Release and Version History +.SS 2018.11.26 (2018\-11\-26) +.SS Bug Fixes +.INDENT 0.0 +.IP \(bu 2 +Environment variables are expanded correctly before running scripts on POSIX. \fI\%#3178\fP +.IP \(bu 2 +Pipenv will no longer disable user\-mode installation when the \fB\-\-system\fP flag is passed in. \fI\%#3222\fP +.IP \(bu 2 +Fixed an issue with attempting to render unicode output in non\-unicode locales. \fI\%#3223\fP +.IP \(bu 2 +Fixed a bug which could cause failures to occur when parsing python entries from global pyenv version files. \fI\%#3224\fP +.IP \(bu 2 +Fixed an issue which prevented the parsing of named extras sections from certain \fBsetup.py\fP files. \fI\%#3230\fP +.IP \(bu 2 +Correctly detect the virtualenv location inside an activated virtualenv. \fI\%#3231\fP +.IP \(bu 2 +Fixed a bug which caused spinner frames to be written to stdout during locking operations which could cause redirection pipes to fail. \fI\%#3239\fP +.IP \(bu 2 +Fixed a bug that editable pacakges can\(aqt be uninstalled correctly. \fI\%#3240\fP +.IP \(bu 2 +Corrected an issue with installation timeouts which caused dependency resolution to fail for longer duration resolution steps. \fI\%#3244\fP +.IP \(bu 2 +Adding normal pep 508 compatible markers is now fully functional when using VCS dependencies. \fI\%#3249\fP +.IP \(bu 2 +Updated \fBrequirementslib\fP and \fBpythonfinder\fP for multiple bugfixes. \fI\%#3254\fP +.IP \(bu 2 +Pipenv will now ignore hashes when installing with \fB\-\-skip\-lock\fP\&. \fI\%#3255\fP +.IP \(bu 2 +Fixed an issue where pipenv could crash when multiple pipenv processes attempted to create the same directory. \fI\%#3257\fP +.IP \(bu 2 +Fixed an issue which sometimes prevented successful creation of project pipfiles. \fI\%#3260\fP +.IP \(bu 2 +\fBpipenv install\fP will now unset the \fBPYTHONHOME\fP environment variable when not combined with \fB\-\-system\fP\&. \fI\%#3261\fP +.IP \(bu 2 +Pipenv will ensure that warnings do not interfere with the resolution process by suppressing warnings\(aq usage of standard output and writing to standard error instead. \fI\%#3273\fP +.IP \(bu 2 +Fixed an issue which prevented variables from the environment, such as \fBPIPENV_DEV\fP or \fBPIPENV_SYSTEM\fP, from being parsed and implemented correctly. \fI\%#3278\fP +.IP \(bu 2 +Clear pythonfinder cache after Python install. \fI\%#3287\fP +.IP \(bu 2 +Fixed a race condition in hash resolution for dependencies for certain dependencies with missing cache entries or fresh Pipenv installs. \fI\%#3289\fP +.IP \(bu 2 +Pipenv will now respect top\-level pins over VCS dependency locks. \fI\%#3296\fP +.UNINDENT +.SS Vendored Libraries +.INDENT 0.0 +.IP \(bu 2 +.INDENT 2.0 +.TP +.B Update vendored dependencies to resolve resolution output parsing and python finding: +.INDENT 7.0 +.IP \(bu 2 +\fBpythonfinder 1.1.9 \-> 1.1.10\fP +.IP \(bu 2 +\fBrequirementslib 1.3.1 \-> 1.3.3\fP +.IP \(bu 2 +\fBvistir 0.2.3 \-> 0.2.5\fP \fI\%#3280\fP +.UNINDENT +.UNINDENT +.UNINDENT .SS 2018.11.14 (2018\-11\-14) .SS Features & Improvements .INDENT 0.0 @@ -1108,9 +1169,6 @@ In addition, scandir was vendored and patched to avoid importing host system bin .SH USER TESTIMONIALS .INDENT 0.0 .TP -\fBJannis Leidel\fP, former pip maintainer— -\fIPipenv is the porcelain I always wanted to build for pip. It fits my brain and mostly replaces virtualenvwrapper and manual pip calls for me. Use it.\fP -.TP \fBDavid Gang\fP— \fIThis package manager is really awesome. For the first time I know exactly what my dependencies are which I installed and what the transitive dependencies are. Combined with the fact that installs are deterministic, makes this package manager first class, like cargo\fP\&. .TP @@ -1293,6 +1351,8 @@ Do not keep \fBPipfile.lock\fP in version control if multiple versions of Python Specify your target Python version in your \fIPipfile\fP\(aqs \fB[requires]\fP section. Ideally, you should only have one target Python version, as this is a deployment tool. .IP \(bu 2 \fBpipenv install\fP is fully compatible with \fBpip install\fP syntax, for which the full documentation can be found \fI\%here\fP\&. +.IP \(bu 2 +Note that the \fBPipfile\fP uses the \fI\%TOML Spec\fP\&. .UNINDENT .SS ☤ Example Pipenv Workflow .sp @@ -1415,13 +1475,13 @@ $ pipenv install "requests>2.19" # will install 2.19.1 but not 2.19.0 \fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -The use of \fB" "\fP around the package and version specification is highly recommended +The use of double quotes around the package and version specification (i.e. \fB"requests>2.19"\fP) is highly recommended to avoid issues with \fI\%Input and output redirection\fP in Unix\-based operating systems. .UNINDENT .UNINDENT .sp -The use of \fB~=\fP is preferred over the \fB==\fP identifier as the former prevents pipenv from updating the packages: +The use of \fB~=\fP is preferred over the \fB==\fP identifier as the latter prevents pipenv from updating the packages: .INDENT 0.0 .INDENT 3.5 .sp @@ -1799,6 +1859,19 @@ name = "pypi" Luckily \- pipenv will hash your Pipfile \fIbefore\fP expanding environment variables (and, helpfully, will substitute the environment variables again when you install from the lock file \- so no need to commit any secrets! Woo!) +.sp +If your credentials contain a special character, surround the references to the environment variables with quotation marks. For example, if your password contain a double quotation mark, surround the password variable with single quotation marks. Otherwise, you may get a \fBValueError, "No closing quotation"\fP error while installing dependencies. +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +[[source]] +url = "https://$USERNAME:\(aq${PASSWORD}\(aq@mypypi.example.com/simple" +.ft P +.fi +.UNINDENT +.UNINDENT .SS ☤ Specifying Basically Anything .sp If you\(aqd like to specify that a specific package only be installed on certain systems, @@ -2252,9 +2325,6 @@ For example: .sp .nf .ft C -[scripts] -echospam = "echo I am really a very silly example" - $ pipenv run echospam "indeed" I am really a very silly example indeed .ft P @@ -2263,7 +2333,9 @@ I am really a very silly example indeed .UNINDENT .SS ☤ Support for Environment Variables .sp -Pipenv supports the usage of environment variables in values. For example: +Pipenv supports the usage of environment variables in place of authentication fragments +in your Pipfile. These will only be parsed if they are present in the \fB[[source]]\fP +section. For example: .INDENT 0.0 .INDENT 3.5 .sp @@ -2286,6 +2358,7 @@ records = "*" .UNINDENT .sp Environment variables may be specified as \fB${MY_ENVAR}\fP or \fB$MY_ENVAR\fP\&. +.sp On Windows, \fB%MY_ENVAR%\fP is supported in addition to \fB${MY_ENVAR}\fP or \fB$MY_ENVAR\fP\&. .SS ☤ Configuration With Environment Variables .sp @@ -2392,7 +2465,7 @@ Default is 16, an arbitrary number that works most of the time. .UNINDENT .INDENT 0.0 .TP -.B pipenv.environments.PIPENV_MAX_SUBPROCESS = 16 +.B pipenv.environments.PIPENV_MAX_SUBPROCESS = 8 How many subprocesses should Pipenv use when installing. .sp Default is 16, an arbitrary number that seems to work. @@ -2427,7 +2500,7 @@ See also \fBPIPENV_MAX_DEPTH\fP\&. .UNINDENT .INDENT 0.0 .TP -.B pipenv.environments.PIPENV_PYPI_MIRROR = None +.B pipenv.environments.PIPENV_PYPI_MIRROR = \(aqhttps://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple\(aq If set, tells pipenv to override PyPI index urls with a mirror. .sp Default is to not mirror PyPI, i.e. use the real one, pypi.org. The @@ -2435,6 +2508,15 @@ Default is to not mirror PyPI, i.e. use the real one, pypi.org. The .UNINDENT .INDENT 0.0 .TP +.B pipenv.environments.PIPENV_RESOLVE_VCS = False +Tells Pipenv whether to resolve all VCS dependencies in full. +.sp +As of Pipenv 2018.11.26, only editable VCS dependencies were resolved in full. +To retain this behavior and avoid handling any conflicts that arise from the new +approach, you may set this to \(aq0\(aq, \(aqoff\(aq, or \(aqfalse\(aq. +.UNINDENT +.INDENT 0.0 +.TP .B pipenv.environments.PIPENV_SHELL = \(aq/bin/zsh\(aq An absolute path to the preferred shell for \fBpipenv shell\fP\&. .sp @@ -2489,6 +2571,26 @@ If set, Pipenv automatically assumes "yes" at all prompts. Default is to prompt the user for an answer if the current command line session if interactive. .UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIP_EXISTS_ACTION = \(aqw\(aq +Specifies the value for pip\(aqs \-\-exists\-action option +.sp +Defaullts to (w)ipe +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.is_in_virtualenv() +Check virtualenv membership dynamically +.INDENT 7.0 +.TP +.B Returns +True or false depending on whether we are in a regular virtualenv or not +.TP +.B Return type +bool +.UNINDENT +.UNINDENT .sp If you\(aqd like to set these environment variables on a per\-project basis, I recommend utilizing the fantastic \fI\%direnv\fP project, in order to do so. .sp @@ -2569,7 +2671,7 @@ init: pipenv install \-\-dev test: - pipenv run py.test tests + pipenv run pytest tests .ft P .fi .UNINDENT @@ -2590,7 +2692,7 @@ envlist = flake8\-py3, py26, py27, py33, py34, py35, py36, pypy deps = pipenv commands= pipenv install \-\-dev - pipenv run py.test tests + pipenv run pytest tests [testenv:flake8\-py3] basepython = python3.4 @@ -2603,7 +2705,7 @@ commands= .UNINDENT .UNINDENT .sp -Pipenv will automatically use the virtualenv provided by \fBtox\fP\&. If \fBpipenv install \-\-dev\fP installs e.g. \fBpytest\fP, then installed command \fBpy.test\fP will be present in given virtualenv and can be called directly by \fBpy.test tests\fP instead of \fBpipenv run py.test tests\fP\&. +Pipenv will automatically use the virtualenv provided by \fBtox\fP\&. If \fBpipenv install \-\-dev\fP installs e.g. \fBpytest\fP, then installed command \fBpytest\fP will be present in given virtualenv and can be called directly by \fBpytest tests\fP instead of \fBpipenv run pytest tests\fP\&. .sp You might also want to add \fB\-\-ignore\-pipfile\fP to \fBpipenv install\fP, as to not accidentally modify the lock\-file on each test run. This causes Pipenv @@ -2922,6 +3024,7 @@ or \fI\%Nate Prewitt\fP, the primary maintainers. The guide is split into sections based on the type of contribution you\(aqre thinking of making, with a section that covers general guidelines for all contributors. +.SS General Guidelines .SS Be Cordial .INDENT 0.0 .INDENT 3.5 @@ -2953,32 +3056,90 @@ current goals or needs of the project. If your contribution is rejected, don\(aqt despair! As long as you followed these guidelines, you will have a much better chance of getting your next contribution accepted. +.SS Questions +.sp +The GitHub issue tracker is for \fIbug reports\fP and \fIfeature requests\fP\&. Please do +not use it to ask questions about how to use Pipenv. These questions should +instead be directed to \fI\%Stack Overflow\fP\&. Make sure that your question is tagged +with the \fBpipenv\fP tag when asking it on Stack Overflow, to ensure that it is +answered promptly and accurately. .SS Code Contributions .SS Steps for Submitting Code .sp When contributing code, you\(aqll want to follow this checklist: .INDENT 0.0 .IP 1. 3 -Fork the repository on GitHub. +Understand our \fI\%development philosophy\fP\&. .IP 2. 3 -\fI\%Run the tests\fP to confirm they all pass on your system. If they don\(aqt, you\(aqll -need to investigate why they fail. If you\(aqre unable to diagnose this -yourself, raise it as a bug report by following the guidelines in this -document: \fI\%Bug Reports\fP\&. +Fork the repository on GitHub. .IP 3. 3 -Write tests that demonstrate your bug or feature. Ensure that they fail. +Set up your \fI\%Development Setup\fP .IP 4. 3 -Make your change. +Run the tests (\fI\%Testing\fP) to confirm they all pass on your system. +If they don\(aqt, you\(aqll need to investigate why they fail. If you\(aqre unable +to diagnose this yourself, raise it as a bug report by following the guidelines +in this document: \fI\%Bug Reports\fP\&. .IP 5. 3 +Write tests that demonstrate your bug or feature. Ensure that they fail. +.IP 6. 3 +Make your change. +.IP 7. 3 Run the entire test suite again, confirming that all tests pass \fIincluding the ones you just added\fP\&. -.IP 6. 3 +.IP 8. 3 Send a GitHub Pull Request to the main repository\(aqs \fBmaster\fP branch. GitHub Pull Requests are the expected method of code collaboration on this project. .UNINDENT .sp The following sub\-sections go into more detail on some of the points above. +.SS Development Setup +.sp +To get your development environment setup, run: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +pip install \-e . +pipenv install \-\-dev +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +This will install the repo version of Pipenv and then install the development +dependencies. Once that has completed, you can start developing. +.sp +The repo version of Pipenv must be installed over other global versions to +resolve conflicts with the \fBpipenv\fP folder being implicitly added to \fBsys.path\fP\&. +See \fI\%pypa/pipenv#2557\fP for more details. +.SS Testing +.sp +Tests are written in \fBpytest\fP style and can be run very simply: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +pytest +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +This will run all Pipenv tests, which can take awhile. To run a subset of the +tests, the standard pytest filters are available, such as: +.INDENT 0.0 +.IP \(bu 2 +provide a directory or file: \fBpytest tests/unit\fP or \fBpytest tests/unit/test_cmdparse.py\fP +.IP \(bu 2 +provide a keyword expression: \fBpytest \-k test_lock_editable_vcs_without_install\fP +.IP \(bu 2 +provide a nodeid: \fBpytest tests/unit/test_cmdparse.py::test_parse\fP +.IP \(bu 2 +provide a test marker: \fBpytest \-m lock\fP +.UNINDENT .SS Code Review .sp Contributions will not be merged until they\(aqve been code reviewed. You should @@ -2986,6 +3147,15 @@ implement any code review feedback unless you strongly object to it. In the event that you object to the code review feedback, you should make your case clearly and calmly. If, after doing so, the feedback is judged to still apply, you must either apply the feedback or withdraw your contribution. +.SS Package Index +.sp +To speed up testing, tests that rely on a package index for locking and +installing use a local server that contains vendored packages in the +\fBtests/pypi\fP directory. Each vendored package should have it\(aqs own folder +containing the necessary releases. When adding a release for a package, it is +easiest to use either the \fB\&.tar.gz\fP or universal wheels (ex: \fBpy2.py3\-none\fP). If +a \fB\&.tar.gz\fP or universal wheel is not available, add wheels for all available +architectures and platforms. .SS Documentation Contributions .sp Documentation improvements are always welcome! The documentation files live in @@ -3001,10 +3171,52 @@ When presenting Python code, use single\-quoted strings (\fB\(aqhello\(aq\fP ins \fB"hello"\fP). .SS Bug Reports .sp -Bug reports are hugely important! Before you raise one, though, please check -through the \fI\%GitHub issues\fP, \fBboth open and closed\fP, to confirm that the bug -hasn\(aqt been reported before. Duplicate bug reports are a huge drain on the time -of other contributors, and should be avoided as much as possible. +Bug reports are hugely important! They are recorded as \fI\%GitHub issues\fP\&. Please +be aware of the following things when filing bug reports: +.INDENT 0.0 +.IP 1. 3 +Avoid raising duplicate issues. \fIPlease\fP use the GitHub issue search feature +to check whether your bug report or feature request has been mentioned in +the past. Duplicate bug reports and feature requests are a huge maintenance +burden on the limited resources of the project. If it is clear from your +report that you would have struggled to find the original, that\(aqs ok, but +if searching for a selection of words in your issue title would have found +the duplicate then the issue will likely be closed extremely abruptly. +.IP 2. 3 +When filing bug reports about exceptions or tracebacks, please include the +\fIcomplete\fP traceback. Partial tracebacks, or just the exception text, are +not helpful. Issues that do not contain complete tracebacks may be closed +without warning. +.IP 3. 3 +Make sure you provide a suitable amount of information to work with. This +means you should provide: +.INDENT 3.0 +.IP \(bu 2 +Guidance on \fBhow to reproduce the issue\fP\&. Ideally, this should be a +\fIsmall\fP code sample that can be run immediately by the maintainers. +Failing that, let us know what you\(aqre doing, how often it happens, what +environment you\(aqre using, etc. Be thorough: it prevents us needing to ask +further questions. +.IP \(bu 2 +Tell us \fBwhat you expected to happen\fP\&. When we run your example code, +what are we expecting to happen? What does "success" look like for your +code? +.IP \(bu 2 +Tell us \fBwhat actually happens\fP\&. It\(aqs not helpful for you to say "it +doesn\(aqt work" or "it fails". Tell us \fIhow\fP it fails: do you get an +exception? A hang? The packages installed seem incorrect? +How was the actual result different from your expected result? +.IP \(bu 2 +Tell us \fBwhat version of Pipenv you\(aqre using\fP, and +\fBhow you installed it\fP\&. Different versions of Pipenv behave +differently and have different bugs, and some distributors of Pipenv +ship patches on top of the code we supply. +.UNINDENT +.sp +If you do not provide all of these things, it will take us much longer to +fix your problem. If we ask you to clarify these and you never respond, we +will close your issue without fixing it. +.UNINDENT .SS Run the tests .sp Three ways of running the tests are as follows: @@ -3015,18 +3227,21 @@ Three ways of running the tests are as follows: \fB\&./run\-tests.sh\fP or \fBrun\-tests.bat\fP .IP 3. 3 Using pipenv: -.INDENT 3.0 +.UNINDENT +.INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -pipenv install \-\-dev -pipenv run pytest +$ git clone https://github.com/pypa/pipenv.git +$ cd pipenv +$ git submodule sync && git submodule update \-\-init \-\-recursive +$ pipenv install \-\-dev +$ pipenv run pytest .ft P .fi .UNINDENT .UNINDENT -.UNINDENT .sp For the last two, it is important that your environment is setup correctly, and this may take some work, for example, on a specific Mac installation, the following @@ -3300,7 +3515,7 @@ System pip management. .INDENT 0.0 .TP .B \-c, \-\-code -Import from codebase. +Install packages automatically discovered from import statements. .UNINDENT .INDENT 0.0 .TP @@ -3329,11 +3544,6 @@ Update specified packages. .UNINDENT .INDENT 0.0 .TP -.B \-\-pre -Allow pre\-releases. -.UNINDENT -.INDENT 0.0 -.TP .B \-r, \-\-requirements Import a requirements.txt file. .UNINDENT @@ -3400,18 +3610,30 @@ Optional argument(s) Environment variables.INDENT 0.0 .TP .B PIPENV_SKIP_LOCK +.INDENT 7.0 +.INDENT 3.5 Provide a default for \fI\%\-\-skip\-lock\fP .UNINDENT +.UNINDENT +.UNINDENT .INDENT 0.0 .TP .B PIP_EXTRA_INDEX_URL +.INDENT 7.0 +.INDENT 3.5 Provide a default for \fI\%\-\-extra\-index\-url\fP .UNINDENT +.UNINDENT +.UNINDENT .INDENT 0.0 .TP .B PIP_INDEX_URL +.INDENT 7.0 +.INDENT 3.5 Provide a default for \fI\%\-i\fP .UNINDENT +.UNINDENT +.UNINDENT .SS lock .sp Generates Pipfile.lock. @@ -3690,11 +3912,6 @@ pipenv uninstall [OPTIONS] [PACKAGES]... .UNINDENT Options.INDENT 0.0 .TP -.B \-\-skip\-lock, \-\-lock -Lock afterwards. -.UNINDENT -.INDENT 0.0 -.TP .B \-\-all\-dev Un\-install all package from [dev\-packages]. .UNINDENT @@ -3761,8 +3978,12 @@ Optional argument(s) Environment variables.INDENT 0.0 .TP .B PIPENV_SKIP_LOCK +.INDENT 7.0 +.INDENT 3.5 Provide a default for \fI\%\-\-skip\-lock\fP .UNINDENT +.UNINDENT +.UNINDENT .SS update .sp Runs lock, then sync. @@ -3808,11 +4029,6 @@ Update specified packages. .UNINDENT .INDENT 0.0 .TP -.B \-\-pre -Allow pre\-releases. -.UNINDENT -.INDENT 0.0 -.TP .B \-r, \-\-requirements Import a requirements.txt file. .UNINDENT @@ -3879,13 +4095,21 @@ Optional argument(s) Environment variables.INDENT 0.0 .TP .B PIP_EXTRA_INDEX_URL +.INDENT 7.0 +.INDENT 3.5 Provide a default for \fI\%\-\-extra\-index\-url\fP .UNINDENT +.UNINDENT +.UNINDENT .INDENT 0.0 .TP .B PIP_INDEX_URL +.INDENT 7.0 +.INDENT 3.5 Provide a default for \fI\%\-i\fP .UNINDENT +.UNINDENT +.UNINDENT .INDENT 0.0 .IP \(bu 2 genindex diff --git a/pipenv/progress.py b/pipenv/progress.py index c126fe73..8968150c 100644 --- a/pipenv/progress.py +++ b/pipenv/progress.py @@ -12,31 +12,29 @@ from __future__ import absolute_import import os import sys import time + import crayons + from .environments import PIPENV_COLORBLIND, PIPENV_HIDE_EMOJIS + STREAM = sys.stderr MILL_TEMPLATE = "%s %s %i/%i\r" DOTS_CHAR = "." -if os.name != "nt": - if PIPENV_HIDE_EMOJIS: - if PIPENV_COLORBLIND: - BAR_FILLED_CHAR = "=" - BAR_EMPTY_CHAR = "-" - else: - BAR_FILLED_CHAR = str(crayons.green("=", bold=True)) - BAR_EMPTY_CHAR = str(crayons.black("-")) +if PIPENV_HIDE_EMOJIS: + if PIPENV_COLORBLIND: + BAR_FILLED_CHAR = "=" + BAR_EMPTY_CHAR = "-" else: - if PIPENV_COLORBLIND: - BAR_FILLED_CHAR = "▉" - BAR_EMPTY_CHAR = " " - else: - BAR_FILLED_CHAR = str(crayons.green("▉", bold=True)) - BAR_EMPTY_CHAR = str(crayons.black("▉")) - + BAR_FILLED_CHAR = str(crayons.green("=", bold=True)) + BAR_EMPTY_CHAR = str(crayons.black("-")) else: - BAR_FILLED_CHAR = "=" - BAR_EMPTY_CHAR = "-" + if PIPENV_COLORBLIND: + BAR_FILLED_CHAR = "▉" + BAR_EMPTY_CHAR = " " + else: + BAR_FILLED_CHAR = str(crayons.green("▉", bold=True)) + BAR_EMPTY_CHAR = str(crayons.black("▉")) if (sys.version_info[0] >= 3) and (os.name != "nt"): BAR_TEMPLATE = u" %s%s%s %i/%i — {0}\r".format(crayons.black("%s")) diff --git a/pipenv/project.py b/pipenv/project.py index 69f3ad5d..c4b0c941 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -1,72 +1,63 @@ # -*- coding: utf-8 -*- +import base64 +import fnmatch +import glob +import hashlib import io import json +import operator import os import re import sys -import glob -import base64 -import fnmatch -import hashlib -from first import first -from cached_property import cached_property -import operator -import pipfile -import pipfile.api + import six -import vistir import toml import tomlkit +import vistir + +from first import first + +import pipfile +import pipfile.api + +from .vendor.cached_property import cached_property -from .environment import Environment from .cmdparse import Script -from .utils import ( - pep423_name, - proper_case, - find_requirements, - is_editable, - cleanup_toml, - convert_toml_outline_tables, - is_installable_file, - is_valid_url, - get_url_name, - normalize_drive, - python_version, - safe_expandvars, - is_star, - get_workon_home, - is_virtual_environment, - looks_like_dir, - get_canonical_names -) +from .environment import Environment from .environments import ( - PIPENV_MAX_DEPTH, - PIPENV_PIPFILE, - PIPENV_VENV_IN_PROJECT, - PIPENV_TEST_INDEX, - PIPENV_PYTHON, - PIPENV_DEFAULT_PYTHON_VERSION, - PIPENV_IGNORE_VIRTUALENVS, - is_in_virtualenv + PIPENV_DEFAULT_PYTHON_VERSION, PIPENV_IGNORE_VIRTUALENVS, PIPENV_MAX_DEPTH, + PIPENV_PIPFILE, PIPENV_PYTHON, PIPENV_TEST_INDEX, PIPENV_VENV_IN_PROJECT, + is_in_virtualenv, is_type_checking ) +from .vendor.requirementslib.models.utils import get_default_pyproject_backend +from .utils import ( + cleanup_toml, convert_toml_outline_tables, find_requirements, + get_canonical_names, get_url_name, get_workon_home, is_editable, + is_installable_file, is_star, is_valid_url, is_virtual_environment, + looks_like_dir, normalize_drive, pep423_name, proper_case, python_version, + safe_expandvars, get_pipenv_dist +) + +if is_type_checking(): + from typing import Dict, Text, Union + TSource = Dict[Text, Union[Text, bool]] def _normalized(p): if p is None: return None loc = vistir.compat.Path(p) - if not loc.is_absolute(): - try: - loc = loc.resolve() - except OSError: - loc = loc.absolute() + try: + loc = loc.resolve() + except OSError: + loc = loc.absolute() # Recase the path properly on Windows. From https://stackoverflow.com/a/35229734/5043728 if os.name == 'nt': matches = glob.glob(re.sub(r'([^:/\\])(?=[/\\]|$)', r'[\1]', str(loc))) path_str = matches and matches[0] or str(loc) else: path_str = str(loc) - return normalize_drive(path_str) + return normalize_drive(os.path.abspath(path_str)) DEFAULT_NEWLINES = u"\n" @@ -111,6 +102,9 @@ if PIPENV_PIPFILE: else: PIPENV_PIPFILE = _normalized(PIPENV_PIPFILE) + # Overwrite environment variable so that subprocesses can get the correct path. + # See https://github.com/pypa/pipenv/issues/3584 + os.environ['PIPENV_PIPFILE'] = PIPENV_PIPFILE # (path, file contents) => TOMLFile # keeps track of pipfiles that we've seen so we do not need to re-parse 'em _pipfile_cache = {} @@ -235,7 +229,7 @@ class Project(object): @property def pipfile_exists(self): - return bool(self.pipfile_location) + return os.path.isfile(self.pipfile_location) @property def required_python_version(self): @@ -250,11 +244,7 @@ class Project(object): @property def project_directory(self): - if self.pipfile_location is not None: - return os.path.abspath(os.path.join(self.pipfile_location, os.pardir)) - - else: - return None + return os.path.abspath(os.path.join(self.pipfile_location, os.pardir)) @property def requirements_exists(self): @@ -268,8 +258,7 @@ class Project(object): @property def virtualenv_exists(self): - # TODO: Decouple project from existence of Pipfile. - if self.pipfile_exists and os.path.exists(self.virtualenv_location): + if os.path.exists(self.virtualenv_location): if os.name == "nt": extra = ["Scripts", "activate.bat"] else: @@ -353,7 +342,11 @@ class Project(object): prefix=prefix, is_venv=is_venv, sources=sources, pipfile=self.parsed_pipfile, project=self ) - self._environment.add_dist("pipenv") + pipenv_dist = get_pipenv_dist(pkg="pipenv") + if pipenv_dist: + self._environment.extend_dists(pipenv_dist) + else: + self._environment.add_dist("pipenv") return self._environment def get_outdated_packages(self): @@ -483,7 +476,7 @@ class Project(object): try: loc = pipfile.Pipfile.find(max_depth=PIPENV_MAX_DEPTH) except RuntimeError: - loc = None + loc = "Pipfile" self._pipfile_location = _normalized(loc) return self._pipfile_location @@ -512,6 +505,8 @@ class Project(object): def read_pipfile(self): # Open the pipfile, read it into memory. + if not self.pipfile_exists: + return "" with io.open(self.pipfile_location) as f: contents = f.read() self._pipfile_newlines = preferred_newlines(f) @@ -538,18 +533,19 @@ class Project(object): if not os.path.exists(self.path_to("setup.py")): if not build_system or not build_system.get("requires"): build_system = { - "requires": ["setuptools>=38.2.5", "wheel"], - "build-backend": "setuptools.build_meta", + "requires": ["setuptools>=40.8.0", "wheel"], + "build-backend": get_default_pyproject_backend(), } self._build_system = build_system @property def build_requires(self): - return self._build_system.get("requires", []) + return self._build_system.get("requires", ["setuptools>=40.8.0", "wheel"]) + @property def build_backend(self): - return self._build_system.get("build-backend", None) + return self._build_system.get("build-backend", get_default_pyproject_backend()) @property def settings(self): @@ -616,10 +612,8 @@ class Project(object): def _get_editable_packages(self, dev=False): section = "dev-packages" if dev else "packages" - # section = "{0}-editable".format(section) packages = { k: v - # for k, v in self._pipfile[section].items() for k, v in self.parsed_pipfile.get(section, {}).items() if is_editable(k) or is_editable(v) } @@ -628,10 +622,8 @@ class Project(object): def _get_vcs_packages(self, dev=False): from pipenv.vendor.requirementslib.utils import is_vcs section = "dev-packages" if dev else "packages" - # section = "{0}-vcs".format(section) packages = { k: v - # for k, v in self._pipfile[section].items() for k, v in self.parsed_pipfile.get(section, {}).items() if is_vcs(v) or is_vcs(k) } @@ -672,11 +664,6 @@ class Project(object): """Returns a list of dev-packages, for pip-tools to consume.""" return self._build_package_list("dev-packages") - def touch_pipfile(self): - """Simply touches the Pipfile, for later use.""" - with open("Pipfile", "a"): - os.utime("Pipfile", None) - @property def pipfile_is_empty(self): if not self.pipfile_exists: @@ -693,7 +680,6 @@ class Project(object): ConfigOptionParser, make_option_group, index_group ) - name = self.name if self.name is not None else "Pipfile" config_parser = ConfigOptionParser(name=self.name) config_parser.add_option_group(make_option_group(index_group, config_parser)) install = config_parser.option_groups[0] @@ -743,10 +729,19 @@ class Project(object): source["verify_ssl"] = source["verify_ssl"].lower() == "true" return source - def get_or_create_lockfile(self): + def get_or_create_lockfile(self, from_pipfile=False): from pipenv.vendor.requirementslib.models.lockfile import Lockfile as Req_Lockfile lockfile = None - if self.lockfile_exists: + if from_pipfile and self.pipfile_exists: + lockfile_dict = { + "default": self._lockfile["default"].copy(), + "develop": self._lockfile["develop"].copy() + } + lockfile_dict.update({"_meta": self.get_lockfile_meta()}) + lockfile = Req_Lockfile.from_data( + path=self.lockfile_location, data=lockfile_dict, meta_from_project=False + ) + elif self.lockfile_exists: try: lockfile = Req_Lockfile.load(self.lockfile_location) except OSError: @@ -770,29 +765,21 @@ class Project(object): ) lockfile._lockfile = lockfile.projectfile.model = _created_lockfile return lockfile - elif self.pipfile_exists: - lockfile_dict = { - "default": self._lockfile["default"].copy(), - "develop": self._lockfile["develop"].copy() - } - lockfile_dict.update({"_meta": self.get_lockfile_meta()}) - _created_lockfile = Req_Lockfile.from_data( - path=self.lockfile_location, data=lockfile_dict, meta_from_project=False - ) - lockfile._lockfile = _created_lockfile - return lockfile + else: + return self.get_or_create_lockfile(from_pipfile=True) def get_lockfile_meta(self): from .vendor.plette.lockfiles import PIPFILE_SPEC_CURRENT - sources = self.lockfile_content.get("_meta", {}).get("sources", []) - if not sources: - sources = self.pipfile_sources - elif not isinstance(sources, list): + if self.lockfile_exists: + sources = self.lockfile_content.get("_meta", {}).get("sources", []) + else: + sources = [dict(source) for source in self.parsed_pipfile["source"]] + if not isinstance(sources, list): sources = [sources,] return { "hash": {"sha256": self.calculate_pipfile_hash()}, "pipfile-spec": PIPFILE_SPEC_CURRENT, - "sources": sources, + "sources": [self.populate_source(s) for s in sources], "requires": self.parsed_pipfile.get("requires", {}) } @@ -846,7 +833,7 @@ class Project(object): @property def pipfile_sources(self): - if "source" not in self.parsed_pipfile: + if self.pipfile_is_empty or "source" not in self.parsed_pipfile: return [DEFAULT_SOURCE] # We need to make copies of the source info so we don't # accidentally modify the cache. See #2100 where values are @@ -867,8 +854,13 @@ class Project(object): else: return self.pipfile_sources + @property + def index_urls(self): + return [src.get("url") for src in self.sources] + def find_source(self, source): - """given a source, find it. + """ + Given a source, find it. source can be a url or an index name. """ @@ -881,23 +873,34 @@ class Project(object): source = self.get_source(url=source) return source - def get_source(self, name=None, url=None): + def get_source(self, name=None, url=None, refresh=False): + from .utils import is_url_equal + def find_source(sources, name=None, url=None): source = None if name: - source = [s for s in sources if s.get("name") == name] + source = next(iter( + s for s in sources if "name" in s and s["name"] == name + ), None) elif url: - source = [s for s in sources if url.startswith(s.get("url"))] - if source: - return first(source) + source = next(iter( + s for s in sources + if "url" in s and is_url_equal(url, s.get("url", "")) + ), None) + if source is not None: + return source - found_source = find_source(self.sources, name=name, url=url) - if found_source: - return found_source - found_source = find_source(self.pipfile_sources, name=name, url=url) - if found_source: - return found_source - raise SourceNotFound(name or url) + sources = (self.sources, self.pipfile_sources) + if refresh: + self.clear_pipfile_cache() + sources = reversed(sources) + found = next( + iter(find_source(source, name=name, url=url) for source in sources), None + ) + target = next(iter(t for t in (name, url) if t is not None)) + if found is None: + raise SourceNotFound(target) + return found def get_package_name_in_pipfile(self, package_name, dev=False): """Get the equivalent package name in pipfile""" @@ -942,17 +945,17 @@ class Project(object): # Don't re-capitalize file URLs or VCSs. if not isinstance(package, Requirement): package = Requirement.from_line(package.strip()) - _, converted = package.pipfile_entry + req_name, converted = package.pipfile_entry key = "dev-packages" if dev else "packages" # Set empty group if it doesn't exist yet. if key not in p: p[key] = {} - name = self.get_package_name_in_pipfile(package.name, dev) + name = self.get_package_name_in_pipfile(req_name, dev) if name and is_star(converted): # Skip for wildcard version return # Add the package to the group. - p[key][name or package.normalized_name] = converted + p[key][name or pep423_name(req_name)] = converted # Write Pipfile. self.write_toml(p) diff --git a/pipenv/pyenv.py b/pipenv/pyenv.py index bbfd1a97..941e5991 100644 --- a/pipenv/pyenv.py +++ b/pipenv/pyenv.py @@ -1,9 +1,8 @@ import operator import re -from .vendor import attr, delegator - from .environments import PIPENV_INSTALL_TIMEOUT +from .vendor import attr, delegator @attr.s diff --git a/pipenv/resolver.py b/pipenv/resolver.py index e87f3243..1219cc24 100644 --- a/pipenv/resolver.py +++ b/pipenv/resolver.py @@ -1,17 +1,57 @@ -import os -import sys +# -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function import json import logging +import os +import sys + os.environ["PIP_PYTHON_PATH"] = str(sys.executable) -def _patch_path(): +def find_site_path(pkg, site_dir=None): + import pkg_resources + if site_dir is not None: + site_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + working_set = pkg_resources.WorkingSet([site_dir] + sys.path[:]) + for dist in working_set: + root = dist.location + base_name = dist.project_name if dist.project_name else dist.key + name = None + if "top_level.txt" in dist.metadata_listdir(""): + name = next(iter([l.strip() for l in dist.get_metadata_lines("top_level.txt") if l is not None]), None) + if name is None: + name = pkg_resources.safe_name(base_name).replace("-", "_") + if not any(pkg == _ for _ in [base_name, name]): + continue + path_options = [name, "{0}.py".format(name)] + path_options = [os.path.join(root, p) for p in path_options if p is not None] + path = next(iter(p for p in path_options if os.path.exists(p)), None) + if path is not None: + return (dist, path) + return (None, None) + + +def _patch_path(pipenv_site=None): import site pipenv_libdir = os.path.dirname(os.path.abspath(__file__)) pipenv_site_dir = os.path.dirname(pipenv_libdir) - site.addsitedir(pipenv_site_dir) - for _dir in ("vendor", "patched"): + pipenv_dist = None + if pipenv_site is not None: + pipenv_dist, pipenv_path = find_site_path("pipenv", site_dir=pipenv_site) + else: + pipenv_dist, pipenv_path = find_site_path("pipenv", site_dir=pipenv_site_dir) + if pipenv_dist is not None: + pipenv_dist.activate() + else: + site.addsitedir(next(iter( + sitedir for sitedir in (pipenv_site, pipenv_site_dir) + if sitedir is not None + ), None)) + if pipenv_path is not None: + pipenv_libdir = pipenv_path + for _dir in ("vendor", "patched", pipenv_libdir): sys.path.insert(0, os.path.join(pipenv_libdir, _dir)) @@ -21,10 +61,16 @@ def get_parser(): parser.add_argument("--pre", action="store_true", default=False) parser.add_argument("--clear", action="store_true", default=False) parser.add_argument("--verbose", "-v", action="count", default=False) + parser.add_argument("--dev", action="store_true", default=False) parser.add_argument("--debug", action="store_true", default=False) parser.add_argument("--system", action="store_true", default=False) + parser.add_argument("--parse-only", action="store_true", default=False) + parser.add_argument("--pipenv-site", metavar="pipenv_site_dir", action="store", + default=os.environ.get("PIPENV_SITE_DIR")) parser.add_argument("--requirements-dir", metavar="requirements_dir", action="store", - default=os.environ.get("PIPENV_REQ_DIR")) + default=os.environ.get("PIPENV_REQ_DIR")) + parser.add_argument("--write", metavar="write", action="store", + default=os.environ.get("PIPENV_RESOLVER_FILE")) parser.add_argument("packages", nargs="*") return parser @@ -40,17 +86,645 @@ def handle_parsed_args(parsed): logging.getLogger("notpip").setLevel(logging.DEBUG) elif parsed.verbose > 0: logging.getLogger("notpip").setLevel(logging.INFO) + os.environ["PIPENV_VERBOSITY"] = str(parsed.verbose) if "PIPENV_PACKAGES" in os.environ: parsed.packages += os.environ.get("PIPENV_PACKAGES", "").strip().split("\n") return parsed -def _main(pre, clear, verbose, system, requirements_dir, packages): - os.environ["PIP_PYTHON_VERSION"] = ".".join([str(s) for s in sys.version_info[:3]]) - os.environ["PIP_PYTHON_PATH"] = str(sys.executable) +class Entry(object): + """A resolved entry from a resolver run""" + def __init__(self, name, entry_dict, project, resolver, reverse_deps=None, dev=False): + super(Entry, self).__init__() + from pipenv.vendor.requirementslib.models.utils import tomlkit_value_to_python + self.name = name + if isinstance(entry_dict, dict): + self.entry_dict = self.clean_initial_dict(entry_dict) + else: + self.entry_dict = entry_dict + self.project = project + section = "develop" if dev else "default" + pipfile_section = "dev-packages" if dev else "packages" + self.dev = dev + self.pipfile = tomlkit_value_to_python( + project.parsed_pipfile.get(pipfile_section, {}) + ) + self.lockfile = project.lockfile_content.get(section, {}) + self.pipfile_dict = self.pipfile.get(self.pipfile_name, {}) + if self.dev and self.name in project.lockfile_content.get("default", {}): + self.lockfile_dict = project.lockfile_content["default"][name] + else: + self.lockfile_dict = self.lockfile.get(name, entry_dict) + self.resolver = resolver + self.reverse_deps = reverse_deps + self._original_markers = None + self._markers = None + self._entry = None + self._lockfile_entry = None + self._pipfile_entry = None + self._parent_deps = [] + self._flattened_parents = [] + self._requires = None + self._deptree = None + self._parents_in_pipfile = [] + + @staticmethod + def make_requirement(name=None, entry=None, from_ireq=False): + from pipenv.vendor.requirementslib.models.requirements import Requirement + if from_ireq: + return Requirement.from_ireq(entry) + return Requirement.from_pipfile(name, entry) + + @classmethod + def clean_initial_dict(cls, entry_dict): + if not entry_dict.get("version", "").startswith("=="): + entry_dict["version"] = cls.clean_specifier(entry_dict.get("version", "")) + if "name" in entry_dict: + del entry_dict["name"] + return entry_dict + + @classmethod + def parse_pyparsing_exprs(cls, expr_iterable): + from pipenv.vendor.pyparsing import Literal, MatchFirst + keys = [] + expr_list = [] + expr = expr_iterable.copy() + if isinstance(expr, Literal) or ( + expr.__class__.__name__ == Literal.__name__ + ): + keys.append(expr.match) + elif isinstance(expr, MatchFirst) or ( + expr.__class__.__name__ == MatchFirst.__name__ + ): + expr_list = expr.exprs + elif isinstance(expr, list): + expr_list = expr + if expr_list: + for part in expr_list: + keys.extend(cls.parse_pyparsing_exprs(part)) + return keys + + @classmethod + def get_markers_from_dict(cls, entry_dict): + from pipenv.vendor.packaging import markers as packaging_markers + from pipenv.vendor.requirementslib.models.markers import normalize_marker_str + marker_keys = cls.parse_pyparsing_exprs(packaging_markers.VARIABLE) + markers = set() + keys_in_dict = [k for k in marker_keys if k in entry_dict] + markers = { + normalize_marker_str("{k} {v}".format(k=k, v=entry_dict.pop(k))) + for k in keys_in_dict + } + if "markers" in entry_dict: + markers.add(normalize_marker_str(entry_dict["markers"])) + if None in markers: + markers.remove(None) + if markers: + entry_dict["markers"] = " and ".join(list(markers)) + else: + markers = None + return markers, entry_dict + + @property + def markers(self): + self._markers, self.entry_dict = self.get_markers_from_dict(self.entry_dict) + return self._markers + + @markers.setter + def markers(self, markers): + if not markers: + marker_str = self.marker_to_str(markers) + if marker_str: + self._entry = self.entry.merge_markers(marker_str) + self._markers = self.marker_to_str(self._entry.markers) + entry_dict = self.entry_dict.copy() + entry_dict["markers"] = self.marker_to_str(self._entry.markers) + self.entry_dict = entry_dict + + @property + def original_markers(self): + original_markers, lockfile_dict = self.get_markers_from_dict( + self.lockfile_dict + ) + self.lockfile_dict = lockfile_dict + self._original_markers = self.marker_to_str(original_markers) + return self._original_markers + + @staticmethod + def marker_to_str(marker): + from pipenv.vendor.requirementslib.models.markers import normalize_marker_str + if not marker: + return None + from pipenv.vendor import six + from pipenv.vendor.vistir.compat import Mapping + marker_str = None + if isinstance(marker, Mapping): + marker_dict, _ = Entry.get_markers_from_dict(marker) + if marker_dict: + marker_str = "{0}".format(marker_dict.popitem()[1]) + elif isinstance(marker, (list, set, tuple)): + marker_str = " and ".join([normalize_marker_str(m) for m in marker if m]) + elif isinstance(marker, six.string_types): + marker_str = "{0}".format(normalize_marker_str(marker)) + if isinstance(marker_str, six.string_types): + return marker_str + return None + + def get_cleaned_dict(self, keep_outdated=False): + if keep_outdated and self.is_updated: + self.validate_constraints() + self.ensure_least_updates_possible() + elif not keep_outdated: + self.validate_constraints() + if self.entry.extras != self.lockfile_entry.extras: + entry_extras = list(self.entry.extras) + if self.lockfile_entry.extras: + entry_extras.extend(list(self.lockfile_entry.extras)) + self._entry.req.extras = entry_extras + self.entry_dict["extras"] = self.entry.extras + if self.original_markers and not self.markers: + original_markers = self.marker_to_str(self.original_markers) + self.markers = original_markers + self.entry_dict["markers"] = self.marker_to_str(original_markers) + entry_hashes = set(self.entry.hashes) + locked_hashes = set(self.lockfile_entry.hashes) + if entry_hashes != locked_hashes and not self.is_updated: + self.entry_dict["hashes"] = list(entry_hashes | locked_hashes) + self.entry_dict["name"] = self.name + if "version" in self.entry_dict: + self.entry_dict["version"] = self.strip_version(self.entry_dict["version"]) + _, self.entry_dict = self.get_markers_from_dict(self.entry_dict) + return self.entry_dict + + @property + def lockfile_entry(self): + if self._lockfile_entry is None: + self._lockfile_entry = self.make_requirement(self.name, self.lockfile_dict) + return self._lockfile_entry + + @lockfile_entry.setter + def lockfile_entry(self, entry): + self._lockfile_entry = entry + + @property + def pipfile_entry(self): + if self._pipfile_entry is None: + self._pipfile_entry = self.make_requirement(self.pipfile_name, self.pipfile_dict) + return self._pipfile_entry + + @property + def entry(self): + if self._entry is None: + self._entry = self.make_requirement(self.name, self.entry_dict) + return self._entry + + @property + def normalized_name(self): + return self.entry.normalized_name + + @property + def pipfile_name(self): + return self.project.get_package_name_in_pipfile(self.name, dev=self.dev) + + @property + def is_in_pipfile(self): + return True if self.pipfile_name else False + + @property + def pipfile_packages(self): + return self.project.pipfile_package_names["dev" if self.dev else "default"] + + def create_parent(self, name, specifier="*"): + parent = self.create(name, specifier, self.project, self.resolver, + self.reverse_deps, self.dev) + parent._deptree = self.deptree + return parent + + @property + def deptree(self): + if not self._deptree: + self._deptree = self.project.environment.get_package_requirements() + return self._deptree + + @classmethod + def create(cls, name, entry_dict, project, resolver, reverse_deps=None, dev=False): + return cls(name, entry_dict, project, resolver, reverse_deps, dev) + + @staticmethod + def clean_specifier(specifier): + from pipenv.vendor.packaging.specifiers import Specifier + if not any(specifier.startswith(k) for k in Specifier._operators.keys()): + if specifier.strip().lower() in ["any", "", "*"]: + return "*" + specifier = "=={0}".format(specifier) + elif specifier.startswith("==") and specifier.count("=") > 3: + specifier = "=={0}".format(specifier.lstrip("=")) + return specifier + + @staticmethod + def strip_version(specifier): + from pipenv.vendor.packaging.specifiers import Specifier + op = next(iter( + k for k in Specifier._operators.keys() if specifier.startswith(k) + ), None) + if op: + specifier = specifier[len(op):] + while op: + op = next(iter( + k for k in Specifier._operators.keys() if specifier.startswith(k) + ), None) + if op: + specifier = specifier[len(op):] + return specifier + + @property + def parent_deps(self): + if not self._parent_deps: + self._parent_deps = self.get_parent_deps(unnest=False) + return self._parent_deps + + @property + def flattened_parents(self): + if not self._flattened_parents: + self._flattened_parents = self.get_parent_deps(unnest=True) + return self._flattened_parents + + @property + def parents_in_pipfile(self): + if not self._parents_in_pipfile: + self._parents_in_pipfile = [ + p for p in self.flattened_parents + if p.normalized_name in self.pipfile_packages + ] + return self._parents_in_pipfile + + @property + def is_updated(self): + return self.entry.specifiers != self.lockfile_entry.specifiers + + @property + def requirements(self): + if not self._requires: + self._requires = next(iter( + self.project.environment.get_package_requirements(self.name) + ), {}) + return self._requires + + @property + def updated_version(self): + version = self.entry.specifiers + return self.strip_version(version) + + @property + def updated_specifier(self): + # type: () -> str + return self.entry.specifiers + + @property + def original_specifier(self): + # type: () -> str + return self.lockfile_entry.specifiers + + @property + def original_version(self): + if self.original_specifier: + return self.strip_version(self.original_specifier) + return None + + def validate_specifiers(self): + if self.is_in_pipfile and not self.pipfile_entry.editable: + return self.pipfile_entry.requirement.specifier.contains(self.updated_version) + return True + + def get_dependency(self, name): + if self.requirements: + return next(iter( + dep for dep in self.requirements.get("dependencies", []) + if dep and dep.get("package_name", "") == name + ), {}) + return {} + + def get_parent_deps(self, unnest=False): + from pipenv.vendor.packaging.specifiers import Specifier + parents = [] + for spec in self.reverse_deps.get(self.normalized_name, {}).get("parents", set()): + spec_match = next(iter(c for c in Specifier._operators if c in spec), None) + name = spec + parent = None + if spec_match is not None: + spec_index = spec.index(spec_match) + specifier = self.clean_specifier(spec[spec_index:len(spec_match)]).strip() + name_start = spec_index + len(spec_match) + name = spec[name_start:].strip() + parent = self.create_parent(name, specifier) + else: + name = spec + parent = self.create_parent(name) + if parent is not None: + parents.append(parent) + if not unnest or parent.pipfile_name is not None: + continue + if self.reverse_deps.get(parent.normalized_name, {}).get("parents", set()): + parents.extend(parent.flattened_parents) + return parents + + def ensure_least_updates_possible(self): + """ + Mutate the current entry to ensure that we are making the smallest amount of + changes possible to the existing lockfile -- this will keep the old locked + versions of packages if they satisfy new constraints. + + :return: None + """ + constraints = self.get_constraints() + can_use_original = True + can_use_updated = True + satisfied_by_versions = set() + for constraint in constraints: + if not constraint.specifier.contains(self.original_version): + self.can_use_original = False + if not constraint.specifier.contains(self.updated_version): + self.can_use_updated = False + satisfied_by_value = getattr(constraint, "satisfied_by", None) + if satisfied_by_value: + satisfied_by = "{0}".format( + self.clean_specifier(str(satisfied_by_value.version)) + ) + satisfied_by_versions.add(satisfied_by) + if can_use_original: + self.entry_dict = self.lockfile_dict.copy() + elif can_use_updated: + if len(satisfied_by_versions) == 1: + self.entry_dict["version"] = next(iter( + sat_by for sat_by in satisfied_by_versions if sat_by + ), None) + hashes = None + if self.lockfile_entry.specifiers == satisfied_by: + ireq = self.lockfile_entry.as_ireq() + if not self.lockfile_entry.hashes and self.resolver._should_include_hash(ireq): + hashes = self.resolver.get_hash(ireq) + else: + hashes = self.lockfile_entry.hashes + else: + if self.resolver._should_include_hash(constraint): + hashes = self.resolver.get_hash(constraint) + if hashes: + self.entry_dict["hashes"] = list(hashes) + self._entry.hashes = frozenset(hashes) + else: + # check for any parents, since they depend on this and the current + # installed versions are not compatible with the new version, so + # we will need to update the top level dependency if possible + self.check_flattened_parents() + + def get_constraints(self): + """ + Retrieve all of the relevant constraints, aggregated from the pipfile, resolver, + and parent dependencies and their respective conflict resolution where possible. + + :return: A set of **InstallRequirement** instances representing constraints + :rtype: Set + """ + constraints = { + c for c in self.resolver.parsed_constraints + if c and c.name == self.entry.name + } + pipfile_constraint = self.get_pipfile_constraint() + if pipfile_constraint and not (self.pipfile_entry.editable or pipfile_constraint.editable): + constraints.add(pipfile_constraint) + return constraints + + def get_pipfile_constraint(self): + """ + Retrieve the version constraint from the pipfile if it is specified there, + otherwise check the constraints of the parent dependencies and their conflicts. + + :return: An **InstallRequirement** instance representing a version constraint + """ + if self.is_in_pipfile: + return self.pipfile_entry.as_ireq() + return self.constraint_from_parent_conflicts() + + def constraint_from_parent_conflicts(self): + """ + Given a resolved entry with multiple parent dependencies with different + constraints, searches for the resolution that satisfies all of the parent + constraints. + + :return: A new **InstallRequirement** satisfying all parent constraints + :raises: :exc:`~pipenv.exceptions.DependencyConflict` if resolution is impossible + """ + # ensure that we satisfy the parent dependencies of this dep + from pipenv.vendor.packaging.specifiers import Specifier + parent_dependencies = set() + has_mismatch = False + can_use_original = True + for p in self.parent_deps: + # updated dependencies should be satisfied since they were resolved already + if p.is_updated: + continue + # parents with no requirements can't conflict + if not p.requirements: + continue + needed = p.requirements.get("dependencies", []) + entry_ref = p.get_dependency(self.name) + required = entry_ref.get("required_version", "*") + required = self.clean_specifier(required) + parent_requires = self.make_requirement(self.name, required) + parent_dependencies.add("{0} => {1} ({2})".format(p.name, self.name, required)) + # use pre=True here or else prereleases dont satisfy constraints + if parent_requires.requirement.specifier and ( + not parent_requires.requirement.specifier.contains(self.original_version, prereleases=True) + ): + can_use_original = False + if parent_requires.requirement.specifier and ( + not parent_requires.requirement.specifier.contains(self.updated_version, prereleases=True) + ): + if not self.entry.editable and self.updated_version != self.original_version: + has_mismatch = True + if has_mismatch and not can_use_original: + from pipenv.exceptions import DependencyConflict + msg = ( + "Cannot resolve {0} ({1}) due to conflicting parent dependencies: " + "\n\t{2}".format( + self.name, self.updated_version, "\n\t".join(parent_dependencies) + ) + ) + raise DependencyConflict(msg) + elif can_use_original: + return self.lockfile_entry.as_ireq() + return self.entry.as_ireq() + + def validate_constraints(self): + """ + Retrieves the full set of available constraints and iterate over them, validating + that they exist and that they are not causing unresolvable conflicts. + + :return: True if the constraints are satisfied by the resolution provided + :raises: :exc:`pipenv.exceptions.DependencyConflict` if the constraints dont exist + """ + constraints = self.get_constraints() + for constraint in constraints: + try: + constraint.check_if_exists(False) + except Exception: + from pipenv.exceptions import DependencyConflict + from pipenv.environments import is_verbose + if is_verbose(): + print("Tried constraint: {0!r}".format(constraint), file=sys.stderr) + msg = ( + "Cannot resolve conflicting version {0}{1} while {2}{3} is " + "locked.".format( + self.name, self.updated_specifier, self.old_name, self.old_specifiers + ) + ) + raise DependencyConflict(msg) + return True + + def check_flattened_parents(self): + for parent in self.parents_in_pipfile: + if not parent.updated_specifier: + continue + if not parent.validate_specifiers(): + from pipenv.exceptions import DependencyConflict + msg = ( + "Cannot resolve conflicting versions: (Root: {0}) {1}{2} (Pipfile) " + "Incompatible with {3}{4} (resolved)\n".format( + self.name, parent.pipfile_name, + parent.pipfile_entry.requirement.specifiers, parent.name, + parent.updated_specifiers + ) + ) + raise DependencyConflict(msg) + + def __getattribute__(self, key): + result = None + old_version = ["was_", "had_", "old_"] + new_version = ["is_", "has_", "new_"] + if any(key.startswith(v) for v in new_version): + entry = Entry.__getattribute__(self, "entry") + try: + keystart = key.index("_") + 1 + try: + result = getattr(entry, key[keystart:]) + except AttributeError: + result = getattr(entry, key) + except AttributeError: + result = super(Entry, self).__getattribute__(key) + return result + if any(key.startswith(v) for v in old_version): + lockfile_entry = Entry.__getattribute__(self, "lockfile_entry") + try: + keystart = key.index("_") + 1 + try: + result = getattr(lockfile_entry, key[keystart:]) + except AttributeError: + result = getattr(lockfile_entry, key) + except AttributeError: + result = super(Entry, self).__getattribute__(key) + return result + return super(Entry, self).__getattribute__(key) + + +def clean_results(results, resolver, project, dev=False): + from pipenv.utils import translate_markers + if not project.lockfile_exists: + return results + lockfile = project.lockfile_content + section = "develop" if dev else "default" + pipfile_section = "dev-packages" if dev else "packages" + reverse_deps = project.environment.reverse_dependencies() + new_results = [r for r in results if r["name"] not in lockfile[section]] + for result in results: + name = result.get("name") + entry_dict = result.copy() + entry = Entry(name, entry_dict, project, resolver, reverse_deps=reverse_deps, dev=dev) + entry_dict = translate_markers(entry.get_cleaned_dict(keep_outdated=False)) + new_results.append(entry_dict) + return new_results + + +def clean_outdated(results, resolver, project, dev=False): + from pipenv.vendor.requirementslib.models.requirements import Requirement + from pipenv.environments import is_verbose + if not project.lockfile_exists: + return results + lockfile = project.lockfile_content + section = "develop" if dev else "default" + pipfile_section = "dev-packages" if dev else "packages" + pipfile = project.parsed_pipfile[pipfile_section] + reverse_deps = project.environment.reverse_dependencies() + deptree = project.environment.get_package_requirements() + overlapping_results = [r for r in results if r["name"] in lockfile[section]] + new_results = [r for r in results if r["name"] not in lockfile[section]] + for result in results: + name = result.get("name") + entry_dict = result.copy() + entry = Entry(name, entry_dict, project, resolver, reverse_deps=reverse_deps, dev=dev) + # The old entry was editable but this one isnt; prefer the old one + # TODO: Should this be the case for all locking? + if entry.was_editable and not entry.is_editable: + continue + lockfile_entry = lockfile[section].get(name, None) + if not lockfile_entry: + alternate_section = "develop" if not dev else "default" + if name in lockfile[alternate_section]: + lockfile_entry = lockfile[alternate_section][name] + if lockfile_entry and not entry.is_updated: + old_markers = next(iter(m for m in ( + entry.lockfile_entry.markers, lockfile_entry.get("markers", None) + ) if m is not None), None) + new_markers = entry_dict.get("markers", None) + if old_markers: + old_markers = Entry.marker_to_str(old_markers) + if old_markers and not new_markers: + entry.markers = old_markers + elif new_markers and not old_markers: + del entry.entry_dict["markers"] + entry._entry.req.req.marker = None + entry._entry.markers = None + # if the entry has not changed versions since the previous lock, + # don't introduce new markers since that is more restrictive + # if entry.has_markers and not entry.had_markers and not entry.is_updated: + # do make sure we retain the original markers for entries that are not changed + entry_dict = entry.get_cleaned_dict(keep_outdated=True) + new_results.append(entry_dict) + return new_results + + +def parse_packages(packages, pre, clear, system, requirements_dir=None): + from pipenv.vendor.requirementslib.models.requirements import Requirement + from pipenv.vendor.vistir.contextmanagers import cd, temp_path + from pipenv.utils import parse_indexes + parsed_packages = [] + for package in packages: + indexes, trusted_hosts, line = parse_indexes(package) + line = " ".join(line) + pf = dict() + req = Requirement.from_line(line) + if not req.name: + with temp_path(), cd(req.req.setup_info.base_dir): + sys.path.insert(0, req.req.setup_info.base_dir) + req.req._setup_info.get_info() + req.update_name_from_path(req.req.setup_info.base_dir) + try: + name, entry = req.pipfile_entry + except Exception: + continue + else: + if name is not None and entry is not None: + pf[name] = entry + parsed_packages.append(pf) + print("RESULTS:") + if parsed_packages: + print(json.dumps(parsed_packages)) + else: + print(json.dumps([])) + + +def resolve_packages(pre, clear, verbose, system, write, requirements_dir, packages): from pipenv.utils import create_mirror_source, resolve_deps, replace_pypi_sources - pypi_mirror_source = ( create_mirror_source(os.environ["PIPENV_PYPI_MIRROR"]) if "PIPENV_PYPI_MIRROR" in os.environ @@ -58,6 +732,8 @@ def _main(pre, clear, verbose, system, requirements_dir, packages): ) def resolve(packages, pre, project, sources, clear, system, requirements_dir=None): + from pipenv.patched.piptools import logging as piptools_logging + piptools_logging.log.verbosity = 1 if verbose else 0 return resolve_deps( packages, which, @@ -75,7 +751,8 @@ def _main(pre, clear, verbose, system, requirements_dir, packages): if pypi_mirror_source else project.pipfile_sources ) - results = resolve( + keep_outdated = os.environ.get("PIPENV_KEEP_OUTDATED", False) + results, resolver = resolve( packages, pre=pre, project=project, @@ -84,43 +761,56 @@ def _main(pre, clear, verbose, system, requirements_dir, packages): system=system, requirements_dir=requirements_dir, ) - print("RESULTS:") - if results: - print(json.dumps(results)) + if keep_outdated: + results = clean_outdated(results, resolver, project) else: - print(json.dumps([])) + results = clean_results(results, resolver, project) + if write: + with open(write, "w") as fh: + if not results: + json.dump([], fh) + else: + json.dump(results, fh) + else: + print("RESULTS:") + if results: + print(json.dumps(results)) + else: + print(json.dumps([])) + + +def _main(pre, clear, verbose, system, write, requirements_dir, packages, parse_only=False): + os.environ["PIP_PYTHON_VERSION"] = ".".join([str(s) for s in sys.version_info[:3]]) + os.environ["PIP_PYTHON_PATH"] = str(sys.executable) + if parse_only: + parse_packages( + packages, + pre=pre, + clear=clear, + system=system, + requirements_dir=requirements_dir, + ) + else: + resolve_packages(pre, clear, verbose, system, write, requirements_dir, packages) def main(): - _patch_path() - import warnings - from pipenv.vendor.vistir.compat import ResourceWarning - warnings.simplefilter("ignore", category=ResourceWarning) - import io - import six - if six.PY3: - import atexit - stdout_wrapper = io.TextIOWrapper(sys.stdout.buffer, encoding='utf8') - atexit.register(stdout_wrapper.close) - stderr_wrapper = io.TextIOWrapper(sys.stderr.buffer, encoding='utf8') - atexit.register(stderr_wrapper.close) - sys.stdout = stdout_wrapper - sys.stderr = stderr_wrapper - else: - from pipenv._compat import force_encoding - force_encoding() - os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = str("1") - os.environ["PYTHONIOENCODING"] = str("utf-8") parser = get_parser() parsed, remaining = parser.parse_known_args() - # sys.argv = remaining + _patch_path(pipenv_site=parsed.pipenv_site) + import warnings + from pipenv.vendor.vistir.compat import ResourceWarning + from pipenv.vendor.vistir.misc import replace_with_text_stream + warnings.simplefilter("ignore", category=ResourceWarning) + replace_with_text_stream("stdout") + replace_with_text_stream("stderr") + os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = str("1") + os.environ["PYTHONIOENCODING"] = str("utf-8") + os.environ["PYTHONUNBUFFERED"] = str("1") parsed = handle_parsed_args(parsed) - _main(parsed.pre, parsed.clear, parsed.verbose, parsed.system, - parsed.requirements_dir, parsed.packages) + _main(parsed.pre, parsed.clear, parsed.verbose, parsed.system, parsed.write, + parsed.requirements_dir, parsed.packages, parse_only=parsed.parse_only) if __name__ == "__main__": - _patch_path() - from pipenv.vendor import colorama - colorama.init() main() diff --git a/pipenv/shells.py b/pipenv/shells.py index 352c01f4..1d8b073a 100644 --- a/pipenv/shells.py +++ b/pipenv/shells.py @@ -5,10 +5,10 @@ import signal import subprocess import sys -from .environments import PIPENV_SHELL_EXPLICIT, PIPENV_SHELL, PIPENV_EMULATOR -from .vendor.vistir.compat import get_terminal_size, Path -from .vendor.vistir.contextmanagers import temp_environ +from .environments import PIPENV_EMULATOR, PIPENV_SHELL, PIPENV_SHELL_EXPLICIT from .vendor import shellingham +from .vendor.vistir.compat import Path, get_terminal_size +from .vendor.vistir.contextmanagers import temp_environ ShellDetectionFailure = shellingham.ShellDetectionFailure diff --git a/pipenv/utils.py b/pipenv/utils.py index 84a4c105..2b0a8cdb 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -1,61 +1,56 @@ # -*- coding: utf-8 -*- +from __future__ import print_function import contextlib import errno import logging import os +import posixpath import re +import signal import shutil import stat import sys +import warnings + +from contextlib import contextmanager +from distutils.spawn import find_executable + +import six import toml import tomlkit -import warnings + +from click import echo as click_echo +from six.moves.urllib.parse import urlparse +from .vendor.vistir.compat import ResourceWarning, lru_cache, Mapping, Sequence, Set +from .vendor.vistir.misc import fs_str, run import crayons import parse -import six -from click import echo as click_echo -from first import first -from vistir.misc import fs_str +from . import environments +from .exceptions import ( + PipenvUsageError, RequirementError, PipenvCmdError, ResolutionFailure +) +from .pep508checker import lookup +from .vendor.urllib3 import util as urllib3_util -six.add_move(six.MovedAttribute("Mapping", "collections", "collections.abc")) -six.add_move(six.MovedAttribute("Sequence", "collections", "collections.abc")) -from six.moves import Mapping, Sequence -from vistir.compat import ResourceWarning - -try: - from weakref import finalize -except ImportError: - try: - from .vendor.backports.weakref import finalize - except ImportError: - - class finalize(object): - def __init__(self, *args, **kwargs): - logging.warn("weakref.finalize unavailable, not cleaning...") - - def detach(self): - return False +if environments.MYPY_RUNNING: + from typing import Tuple, Dict, Any, List, Union, Optional, Text + from .vendor.requirementslib.models.requirements import Requirement, Line + from .vendor.requirementslib.models.pipfile import Pipfile + from .vendor.packaging.markers import Marker + from .vendor.packaging.specifiers import Specifier + from .project import Project, TSource logging.basicConfig(level=logging.ERROR) -from distutils.spawn import find_executable -from contextlib import contextmanager -from . import environments -from .pep508checker import lookup - -from six.moves.urllib.parse import urlparse -from urllib3 import util as urllib3_util - - specifiers = [k for k in lookup.keys()] # List of version control systems we support. VCS_LIST = ("git", "svn", "hg", "bzr") SCHEME_LIST = ("http://", "https://", "ftp://", "ftps://", "file://") -requests_session = None +requests_session = None # type: ignore def _get_requests_session(): @@ -98,22 +93,76 @@ def cleanup_toml(tml): def convert_toml_outline_tables(parsed): """Converts all outline tables to inline tables.""" - if isinstance(parsed, tomlkit.container.Container): - empty_inline_table = tomlkit.inline_table - else: - empty_inline_table = toml.TomlDecoder().get_empty_inline_table + def convert_tomlkit_table(section): + for key, value in section._body: + if not key: + continue + if hasattr(value, "keys") and not isinstance(value, tomlkit.items.InlineTable): + table = tomlkit.inline_table() + table.update(value.value) + section[key.key] = table + + def convert_toml_table(section): + for package, value in section.items(): + if hasattr(value, "keys") and not isinstance(value, toml.decoder.InlineTableDict): + table = toml.TomlDecoder().get_empty_inline_table() + table.update(value) + section[package] = table + + is_tomlkit_parsed = isinstance(parsed, tomlkit.container.Container) for section in ("packages", "dev-packages"): table_data = parsed.get(section, {}) - for package, value in table_data.items(): - if hasattr(value, "keys") and not isinstance( - value, (tomlkit.items.InlineTable, toml.decoder.InlineTableDict) - ): - table = empty_inline_table() - table.update(value) - table_data[package] = table + if not table_data: + continue + if is_tomlkit_parsed: + convert_tomlkit_table(table_data) + else: + convert_toml_table(table_data) + return parsed +def run_command(cmd, *args, **kwargs): + """ + Take an input command and run it, handling exceptions and error codes and returning + its stdout and stderr. + + :param cmd: The list of command and arguments. + :type cmd: list + :returns: A 2-tuple of the output and error from the command + :rtype: Tuple[str, str] + :raises: exceptions.PipenvCmdError + """ + + from pipenv.vendor import delegator + from ._compat import decode_for_output + from .cmdparse import Script + catch_exceptions = kwargs.pop("catch_exceptions", True) + if isinstance(cmd, (six.string_types, list, tuple)): + cmd = Script.parse(cmd) + if not isinstance(cmd, Script): + raise TypeError("Command input must be a string, list or tuple") + if "env" not in kwargs: + kwargs["env"] = os.environ.copy() + kwargs["env"]["PYTHONIOENCODING"] = "UTF-8" + try: + cmd_string = cmd.cmdify() + except TypeError: + click_echo("Error turning command into string: {0}".format(cmd), err=True) + sys.exit(1) + if environments.is_verbose(): + click_echo("Running command: $ {0}".format(cmd_string, err=True)) + c = delegator.run(cmd_string, *args, **kwargs) + return_code = c.return_code + if environments.is_verbose(): + click_echo("Command output: {0}".format( + crayons.blue(decode_for_output(c.out)) + ), err=True) + if not c.ok and catch_exceptions: + raise PipenvCmdError(cmd_string, c.out, c.err, return_code) + return c + + def parse_python_version(output): """Parse a Python version output returned by `python --version`. @@ -205,54 +254,137 @@ def prepare_pip_source_args(sources, pip_args=None): pip_args = [] if sources: # Add the source to notpip. - pip_args.extend(["-i", sources[0]["url"]]) + package_url = sources[0].get("url") + if not package_url: + raise PipenvUsageError("[[source]] section does not contain a URL.") + pip_args.extend(["-i", package_url]) # Trust the host if it's not verified. if not sources[0].get("verify_ssl", True): + url_parts = urllib3_util.parse_url(package_url) + url_port = ":{0}".format(url_parts.port) if url_parts.port else "" pip_args.extend( - ["--trusted-host", urllib3_util.parse_url(sources[0]["url"]).host] + ["--trusted-host", "{0}{1}".format(url_parts.host, url_port)] ) # Add additional sources as extra indexes. if len(sources) > 1: for source in sources[1:]: - pip_args.extend(["--extra-index-url", source["url"]]) + url = source.get("url") + if not url: # not harmless, just don't continue + continue + pip_args.extend(["--extra-index-url", url]) # Trust the host if it's not verified. if not source.get("verify_ssl", True): + url_parts = urllib3_util.parse_url(url) + url_port = ":{0}".format(url_parts.port) if url_parts.port else "" pip_args.extend( - ["--trusted-host", urllib3_util.parse_url(source["url"]).host] + ["--trusted-host", "{0}{1}".format(url_parts.host, url_port)] ) return pip_args -def get_resolver_metadata(deps, index_lookup, markers_lookup, project, sources): - from .vendor.requirementslib.models.requirements import Requirement - constraints = [] - for dep in deps: - if not dep: - continue - url = None - indexes, trusted_hosts, remainder = parse_indexes(dep) - if indexes: - url = indexes[0] - dep = " ".join(remainder) - req = Requirement.from_line(dep) - constraints.append(req.constraint_line) +def get_project_index(index=None, trusted_hosts=None, project=None): + # type: (Optional[Union[str, TSource]], Optional[List[str]], Optional[Project]) -> TSource + from .project import SourceNotFound + if not project: + from .core import project + if trusted_hosts is None: + trusted_hosts = [] + if isinstance(index, Mapping): + return project.find_source(index.get("url")) + try: + source = project.find_source(index) + except SourceNotFound: + index_url = urllib3_util.parse_url(index) + src_name = project.src_name_from_url(index) + verify_ssl = index_url.host not in trusted_hosts + source = {"url": index, "verify_ssl": verify_ssl, "name": src_name} + return source - if url: - index_lookup[req.name] = project.get_source(url=url).get("name") - # strip the marker and re-add it later after resolution - # but we will need a fallback in case resolution fails - # eg pypiwin32 - if req.markers: - markers_lookup[req.name] = req.markers.replace('"', "'") - return constraints + +def get_source_list( + index=None, # type: Optional[Union[str, TSource]] + extra_indexes=None, # type: Optional[List[str]] + trusted_hosts=None, # type: Optional[List[str]] + pypi_mirror=None, # type: Optional[str] + project=None, # type: Optional[Project] +): + # type: (...) -> List[TSource] + sources = [] # type: List[TSource] + if not project: + from .core import project + if index: + sources.append(get_project_index(index)) + if extra_indexes: + if isinstance(extra_indexes, six.string_types): + extra_indexes = [extra_indexes,] + for source in extra_indexes: + extra_src = get_project_index(source) + if not sources or extra_src["url"] != sources[0]["url"]: + sources.append(extra_src) + else: + for source in project.pipfile_sources: + if not sources or source["url"] != sources[0]["url"]: + sources.append(source) + if not sources: + sources = project.pipfile_sources[:] + if pypi_mirror: + sources = [ + create_mirror_source(pypi_mirror) if is_pypi_url(source["url"]) else source + for source in sources + ] + return sources + + +def get_indexes_from_requirement(req, project=None, index=None, extra_indexes=None, trusted_hosts=None, pypi_mirror=None): + # type: (Requirement, Optional[Project], Optional[Text], Optional[List[Text]], Optional[List[Text]], Optional[Text]) -> Tuple[TSource, List[TSource], List[Text]] + if not project: + from .core import project + index_sources = [] # type: List[TSource] + if not trusted_hosts: + trusted_hosts = [] # type: List[Text] + if extra_indexes is None: + extra_indexes = [] + project_indexes = project.pipfile_sources[:] + indexes = [] + if req.index: + indexes.append(req.index) + if getattr(req, "extra_indexes", None): + if not isinstance(req.extra_indexes, list): + indexes.append(req.extra_indexes) + else: + indexes.extend(req.extra_indexes) + indexes.extend(project_indexes) + if len(indexes) > 1: + index, extra_indexes = indexes[0], indexes[1:] + index_sources = get_source_list(index=index, extra_indexes=extra_indexes, trusted_hosts=trusted_hosts, pypi_mirror=pypi_mirror, project=project) + if len(index_sources) > 1: + index_source, extra_index_sources = index_sources[0], index_sources[1:] + else: + index_source, extra_index_sources = index_sources[0], [] + return index_source, extra_index_sources + + +@lru_cache() +def get_pipenv_sitedir(): + # type: () -> Optional[str] + import pkg_resources + site_dir = next( + iter(d for d in pkg_resources.working_set if d.key.lower() == "pipenv"), None + ) + if site_dir is not None: + return site_dir.location + return None class Resolver(object): - def __init__(self, constraints, req_dir, project, sources, clear=False, pre=False): + def __init__( + self, constraints, req_dir, project, sources, index_lookup=None, + markers_lookup=None, skipped=None, clear=False, pre=False + ): from pipenv.patched.piptools import logging as piptools_logging if environments.is_verbose(): logging.log.verbose = True - piptools_logging.log.verbose = True + piptools_logging.log.verbosity = environments.PIPENV_VERBOSITY self.initial_constraints = constraints self.req_dir = req_dir self.project = project @@ -262,6 +394,11 @@ class Resolver(object): self.clear = clear self.pre = pre self.results = None + self.markers_lookup = markers_lookup if markers_lookup is not None else {} + self.index_lookup = index_lookup if index_lookup is not None else {} + self.skipped = skipped if skipped is not None else {} + self.markers = {} + self.requires_python_markers = {} self._pip_args = None self._constraints = None self._parsed_constraints = None @@ -279,16 +416,264 @@ class Resolver(object): "sources={self.sources})>".format(self=self) ) - def _get_pip_command(self): - from pip_shims.shims import Command + @staticmethod + @lru_cache() + def _get_pip_command(): + from .vendor.pip_shims.shims import Command, cmdoptions class PipCommand(Command): """Needed for pip-tools.""" name = "PipCommand" - from pipenv.patched.piptools.scripts.compile import get_pip_command - return get_pip_command() + from pipenv.patched.piptools.pip import get_pip_command + pip_cmd = get_pip_command() + pip_cmd.parser.add_option(cmdoptions.no_use_pep517()) + pip_cmd.parser.add_option(cmdoptions.use_pep517()) + pip_cmd.parser.add_option(cmdoptions.no_build_isolation()) + return pip_cmd + + @classmethod + def get_metadata( + cls, + deps, # type: List[str] + index_lookup, # type: Dict[str, str] + markers_lookup, # type: Dict[str, str] + project, # type: Project + sources, # type: Dict[str, str] + req_dir=None, # type: Optional[str] + pre=False, # type: bool + clear=False, # type: bool + ): + # type: (...) -> Tuple[Set[str], Dict[str, Dict[str, Union[str, bool, List[str]]]], Dict[str, str], Dict[str, str]] + constraints = set() # type: Set[str] + skipped = dict() # type: Dict[str, Dict[str, Union[str, bool, List[str]]]] + if index_lookup is None: + index_lookup = {} + if markers_lookup is None: + markers_lookup = {} + if not req_dir: + from .vendor.vistir.path import create_tracked_tempdir + req_dir = create_tracked_tempdir(prefix="pipenv-", suffix="-reqdir") + transient_resolver = cls( + [], req_dir, project, sources, index_lookup=index_lookup, + markers_lookup=markers_lookup, clear=clear, pre=pre + ) + for dep in deps: + if not dep: + continue + req, req_idx, markers_idx = cls.parse_line( + dep, index_lookup=index_lookup, markers_lookup=markers_lookup, project=project + ) + index_lookup.update(req_idx) + markers_lookup.update(markers_idx) + constraint_update, lockfile_update = cls.get_deps_from_req( + req, resolver=transient_resolver + ) + constraints |= constraint_update + skipped.update(lockfile_update) + return constraints, skipped, index_lookup, markers_lookup + + @classmethod + def parse_line( + cls, + line, # type: str + index_lookup=None, # type: Dict[str, str] + markers_lookup=None, # type: Dict[str, str] + project=None # type: Optional[Project] + ): + # type: (...) -> Tuple[Requirement, Dict[str, str], Dict[str, str]] + from .vendor.requirementslib.models.requirements import Requirement + if index_lookup is None: + index_lookup = {} + if markers_lookup is None: + markers_lookup = {} + if project is None: + from .project import Project + project = Project() + url = None + indexes, trusted_hosts, remainder = parse_indexes(line) + if indexes: + url = indexes[0] + line = " ".join(remainder) + req = None # type: Requirement + try: + req = Requirement.from_line(line) + except ValueError: + raise ResolutionFailure("Failed to resolve requirement from line: {0!s}".format(line)) + if url: + try: + index_lookup[req.normalized_name] = project.get_source( + url=url, refresh=True).get("name") + except TypeError: + pass + try: + req.normalized_name + except TypeError: + raise RequirementError(req=req) + # strip the marker and re-add it later after resolution + # but we will need a fallback in case resolution fails + # eg pypiwin32 + if req.markers: + markers_lookup[req.normalized_name] = req.markers.replace('"', "'") + return req, index_lookup, markers_lookup + + @classmethod + def get_deps_from_line(cls, line): + # type: (str) -> Tuple[Set[str], Dict[str, Dict[str, Union[str, bool, List[str]]]]] + req, _, _ = cls.parse_line(line) + return cls.get_deps_from_req(req) + + @classmethod + def get_deps_from_req(cls, req, resolver=None): + # type: (Requirement, Optional["Resolver"]) -> Tuple[Set[str], Dict[str, Dict[str, Union[str, bool, List[str]]]]] + from .vendor.requirementslib.models.utils import _requirement_to_str_lowercase_name + from .vendor.requirementslib.models.requirements import Requirement + from requirementslib.utils import is_installable_dir + # TODO: this is way too complex, refactor this + constraints = set() # type: Set[str] + locked_deps = dict() # type: Dict[str, Dict[str, Union[str, bool, List[str]]]] + if (req.is_file_or_url or req.is_vcs) and not req.is_wheel: + # for local packages with setup.py files and potential direct url deps: + if req.is_vcs: + req_list, lockfile = get_vcs_deps(reqs=[req]) + req = next(iter(req for req in req_list if req is not None), req_list) + entry = lockfile[pep423_name(req.normalized_name)] + else: + _, entry = req.pipfile_entry + parsed_line = req.req.parsed_line # type: Line + setup_info = None # type: Any + try: + name = req.normalized_name + except TypeError: + raise RequirementError(req=req) + setup_info = req.req.setup_info + setup_info.get_info() + locked_deps[pep423_name(name)] = entry + requirements = [] + # Allow users to toggle resolution off for non-editable VCS packages + # but leave it on for local, installable folders on the filesystem + if environments.PIPENV_RESOLVE_VCS or ( + req.editable or parsed_line.is_wheel or ( + req.is_file_or_url and parsed_line.is_local and + is_installable_dir(parsed_line.path) + ) + ): + requirements = [v for v in getattr(setup_info, "requires", {}).values()] + for r in requirements: + if getattr(r, "url", None) and not getattr(r, "editable", False): + if r is not None: + if not r.url: + continue + line = _requirement_to_str_lowercase_name(r) + new_req, _, _ = cls.parse_line(line) + if r.marker and not r.marker.evaluate(): + new_constraints = {} + _, new_entry = req.pipfile_entry + new_lock = { + pep423_name(new_req.normalized_name): new_entry + } + else: + new_constraints, new_lock = cls.get_deps_from_req( + new_req, resolver + ) + locked_deps.update(new_lock) + constraints |= new_constraints + # if there is no marker or there is a valid marker, add the constraint line + elif r and (not r.marker or (r.marker and r.marker.evaluate())): + line = _requirement_to_str_lowercase_name(r) + constraints.add(line) + # ensure the top level entry remains as provided + # note that we shouldn't pin versions for editable vcs deps + if (not req.is_vcs or (req.is_vcs and not req.editable)): + if req.specifiers: + locked_deps[name]["version"] = req.specifiers + elif parsed_line.setup_info and parsed_line.setup_info.version: + locked_deps[name]["version"] = "=={}".format( + parsed_line.setup_info.version + ) + # if not req.is_vcs: + locked_deps.update({name: entry}) + if req.is_vcs and req.editable: + constraints.add(req.constraint_line) + if req.is_file_or_url and req.req.is_local and req.editable and ( + req.req.setup_path is not None and os.path.exists(req.req.setup_path)): + constraints.add(req.constraint_line) + else: + # if the dependency isn't installable, don't add it to constraints + # and instead add it directly to the lock + if req and req.requirement and ( + req.requirement.marker and not req.requirement.marker.evaluate() + ): + pypi = resolver.repository if resolver else None + best_match = pypi.find_best_match(req.ireq) if pypi else None + if best_match: + hashes = resolver.collect_hashes(best_match) if resolver else [] + new_req = Requirement.from_ireq(best_match) + new_req = new_req.add_hashes(hashes) + name, entry = new_req.pipfile_entry + locked_deps[pep423_name(name)] = translate_markers(entry) + return constraints, locked_deps + constraints.add(req.constraint_line) + return constraints, locked_deps + return constraints, locked_deps + + @classmethod + def create( + cls, + deps, # type: List[str] + index_lookup=None, # type: Dict[str, str] + markers_lookup=None, # type: Dict[str, str] + project=None, # type: Project + sources=None, # type: List[str] + req_dir=None, # type: str + clear=False, # type: bool + pre=False # type: bool + ): + # type: (...) -> "Resolver" + from pipenv.vendor.vistir.path import create_tracked_tempdir + if not req_dir: + req_dir = create_tracked_tempdir(suffix="-requirements", prefix="pipenv-") + if index_lookup is None: + index_lookup = {} + if markers_lookup is None: + markers_lookup = {} + if project is None: + from pipenv.core import project + project = project + if sources is None: + sources = project.sources + constraints, skipped, index_lookup, markers_lookup = cls.get_metadata( + deps, index_lookup, markers_lookup, project, sources, req_dir=req_dir, + pre=pre, clear=clear + ) + return Resolver( + constraints, req_dir, project, sources, index_lookup=index_lookup, + markers_lookup=markers_lookup, skipped=skipped, clear=clear, pre=pre + ) + + @classmethod + def from_pipfile(cls, project=None, pipfile=None, dev=False, pre=False, clear=False): + # type: (Optional[Project], Optional[Pipfile], bool, bool, bool) -> "Resolver" + from pipenv.vendor.vistir.path import create_tracked_tempdir + if not project: + from pipenv.core import project + if not pipfile: + pipfile = project._pipfile + req_dir = create_tracked_tempdir(suffix="-requirements", prefix="pipenv-") + index_lookup, markers_lookup = {}, {} + deps = set() + if dev: + deps.update(set([req.as_line() for req in pipfile.dev_packages])) + deps.update(set([req.as_line() for req in pipfile.packages])) + constraints, skipped, index_lookup, markers_lookup = cls.get_metadata( + list(deps), index_lookup, markers_lookup, project, project.sources, + req_dir=req_dir, pre=pre, clear=clear + ) + return Resolver( + constraints, req_dir, project, project.sources, index_lookup=index_lookup, + markers_lookup=markers_lookup, skipped=skipped, clear=clear, pre=pre + ) @property def pip_command(self): @@ -296,16 +681,29 @@ class Resolver(object): self._pip_command = self._get_pip_command() return self._pip_command - def prepare_pip_args(self): + def prepare_pip_args(self, use_pep517=True, build_isolation=True): pip_args = [] if self.sources: pip_args = prepare_pip_source_args(self.sources, pip_args) + if not use_pep517: + pip_args.append("--no-use-pep517") + if not build_isolation: + pip_args.append("--no-build-isolation") + pip_args.extend(["--cache-dir", environments.PIPENV_CACHE_DIR]) return pip_args @property def pip_args(self): + use_pep517 = False if ( + os.environ.get("PIP_NO_USE_PEP517", None) is not None + ) else (True if os.environ.get("PIP_USE_PEP517", None) is not None else None) + build_isolation = False if ( + os.environ.get("PIP_NO_BUILD_ISOLATION", None) is not None + ) else (True if os.environ.get("PIP_BUILD_ISOLATION", None) is not None else None) if self._pip_args is None: - self._pip_args = self.prepare_pip_args() + self._pip_args = self.prepare_pip_args( + use_pep517=use_pep517, build_isolation=build_isolation + ) return self._pip_args def prepare_constraint_file(self): @@ -317,10 +715,15 @@ class Resolver(object): dir=self.req_dir, delete=False, ) + skip_args = ("build-isolation", "use-pep517", "cache-dir") + args_to_add = [ + arg for arg in self.pip_args + if not any(bad_arg in arg for bad_arg in skip_args) + ] if self.sources: - requirementstxt_sources = " ".join(self.pip_args) if self.pip_args else "" + requirementstxt_sources = " ".join(args_to_add) if args_to_add else "" requirementstxt_sources = requirementstxt_sources.replace(" --", "\n--") - constraints_file.write(u"{0}\n".format(requirementstxt_sources)) + constraints_file.write(u"{0}\n".format(requirementstxt_sources)) constraints = self.initial_constraints constraints_file.write(u"\n".join([c for c in constraints])) constraints_file.close() @@ -338,16 +741,16 @@ class Resolver(object): pip_options, _ = self.pip_command.parser.parse_args(self.pip_args) pip_options.cache_dir = environments.PIPENV_CACHE_DIR self._pip_options = pip_options - if environments.is_verbose(): - click_echo( - crayons.blue("Using pip: {0}".format(" ".join(self.pip_args))), err=True - ) return self._pip_options @property def session(self): if self._session is None: self._session = self.pip_command._build_session(self.pip_options) + # if environments.is_verbose(): + # click_echo( + # crayons.blue("Using pip: {0}".format(" ".join(self.pip_args))), err=True + # ) return self._session @property @@ -355,7 +758,8 @@ class Resolver(object): if self._repository is None: from pipenv.patched.piptools.repositories.pypi import PyPIRepository self._repository = PyPIRepository( - pip_options=self.pip_options, use_json=False, session=self.session + pip_options=self.pip_options, use_json=False, session=self.session, + build_isolation=self.pip_options.build_isolation ) return self._repository @@ -394,59 +798,265 @@ class Resolver(object): from pipenv.patched.piptools.exceptions import NoCandidateFound from pipenv.patched.piptools.cache import CorruptCacheError from .exceptions import CacheError, ResolutionFailure - try: - results = self.resolver.resolve(max_rounds=environments.PIPENV_MAX_ROUNDS) - except CorruptCacheError as e: - if environments.PIPENV_IS_CI or self.clear: - if self._retry_attempts < 3: - self.get_resolver(clear=True, pre=self.pre) - self._retry_attempts += 1 - self.resolve() + with temp_environ(): + os.environ["PIP_NO_USE_PEP517"] = str("") + try: + results = self.resolver.resolve(max_rounds=environments.PIPENV_MAX_ROUNDS) + except CorruptCacheError as e: + if environments.PIPENV_IS_CI or self.clear: + if self._retry_attempts < 3: + self.get_resolver(clear=True, pre=self.pre) + self._retry_attempts += 1 + self.resolve() + else: + raise CacheError(e.path) + except (NoCandidateFound, DistributionNotFound, HTTPError) as e: + raise ResolutionFailure(message=str(e)) else: - raise CacheError(e.path) - except (NoCandidateFound, DistributionNotFound, HTTPError) as e: - raise ResolutionFailure(message=str(e)) + self.results = results + self.resolved_tree.update(results) + return self.resolved_tree + + @lru_cache(maxsize=1024) + def fetch_candidate(self, ireq): + candidates = self.repository.find_all_candidates(ireq.name) + matched_version = next(iter(sorted( + ireq.specifier.filter((c.version for c in candidates), True), reverse=True) + ), None) + if matched_version: + matched_candidate = next(iter( + c for c in candidates if c.version == matched_version + )) + return matched_candidate + return None + + def resolve_constraints(self): + from .vendor.requirementslib.models.markers import marker_from_specifier + new_tree = set() + for result in self.resolved_tree: + if result.markers: + self.markers[result.name] = result.markers + else: + candidate = self.fetch_candidate(result) + requires_python = getattr(candidate, "requires_python", None) + if requires_python: + marker = marker_from_specifier(candidate.requires_python) + self.markers[result.name] = marker + result.markers = marker + if result.req: + result.req.marker = marker + new_tree.add(result) + self.resolved_tree = new_tree + + @classmethod + def prepend_hash_types(cls, checksums): + cleaned_checksums = [] + for checksum in checksums: + if not checksum: + continue + if not checksum.startswith("sha256:"): + checksum = "sha256:{0}".format(checksum) + cleaned_checksums.append(checksum) + return cleaned_checksums + + def collect_hashes(self, ireq): + from .vendor.requests import ConnectionError + collected_hashes = [] + if ireq in self.hashes: + collected_hashes += list(self.hashes.get(ireq, [])) + if self._should_include_hash(ireq): + try: + hash_map = self.get_hash(ireq) + collected_hashes += list(hash_map) + except (ValueError, KeyError, IndexError, ConnectionError): + pass + elif any( + "python.org" in source["url"] or "pypi.org" in source["url"] + for source in self.sources + ): + pkg_url = "https://pypi.org/pypi/{0}/json".format(ireq.name) + session = _get_requests_session() + try: + # Grab the hashes from the new warehouse API. + r = session.get(pkg_url, timeout=10) + api_releases = r.json()["releases"] + cleaned_releases = {} + for api_version, api_info in api_releases.items(): + api_version = clean_pkg_version(api_version) + cleaned_releases[api_version] = api_info + version = "" + if ireq.specifier: + spec = next(iter(s for s in list(ireq.specifier._specs)), None) + if spec: + version = spec.version + for release in cleaned_releases[version]: + collected_hashes.append(release["digests"]["sha256"]) + collected_hashes = self.prepend_hash_types(collected_hashes) + except (ValueError, KeyError, ConnectionError): + if environments.is_verbose(): + click_echo( + "{0}: Error generating hash for {1}".format( + crayons.red("Warning", bold=True), ireq.name + ), err=True + ) + return collected_hashes + + @staticmethod + def _should_include_hash(ireq): + from pipenv.vendor.vistir.compat import Path, to_native_string + from pipenv.vendor.vistir.path import url_to_path + + # We can only hash artifacts. + try: + if not ireq.link.is_artifact: + return False + except AttributeError: + return False + + # But we don't want normal pypi artifcats since the normal resolver + # handles those + if is_pypi_url(ireq.link.url): + return False + + # We also don't want to try to hash directories as this will fail + # as these are editable deps and are not hashable. + if (ireq.link.scheme == "file" and + Path(to_native_string(url_to_path(ireq.link.url))).is_dir()): + return False + return True + + def get_hash(self, ireq, ireq_hashes=None): + """ + Retrieve hashes for a specific ``InstallRequirement`` instance. + + :param ireq: An ``InstallRequirement`` to retrieve hashes for + :type ireq: :class:`~pip_shims.InstallRequirement` + :return: A set of hashes. + :rtype: Set + """ + + # We _ALWAYS MUST PRIORITIZE_ the inclusion of hashes from local sources + # PLEASE *DO NOT MODIFY THIS* TO CHECK WHETHER AN IREQ ALREADY HAS A HASH + # RESOLVED. The resolver will pull hashes from PyPI and only from PyPI. + # The entire purpose of this approach is to include missing hashes. + # This fixes a race condition in resolution for missing dependency caches + # see pypa/pipenv#3289 + if not self._should_include_hash(ireq): + return add_to_set(set(), ireq_hashes) + elif self._should_include_hash(ireq) and ( + not ireq_hashes or ireq.link.scheme == "file" + ): + if not ireq_hashes: + ireq_hashes = set() + new_hashes = self.resolver.repository._hash_cache.get_hash(ireq.link) + ireq_hashes = add_to_set(ireq_hashes, new_hashes) else: - self.results = results - self.resolved_tree.update(results) - return self.resolved_tree + ireq_hashes = set(ireq_hashes) + # The _ONLY CASE_ where we flat out set the value is if it isn't present + # It's a set, so otherwise we *always* need to do a union update + if ireq not in self.hashes: + return ireq_hashes + else: + return self.hashes[ireq] | ireq_hashes def resolve_hashes(self): - def _should_include_hash(ireq): - from pipenv.vendor.vistir.compat import Path, to_native_string - from pipenv.vendor.vistir.path import url_to_path - - # We can only hash artifacts. - try: - if not ireq.link.is_artifact: - return False - except AttributeError: - return False - - # But we don't want normal pypi artifcats since the normal resolver - # handles those - if is_pypi_url(ireq.link.url): - return False - - # We also don't want to try to hash directories as this will fail - # as these are editable deps and are not hashable. - if (ireq.link.scheme == "file" and - Path(to_native_string(url_to_path(ireq.link.url))).is_dir()): - return False - return True - if self.results is not None: resolved_hashes = self.resolver.resolve_hashes(self.results) for ireq, ireq_hashes in resolved_hashes.items(): - if ireq not in self.hashes: - if _should_include_hash(ireq): - self.hashes[ireq] = [ - self.resolver.repository._hash_cache.get_hash(ireq.link) - ] - else: - self.hashes[ireq] = ireq_hashes + self.hashes[ireq] = self.get_hash(ireq, ireq_hashes=ireq_hashes) return self.hashes + def _clean_skipped_result(self, req, value): + ref = None + if req.is_vcs: + ref = req.commit_hash + ireq = req.as_ireq() + entry = value.copy() + entry["name"] = req.name + if entry.get("editable", False) and entry.get("version"): + del entry["version"] + ref = ref if ref is not None else entry.get("ref") + if ref: + entry["ref"] = ref + if self._should_include_hash(ireq): + collected_hashes = self.collect_hashes(ireq) + if collected_hashes: + entry["hashes"] = sorted(set(collected_hashes)) + return req.name, entry + + def clean_results(self): + from pipenv.vendor.requirementslib.models.requirements import Requirement + reqs = [(Requirement.from_ireq(ireq), ireq) for ireq in self.resolved_tree] + results = {} + for req, ireq in reqs: + if (req.vcs and req.editable and not req.is_direct_url): + continue + collected_hashes = self.collect_hashes(ireq) + req = req.add_hashes(collected_hashes) + if not collected_hashes and self._should_include_hash(ireq): + discovered_hashes = self.hashes.get(ireq, set()) | self.get_hash(ireq) + if discovered_hashes: + req = req.add_hashes(discovered_hashes) + self.hashes[ireq] = collected_hashes = discovered_hashes + if collected_hashes: + collected_hashes = sorted(set(collected_hashes)) + name, entry = format_requirement_for_lockfile( + req, self.markers_lookup, self.index_lookup, collected_hashes + ) + entry = translate_markers(entry) + if name in results: + results[name].update(entry) + else: + results[name] = entry + for k in list(self.skipped.keys()): + req = Requirement.from_pipfile(k, self.skipped[k]) + name, entry = self._clean_skipped_result(req, self.skipped[k]) + entry = translate_markers(entry) + if name in results: + results[name].update(entry) + else: + results[name] = entry + results = list(results.values()) + return results + + +def format_requirement_for_lockfile(req, markers_lookup, index_lookup, hashes=None): + if req.specifiers: + version = str(req.get_version()) + else: + version = None + index = index_lookup.get(req.normalized_name) + markers = markers_lookup.get(req.normalized_name) + req.index = index + name, pf_entry = req.pipfile_entry + name = pep423_name(req.name) + entry = {} + if isinstance(pf_entry, six.string_types): + entry["version"] = pf_entry.lstrip("=") + else: + entry.update(pf_entry) + if version is not None: + entry["version"] = version + if req.line_instance.is_direct_url: + entry["file"] = req.req.uri + if hashes: + entry["hashes"] = sorted(set(hashes)) + entry["name"] = name + if index: # and index != next(iter(project.sources), {}).get("name"): + entry.update({"index": index}) + if markers: + entry.update({"markers": markers}) + entry = translate_markers(entry) + if req.vcs or req.editable and entry.get("index"): + del entry["index"] + return name, entry + + +def _show_warning(message, category, filename, lineno, line): + warnings.showwarning(message=message, category=category, filename=filename, + lineno=lineno, file=sys.stderr, line=line) + sys.stderr.flush() + def actually_resolve_deps( deps, @@ -459,98 +1069,142 @@ def actually_resolve_deps( req_dir=None, ): from pipenv.vendor.vistir.path import create_tracked_tempdir + from pipenv.vendor.requirementslib.models.requirements import Requirement if not req_dir: req_dir = create_tracked_tempdir(suffix="-requirements", prefix="pipenv-") - constraints = get_resolver_metadata( - deps, index_lookup, markers_lookup, project, sources, - ) - resolver = Resolver(constraints, req_dir, project, sources, clear=clear, pre=pre) - resolved_tree = resolver.resolve() - hashes = resolver.resolve_hashes() + warning_list = [] - return (resolved_tree, hashes, markers_lookup, resolver) + with warnings.catch_warnings(record=True) as warning_list: + resolver = Resolver.create( + deps, index_lookup, markers_lookup, project, sources, req_dir, clear, pre + ) + resolver.resolve() + hashes = resolver.resolve_hashes() + resolver.resolve_constraints() + results = resolver.clean_results() + for warning in warning_list: + _show_warning(warning.message, warning.category, warning.filename, warning.lineno, + warning.line) + return (results, hashes, resolver.markers_lookup, resolver, resolver.skipped) @contextlib.contextmanager def create_spinner(text, nospin=None, spinner_name=None): - import vistir.spin + from .vendor.vistir import spin + from .vendor.vistir.misc import fs_str if not spinner_name: spinner_name = environments.PIPENV_SPINNER if nospin is None: nospin = environments.PIPENV_NOSPIN - with vistir.spin.create_spinner( - spinner_name=spinner_name, - start_text=vistir.compat.fs_str(text), - nospin=nospin, write_to_stdout=False + with spin.create_spinner( + spinner_name=spinner_name, + start_text=fs_str(text), + nospin=nospin, write_to_stdout=False ) as sp: yield sp def resolve(cmd, sp): - from .vendor import delegator + import delegator from .cmdparse import Script from .vendor.pexpect.exceptions import EOF, TIMEOUT from .vendor.vistir.compat import to_native_string + from .vendor.vistir.misc import echo EOF.__module__ = "pexpect.exceptions" from ._compat import decode_output c = delegator.run(Script.parse(cmd).cmdify(), block=False, env=os.environ.copy()) + if environments.is_verbose(): + c.subprocess.logfile = sys.stderr _out = decode_output("") result = None out = to_native_string("") while True: + result = None try: - result = c.expect(u"\n", timeout=environments.PIPENV_TIMEOUT) - except (EOF, TIMEOUT): + result = c.expect(u"\n", timeout=environments.PIPENV_INSTALL_TIMEOUT) + except TIMEOUT: pass - if result is None: + except EOF: break - _out = c.subprocess.before - if _out is not None: + except KeyboardInterrupt: + c.kill() + break + if result: + _out = c.subprocess.before _out = decode_output("{0}".format(_out)) out += _out - sp.text = to_native_string("{0}".format(_out[:100])) - if environments.is_verbose(): - if _out is not None: - sp._hide_cursor() - sp.write(_out.rstrip()) - sp._show_cursor() + # sp.text = to_native_string("{0}".format(_out[:100])) + if environments.is_verbose(): + sp.hide_and_write(out.splitlines()[-1].rstrip()) + else: + break c.block() if c.return_code != 0: sp.red.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format( "Locking Failed!" )) - click_echo(c.out.strip(), err=True) - click_echo(c.err.strip(), err=True) + echo(c.out.strip(), err=True) + if not environments.is_verbose(): + echo(out, err=True) sys.exit(c.return_code) + if environments.is_verbose(): + echo(c.err.strip(), err=True) return c -def get_locked_dep(dep, pipfile_section): +def get_locked_dep(dep, pipfile_section, prefer_pipfile=True): + # the prefer pipfile flag is not used yet, but we are introducing + # it now for development purposes + # TODO: Is this implementation clear? How can it be improved? entry = None cleaner_kwargs = { "is_top_level": False, "pipfile_entry": None } if isinstance(dep, Mapping) and dep.get("name", ""): - name_options = [dep["name"], pep423_name(dep["name"])] - name = next(iter(k for k in name_options if k in pipfile_section), None) + dep_name = pep423_name(dep["name"]) + name = next(iter( + k for k in pipfile_section.keys() + if pep423_name(k) == dep_name + ), None) entry = pipfile_section[name] if name else None if entry: cleaner_kwargs.update({"is_top_level": True, "pipfile_entry": entry}) lockfile_entry = clean_resolved_dep(dep, **cleaner_kwargs) + if entry and isinstance(entry, Mapping): + version = entry.get("version", "") if entry else "" + else: + version = entry if entry else "" + lockfile_name, lockfile_dict = lockfile_entry.copy().popitem() + lockfile_version = lockfile_dict.get("version", "") + # Keep pins from the lockfile + if prefer_pipfile and lockfile_version != version and version.startswith("=="): + lockfile_dict["version"] = version + lockfile_entry[lockfile_name] = lockfile_dict return lockfile_entry def prepare_lockfile(results, pipfile, lockfile): - from .vendor.requirementslib.utils import is_vcs + # from .vendor.requirementslib.utils import is_vcs for dep in results: + if not dep: + continue # Merge in any relevant information from the pipfile entry, including # markers, normalized names, URL info, etc that we may have dropped during lock - if not is_vcs(dep): - lockfile_entry = get_locked_dep(dep, pipfile) - lockfile.update(lockfile_entry) + # if not is_vcs(dep): + lockfile_entry = get_locked_dep(dep, pipfile) + name = next(iter(k for k in lockfile_entry.keys())) + current_entry = lockfile.get(name) + if current_entry: + if not isinstance(current_entry, Mapping): + lockfile[name] = lockfile_entry[name] + else: + lockfile[name].update(lockfile_entry[name]) + lockfile[name] = translate_markers(lockfile[name]) + else: + lockfile[name] = lockfile_entry[name] return lockfile @@ -564,40 +1218,56 @@ def venv_resolve_deps( pypi_mirror=None, dev=False, pipfile=None, - lockfile=None + lockfile=None, + keep_outdated=False ): + """ + Resolve dependencies for a pipenv project, acts as a portal to the target environment. + + Regardless of whether a virtual environment is present or not, this will spawn + a subproces which is isolated to the target environment and which will perform + dependency resolution. This function reads the output of that call and mutates + the provided lockfile accordingly, returning nothing. + + :param List[:class:`~requirementslib.Requirement`] deps: A list of dependencies to resolve. + :param Callable which: [description] + :param project: The pipenv Project instance to use during resolution + :param Optional[bool] pre: Whether to resolve pre-release candidates, defaults to False + :param Optional[bool] clear: Whether to clear the cache during resolution, defaults to False + :param Optional[bool] allow_global: Whether to use *sys.executable* as the python binary, defaults to False + :param Optional[str] pypi_mirror: A URL to substitute any time *pypi.org* is encountered, defaults to None + :param Optional[bool] dev: Whether to target *dev-packages* or not, defaults to False + :param pipfile: A Pipfile section to operate on, defaults to None + :type pipfile: Optional[Dict[str, Union[str, Dict[str, bool, List[str]]]]] + :param Dict[str, Any] lockfile: A project lockfile to mutate, defaults to None + :param bool keep_outdated: Whether to retain outdated dependencies and resolve with them in mind, defaults to False + :raises RuntimeError: Raised on resolution failure + :return: Nothing + :rtype: None + """ + from .vendor.vistir.misc import fs_str - from .vendor.vistir.compat import Path, to_native_string, JSONDecodeError + from .vendor.vistir.compat import Path, JSONDecodeError, NamedTemporaryFile from .vendor.vistir.path import create_tracked_tempdir from . import resolver + from ._compat import decode_for_output import json - vcs_deps = [] - vcs_lockfile = {} results = [] - pipfile_section = "dev_packages" if dev else "packages" + pipfile_section = "dev-packages" if dev else "packages" lockfile_section = "develop" if dev else "default" - vcs_section = "vcs_{0}".format(pipfile_section) - vcs_deps = getattr(project, vcs_section, []) - if not deps and not vcs_deps: - return {} + if not deps: + if not project.pipfile_exists: + return None + deps = project.parsed_pipfile.get(pipfile_section, {}) + if not deps: + return None if not pipfile: - pipfile = getattr(project, pipfile_section, None) + pipfile = getattr(project, pipfile_section, {}) if not lockfile: lockfile = project._lockfile req_dir = create_tracked_tempdir(prefix="pipenv", suffix="requirements") - if vcs_deps: - with create_spinner(text=fs_str("Pinning VCS Packages...")) as sp: - vcs_reqs, vcs_lockfile = get_vcs_deps( - project, - which=which, - clear=clear, - pre=pre, - allow_global=allow_global, - dev=dev, - ) - vcs_deps = [req.as_line() for req in vcs_reqs if req.editable] cmd = [ which("python", allow_global=allow_global), Path(resolver.__file__.rstrip("co")).as_posix() @@ -608,47 +1278,55 @@ def venv_resolve_deps( cmd.append("--clear") if allow_global: cmd.append("--system") + if dev: + cmd.append("--dev") + target_file = NamedTemporaryFile(prefix="resolver", suffix=".json", delete=False) + target_file.close() + cmd.extend(["--write", make_posix(target_file.name)]) with temp_environ(): - os.environ = {fs_str(k): fs_str(val) for k, val in os.environ.items()} - os.environ["PIPENV_PACKAGES"] = str("\n".join(deps)) + os.environ.update({fs_str(k): fs_str(val) for k, val in os.environ.items()}) if pypi_mirror: os.environ["PIPENV_PYPI_MIRROR"] = str(pypi_mirror) os.environ["PIPENV_VERBOSITY"] = str(environments.PIPENV_VERBOSITY) os.environ["PIPENV_REQ_DIR"] = fs_str(req_dir) os.environ["PIP_NO_INPUT"] = fs_str("1") - with create_spinner(text=fs_str("Locking...")) as sp: + os.environ["PIPENV_SITE_DIR"] = get_pipenv_sitedir() + if keep_outdated: + os.environ["PIPENV_KEEP_OUTDATED"] = fs_str("1") + with create_spinner(text=decode_for_output("Locking...")) as sp: + # This conversion is somewhat slow on local and file-type requirements since + # we now download those requirements / make temporary folders to perform + # dependency resolution on them, so we are including this step inside the + # spinner context manager for the UX improvement + sp.write(decode_for_output("Building requirements...")) + deps = convert_deps_to_pip( + deps, project, r=False, include_index=True + ) + constraints = set(deps) + os.environ["PIPENV_PACKAGES"] = str("\n".join(constraints)) + sp.write(decode_for_output("Resolving dependencies...")) c = resolve(cmd, sp) - results = c.out - if vcs_deps: - with temp_environ(): - os.environ["PIPENV_PACKAGES"] = str("\n".join(vcs_deps)) - sp.text = to_native_string("Locking VCS Dependencies...") - vcs_c = resolve(cmd, sp) - vcs_results, vcs_err = vcs_c.out, vcs_c.err + results = c.out.strip() + if c.ok: + sp.green.ok(environments.PIPENV_SPINNER_OK_TEXT.format("Success!")) else: - vcs_results, vcs_err = "", "" - sp.green.ok(environments.PIPENV_SPINNER_OK_TEXT.format("Success!")) - outputs = [results, vcs_results] - if environments.is_verbose(): - for output in outputs: - click_echo(output.split("RESULTS:")[0], err=True) + sp.red.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format("Locking Failed!")) + click_echo("Output: {0}".format(c.out.strip()), err=True) + click_echo("Error: {0}".format(c.err.strip()), err=True) try: - results = json.loads(results.split("RESULTS:")[1].strip()) - if vcs_results: - # For vcs dependencies, treat the initial pass at locking (i.e. checkout) - # as the pipfile entry because it gets us an actual ref to use - vcs_results = json.loads(vcs_results.split("RESULTS:")[1].strip()) - vcs_lockfile = prepare_lockfile(vcs_results, vcs_lockfile.copy(), vcs_lockfile) - else: - vcs_results = [] - + with open(target_file.name, "r") as fh: + results = json.load(fh) except (IndexError, JSONDecodeError): - for out, err in [(c.out, c.err), (vcs_results, vcs_err)]: - click_echo(out.strip(), err=True) - click_echo(err.strip(), err=True) + click_echo(c.out.strip(), err=True) + click_echo(c.err.strip(), err=True) + if os.path.exists(target_file.name): + os.unlink(target_file.name) raise RuntimeError("There was a problem with locking.") - lockfile[lockfile_section] = prepare_lockfile(results, pipfile, lockfile[lockfile_section]) - lockfile[lockfile_section].update(vcs_lockfile) + if os.path.exists(target_file.name): + os.unlink(target_file.name) + if lockfile_section not in lockfile: + lockfile[lockfile_section] = {} + prepare_lockfile(results, pipfile, lockfile[lockfile_section]) def resolve_deps( @@ -665,9 +1343,6 @@ def resolve_deps( """Given a list of dependencies, return a resolved list of dependencies, using pip-tools -- and their hashes, using the warehouse API / pip. """ - from .vendor.requests.exceptions import ConnectionError - from .vendor.requirementslib.models.requirements import Requirement - index_lookup = {} markers_lookup = {} python_path = which("python", allow_global=allow_global) @@ -675,8 +1350,9 @@ def resolve_deps( os.environ["PIP_SRC"] = project.virtualenv_src_location backup_python_path = sys.executable results = [] + resolver = None if not deps: - return results + return results, resolver # First (proper) attempt: req_dir = req_dir if req_dir else os.environ.get("req_dir", None) if not req_dir: @@ -684,7 +1360,7 @@ def resolve_deps( req_dir = create_tracked_tempdir(prefix="pipenv-", suffix="-requirements") with HackedPythonVersion(python_version=python, python_path=python_path): try: - resolved_tree, hashes, markers_lookup, resolver = actually_resolve_deps( + results, hashes, markers_lookup, resolver, skipped = actually_resolve_deps( deps, index_lookup, markers_lookup, @@ -696,9 +1372,9 @@ def resolve_deps( ) except RuntimeError: # Don't exit here, like usual. - resolved_tree = None + results = None # Second (last-resort) attempt: - if resolved_tree is None: + if results is None: with HackedPythonVersion( python_version=".".join([str(s) for s in sys.version_info[:3]]), python_path=backup_python_path, @@ -706,7 +1382,7 @@ def resolve_deps( try: # Attempt to resolve again, with different Python version information, # particularly for particularly particular packages. - resolved_tree, hashes, markers_lookup, resolver = actually_resolve_deps( + results, hashes, markers_lookup, resolver, skipped = actually_resolve_deps( deps, index_lookup, markers_lookup, @@ -718,64 +1394,7 @@ def resolve_deps( ) except RuntimeError: sys.exit(1) - for result in resolved_tree: - if not result.editable: - req = Requirement.from_ireq(result) - name = pep423_name(req.name) - version = str(req.get_version()) - index = index_lookup.get(result.name) - req.index = index - collected_hashes = [] - if result in hashes: - collected_hashes = list(hashes.get(result)) - elif any( - "python.org" in source["url"] or "pypi.org" in source["url"] - for source in sources - ): - pkg_url = "https://pypi.org/pypi/{0}/json".format(name) - session = _get_requests_session() - try: - # Grab the hashes from the new warehouse API. - r = session.get(pkg_url, timeout=10) - api_releases = r.json()["releases"] - cleaned_releases = {} - for api_version, api_info in api_releases.items(): - api_version = clean_pkg_version(api_version) - cleaned_releases[api_version] = api_info - for release in cleaned_releases[version]: - collected_hashes.append(release["digests"]["sha256"]) - collected_hashes = ["sha256:" + s for s in collected_hashes] - except (ValueError, KeyError, ConnectionError): - if environments.is_verbose(): - click_echo( - "{0}: Error generating hash for {1}".format( - crayons.red("Warning", bold=True), name - ), err=True - ) - # # Collect un-collectable hashes (should work with devpi). - # try: - # collected_hashes = collected_hashes + list( - # list(resolver.resolve_hashes([result]).items())[0][1] - # ) - # except (ValueError, KeyError, ConnectionError, IndexError): - # if verbose: - # print('Error generating hash for {}'.format(name)) - req.hashes = sorted(set(collected_hashes)) - name, _entry = req.pipfile_entry - entry = {} - if isinstance(_entry, six.string_types): - entry["version"] = _entry.lstrip("=") - else: - entry.update(_entry) - entry["version"] = version - entry["name"] = name - # if index: - # d.update({"index": index}) - if markers_lookup.get(result.name): - entry.update({"markers": markers_lookup.get(result.name)}) - entry = translate_markers(entry) - results.append(entry) - return results + return results, resolver def is_star(val): @@ -794,7 +1413,9 @@ def convert_deps_to_pip(deps, project=None, r=True, include_index=True): dependencies = [] for dep_name, dep in deps.items(): - indexes = project.sources if hasattr(project, "sources") else [] + if project: + project.clear_pipfile_cache() + indexes = getattr(project, "pipfile_sources", []) if project is not None else [] new_dep = Requirement.from_pipfile(dep_name, dep) if new_dep.index: include_index = True @@ -843,7 +1464,6 @@ def mkdir_p(newdir): if exn.errno != errno.EEXIST: raise - def is_required_version(version, specified_version): """Check to see if there's a hard requirement for version @@ -953,53 +1573,6 @@ def proper_case(package_name): return good_name -def split_section(input_file, section_suffix, test_function): - """ - Split a pipfile or a lockfile section out by section name and test function - - :param dict input_file: A dictionary containing either a pipfile or lockfile - :param str section_suffix: A string of the name of the section - :param func test_function: A test function to test against the value in the key/value pair - - >>> split_section(my_lockfile, 'vcs', is_vcs) - { - 'default': { - "six": { - "hashes": [ - "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb", - "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9" - ], - "version": "==1.11.0" - } - }, - 'default-vcs': { - "e1839a8": { - "editable": true, - "path": "." - } - } - } - """ - pipfile_sections = ("packages", "dev-packages") - lockfile_sections = ("default", "develop") - if any(section in input_file for section in pipfile_sections): - sections = pipfile_sections - elif any(section in input_file for section in lockfile_sections): - sections = lockfile_sections - else: - # return the original file if we can't find any pipfile or lockfile sections - return input_file - - for section in sections: - split_dict = {} - entries = input_file.get(section, {}) - for k in list(entries.keys()): - if test_function(entries.get(k)): - split_dict[k] = entries.pop(k) - input_file["-".join([section, section_suffix])] = split_dict - return input_file - - def get_windows_path(*args): """Sanitize a path for windows environments @@ -1051,7 +1624,7 @@ def get_canonical_names(packages): if not isinstance(packages, Sequence): if not isinstance(packages, six.string_types): return packages - packages = [packages,] + packages = [packages] return set([canonicalize_name(pkg) for pkg in packages if pkg]) @@ -1084,15 +1657,14 @@ def walk_up(bottom): def find_requirements(max_depth=3): - """Returns the path of a Pipfile in parent directories.""" + """Returns the path of a requirements.txt file in parent directories.""" i = 0 for c, d, f in walk_up(os.getcwd()): i += 1 if i < max_depth: - if "requirements.txt": - r = os.path.join(c, "requirements.txt") - if os.path.isfile(r): - return r + r = os.path.join(c, "requirements.txt") + if os.path.isfile(r): + return r raise RuntimeError("No requirements.txt found!") @@ -1237,7 +1809,7 @@ def handle_remove_readonly(func, path, exc): warnings.warn(default_warning_message.format(path), ResourceWarning) return - raise + raise exc def escape_cmd(cmd): @@ -1255,37 +1827,53 @@ def safe_expandvars(value): def get_vcs_deps( - project, - which=None, - clear=False, - pre=False, - allow_global=False, + project=None, dev=False, pypi_mirror=None, + packages=None, + reqs=None ): from .vendor.requirementslib.models.requirements import Requirement section = "vcs_dev_packages" if dev else "vcs_packages" - reqs = [] + if reqs is None: + reqs = [] lockfile = {} - try: - packages = getattr(project, section) - except AttributeError: - return [], [] - for pkg_name, pkg_pipfile in packages.items(): - requirement = Requirement.from_pipfile(pkg_name, pkg_pipfile) + if not reqs: + if not project and not packages: + raise ValueError( + "Must supply either a project or a pipfile section to lock vcs dependencies." + ) + if not packages: + try: + packages = getattr(project, section) + except AttributeError: + return [], [] + reqs = [Requirement.from_pipfile(name, entry) for name, entry in packages.items()] + result = [] + for requirement in reqs: name = requirement.normalized_name commit_hash = None if requirement.is_vcs: try: - with locked_repository(requirement) as repo: + with temp_path(), locked_repository(requirement) as repo: + from pipenv.vendor.requirementslib.models.requirements import Requirement + # from distutils.sysconfig import get_python_lib + # sys.path = [repo.checkout_directory, "", ".", get_python_lib(plat_specific=0)] commit_hash = repo.get_commit_hash() + name = requirement.normalized_name + version = requirement._specifiers = "=={0}".format(requirement.req.setup_info.version) lockfile[name] = requirement.pipfile_entry[1] lockfile[name]['ref'] = commit_hash - reqs.append(requirement) + result.append(requirement) + version = requirement.specifiers + if not version and requirement.specifiers: + version = requirement.specifiers + if version: + lockfile[name]['version'] = version except OSError: continue - return reqs, lockfile + return result, lockfile def translate_markers(pipfile_entry): @@ -1301,23 +1889,24 @@ def translate_markers(pipfile_entry): """ if not isinstance(pipfile_entry, Mapping): raise TypeError("Entry is not a pipfile formatted mapping.") - from .vendor.distlib.markers import DEFAULT_CONTEXT as marker_context - from .vendor.packaging.markers import Marker + from .vendor.packaging.markers import Marker, default_environment from .vendor.vistir.misc import dedup - allowed_marker_keys = ["markers"] + [k for k in marker_context.keys()] + allowed_marker_keys = ["markers"] + list(default_environment().keys()) provided_keys = list(pipfile_entry.keys()) if hasattr(pipfile_entry, "keys") else [] - pipfile_markers = [k for k in provided_keys if k in allowed_marker_keys] + pipfile_markers = set(provided_keys) & set(allowed_marker_keys) new_pipfile = dict(pipfile_entry).copy() marker_set = set() if "markers" in new_pipfile: - marker = str(Marker(new_pipfile.pop("markers"))) - if 'extra' not in marker: - marker_set.add(marker) + marker_str = new_pipfile.pop("markers") + if marker_str: + marker = str(Marker(marker_str)) + if 'extra' not in marker: + marker_set.add(marker) for m in pipfile_markers: entry = "{0}".format(pipfile_entry[m]) if m != "markers": - marker_set.add(str(Marker("{0}{1}".format(m, entry)))) + marker_set.add(str(Marker("{0} {1}".format(m, entry)))) new_pipfile.pop(m) if marker_set: new_pipfile["markers"] = str(Marker(" or ".join( @@ -1328,35 +1917,50 @@ def translate_markers(pipfile_entry): def clean_resolved_dep(dep, is_top_level=False, pipfile_entry=None): + from .vendor.requirementslib.utils import is_vcs name = pep423_name(dep["name"]) + lockfile = {} # We use this to determine if there are any markers on top level packages # So we can make sure those win out during resolution if the packages reoccur - lockfile = {"version": "=={0}".format(dep["version"])} - for key in ["hashes", "index", "extras"]: + if "version" in dep and dep["version"] and not dep.get("editable", False): + version = "{0}".format(dep["version"]) + if not version.startswith("=="): + version = "=={0}".format(version) + lockfile["version"] = version + if is_vcs(dep): + ref = dep.get("ref", None) + if ref is not None: + lockfile["ref"] = ref + vcs_type = next(iter(k for k in dep.keys() if k in VCS_LIST), None) + if vcs_type: + lockfile[vcs_type] = dep[vcs_type] + if "subdirectory" in dep: + lockfile["subdirectory"] = dep["subdirectory"] + for key in ["hashes", "index", "extras", "editable"]: if key in dep: lockfile[key] = dep[key] # In case we lock a uri or a file when the user supplied a path # remove the uri or file keys from the entry and keep the path - if pipfile_entry and any(k in pipfile_entry for k in ["file", "path"]): - fs_key = next((k for k in ["path", "file"] if k in pipfile_entry), None) - lockfile_key = next((k for k in ["uri", "file", "path"] if k in lockfile), None) - if fs_key != lockfile_key: - try: - del lockfile[lockfile_key] - except KeyError: - # pass when there is no lock file, usually because it's the first time - pass - lockfile[fs_key] = pipfile_entry[fs_key] + fs_key = next(iter(k for k in ["path", "file"] if k in dep), None) + pipfile_fs_key = None + if pipfile_entry: + pipfile_fs_key = next(iter(k for k in ["path", "file"] if k in pipfile_entry), None) + if fs_key and pipfile_fs_key and fs_key != pipfile_fs_key: + lockfile[pipfile_fs_key] = pipfile_entry[pipfile_fs_key] + elif fs_key is not None: + lockfile[fs_key] = dep[fs_key] # If a package is **PRESENT** in the pipfile but has no markers, make sure we # **NEVER** include markers in the lockfile - if "markers" in dep: + if "markers" in dep and dep.get("markers", "").strip(): # First, handle the case where there is no top level dependency in the pipfile if not is_top_level: - try: - lockfile["markers"] = translate_markers(dep)["markers"] - except TypeError: - pass + translated = translate_markers(dep).get("markers", "").strip() + if translated: + try: + lockfile["markers"] = translated + except TypeError: + pass # otherwise make sure we are prioritizing whatever the pipfile says about the markers # If the pipfile says nothing, then we should put nothing in the lockfile else: @@ -1451,11 +2055,11 @@ def parse_indexes(line): ) parser.add_argument( "--extra-index-url", "--extra-index", - metavar="extra_indexes",action="append", + metavar="extra_indexes", action="append", ) parser.add_argument("--trusted-host", metavar="trusted_hosts", action="append") args, remainder = parser.parse_known_args(line.split()) - index = [] if not args.index else [args.index,] + index = [] if not args.index else [args.index] extra_indexes = [] if not args.extra_index_url else args.extra_index_url indexes = index + extra_indexes trusted_hosts = args.trusted_host if args.trusted_host else [] @@ -1474,3 +2078,215 @@ def sys_version(version_tuple): sys.version_info = version_tuple yield sys.version_info = old_version + + +def add_to_set(original_set, element): + """Given a set and some arbitrary element, add the element(s) to the set""" + if not element: + return original_set + if isinstance(element, Set): + original_set |= element + elif isinstance(element, (list, tuple)): + original_set |= set(element) + else: + original_set.add(element) + return original_set + + +def is_url_equal(url, other_url): + # type: (str, str) -> bool + """ + Compare two urls by scheme, host, and path, ignoring auth + + :param str url: The initial URL to compare + :param str url: Second url to compare to the first + :return: Whether the URLs are equal without **auth**, **query**, and **fragment** + :rtype: bool + + >>> is_url_equal("https://user:pass@mydomain.com/some/path?some_query", + "https://user2:pass2@mydomain.com/some/path") + True + + >>> is_url_equal("https://user:pass@mydomain.com/some/path?some_query", + "https://mydomain.com/some?some_query") + False + """ + if not isinstance(url, six.string_types): + raise TypeError("Expected string for url, received {0!r}".format(url)) + if not isinstance(other_url, six.string_types): + raise TypeError("Expected string for url, received {0!r}".format(other_url)) + parsed_url = urllib3_util.parse_url(url) + parsed_other_url = urllib3_util.parse_url(other_url) + unparsed = parsed_url._replace(auth=None, query=None, fragment=None).url + unparsed_other = parsed_other_url._replace(auth=None, query=None, fragment=None).url + return unparsed == unparsed_other + + +@lru_cache() +def make_posix(path): + # type: (str) -> str + """ + Convert a path with possible windows-style separators to a posix-style path + (with **/** separators instead of **\\** separators). + + :param Text path: A path to convert. + :return: A converted posix-style path + :rtype: Text + + >>> make_posix("c:/users/user/venvs/some_venv\\Lib\\site-packages") + "c:/users/user/venvs/some_venv/Lib/site-packages" + + >>> make_posix("c:\\users\\user\\venvs\\some_venv") + "c:/users/user/venvs/some_venv" + """ + if not isinstance(path, six.string_types): + raise TypeError("Expected a string for path, received {0!r}...".format(path)) + starts_with_sep = path.startswith(os.path.sep) + separated = normalize_path(path).split(os.path.sep) + if isinstance(separated, (list, tuple)): + path = posixpath.join(*separated) + if starts_with_sep: + path = "/{0}".format(path) + return path + + +def get_pipenv_dist(pkg="pipenv", pipenv_site=None): + from .resolver import find_site_path + pipenv_libdir = os.path.dirname(os.path.abspath(__file__)) + if pipenv_site is None: + pipenv_site = os.path.dirname(pipenv_libdir) + pipenv_dist, _ = find_site_path(pkg, site_dir=pipenv_site) + return pipenv_dist + + +def find_python(finder, line=None): + """ + Given a `pythonfinder.Finder` instance and an optional line, find a corresponding python + + :param finder: A :class:`pythonfinder.Finder` instance to use for searching + :type finder: :class:pythonfinder.Finder` + :param str line: A version, path, name, or nothing, defaults to None + :return: A path to python + :rtype: str + """ + + if line and not isinstance(line, six.string_types): + raise TypeError( + "Invalid python search type: expected string, received {0!r}".format(line) + ) + if line and os.path.isabs(line): + if os.name == "nt": + line = make_posix(line) + return line + if not finder: + from pipenv.vendor.pythonfinder import Finder + finder = Finder(global_search=True) + if not line: + result = next(iter(finder.find_all_python_versions()), None) + elif line and line[0].isdigit() or re.match(r'[\d\.]+', line): + result = finder.find_python_version(line) + else: + result = finder.find_python_version(name=line) + if not result: + result = finder.which(line) + if not result and not line.startswith("python"): + line = "python{0}".format(line) + result = find_python(finder, line) + if not result: + result = next(iter(finder.find_all_python_versions()), None) + if result: + if not isinstance(result, six.string_types): + return result.path.as_posix() + return result + return + + +def is_python_command(line): + """ + Given an input, checks whether the input is a request for python or notself. + + This can be a version, a python runtime name, or a generic 'python' or 'pythonX.Y' + + :param str line: A potential request to find python + :returns: Whether the line is a python lookup + :rtype: bool + """ + + if not isinstance(line, six.string_types): + raise TypeError("Not a valid command to check: {0!r}".format(line)) + + from pipenv.vendor.pythonfinder.utils import PYTHON_IMPLEMENTATIONS + is_version = re.match(r'[\d\.]+', line) + if (line.startswith("python") or is_version or + any(line.startswith(v) for v in PYTHON_IMPLEMENTATIONS)): + return True + # we are less sure about this but we can guess + if line.startswith("py"): + return True + return False + + +# def make_marker_from_specifier(spec): +# # type: (str) -> Optional[Marker] +# """Given a python version specifier, create a marker + +# :param spec: A specifier +# :type spec: str +# :return: A new marker +# :rtype: Optional[:class:`packaging.marker.Marker`] +# """ +# from .vendor.packaging.markers import Marker +# from .vendor.packaging.specifiers import SpecifierSet, Specifier +# from .vendor.requirementslib.models.markers import cleanup_pyspecs, format_pyversion +# if not any(spec.startswith(k) for k in Specifier._operators.keys()): +# if spec.strip().lower() in ["any", "", "*"]: +# return None +# spec = "=={0}".format(spec) +# elif spec.startswith("==") and spec.count("=") > 3: +# spec = "=={0}".format(spec.lstrip("=")) +# if not spec: +# return None +# marker_segments = [] +# print(spec) +# for marker_segment in cleanup_pyspecs(spec): +# print(marker_segment) +# marker_segments.append(format_pyversion(marker_segment)) +# marker_str = " and ".join(marker_segments) +# return Marker(marker_str) + + +@contextlib.contextmanager +def interrupt_handled_subprocess( + cmd, verbose=False, return_object=True, write_to_stdout=False, combine_stderr=True, + block=True, nospin=True, env=None +): + """Given a :class:`subprocess.Popen` instance, wrap it in exception handlers. + + Terminates the subprocess when and if a `SystemExit` or `KeyboardInterrupt` are + processed. + + Arguments: + :param str cmd: A command to run + :param bool verbose: Whether to run with verbose mode enabled, default False + :param bool return_object: Whether to return a subprocess instance or a 2-tuple, default True + :param bool write_to_stdout: Whether to write directly to stdout, default False + :param bool combine_stderr: Whether to combine stdout and stderr, default True + :param bool block: Whether the subprocess should be a blocking subprocess, default True + :param bool nospin: Whether to suppress the spinner with the subprocess, default True + :param Optional[Dict[str, str]] env: A dictionary to merge into the subprocess environment + :return: A subprocess, wrapped in exception handlers, as a context manager + :rtype: :class:`subprocess.Popen` obj: An instance of a running subprocess + """ + obj = run( + cmd, verbose=verbose, return_object=True, write_to_stdout=False, + combine_stderr=False, block=True, nospin=True, env=env, + ) + try: + yield obj + except (SystemExit, KeyboardInterrupt): + if os.name == "nt": + os.kill(obj.pid, signal.CTRL_BREAK_EVENT) + else: + os.kill(obj.pid, signal.SIGINT) + obj.wait() + raise diff --git a/pipenv/vendor/attr/__init__.py b/pipenv/vendor/attr/__init__.py index debfd57b..0ebe5197 100644 --- a/pipenv/vendor/attr/__init__.py +++ b/pipenv/vendor/attr/__init__.py @@ -18,7 +18,7 @@ from ._make import ( ) -__version__ = "18.2.0" +__version__ = "19.1.0" __title__ = "attrs" __description__ = "Classes Without Boilerplate" diff --git a/pipenv/vendor/attr/__init__.pyi b/pipenv/vendor/attr/__init__.pyi index 492fb85e..fcb93b18 100644 --- a/pipenv/vendor/attr/__init__.pyi +++ b/pipenv/vendor/attr/__init__.pyi @@ -23,9 +23,9 @@ from . import validators as validators _T = TypeVar("_T") _C = TypeVar("_C", bound=type) -_ValidatorType = Callable[[Any, Attribute, _T], Any] +_ValidatorType = Callable[[Any, Attribute[_T], _T], Any] _ConverterType = Callable[[Any], _T] -_FilterType = Callable[[Attribute, Any], bool] +_FilterType = Callable[[Attribute[_T], _T], bool] # FIXME: in reality, if multiple validators are passed they must be in a list or tuple, # but those are invariant and so would prevent subtypes of _ValidatorType from working # when passed in a list or tuple. @@ -57,10 +57,10 @@ class Attribute(Generic[_T]): metadata: Dict[Any, Any] type: Optional[Type[_T]] kw_only: bool - def __lt__(self, x: Attribute) -> bool: ... - def __le__(self, x: Attribute) -> bool: ... - def __gt__(self, x: Attribute) -> bool: ... - def __ge__(self, x: Attribute) -> bool: ... + def __lt__(self, x: Attribute[_T]) -> bool: ... + def __le__(self, x: Attribute[_T]) -> bool: ... + def __gt__(self, x: Attribute[_T]) -> bool: ... + def __ge__(self, x: Attribute[_T]) -> bool: ... # NOTE: We had several choices for the annotation to use for type arg: # 1) Type[_T] @@ -167,6 +167,7 @@ def attrs( auto_attribs: bool = ..., kw_only: bool = ..., cache_hash: bool = ..., + auto_exc: bool = ..., ) -> _C: ... @overload def attrs( @@ -184,14 +185,15 @@ def attrs( auto_attribs: bool = ..., kw_only: bool = ..., cache_hash: bool = ..., + auto_exc: bool = ..., ) -> Callable[[_C], _C]: ... # TODO: add support for returning NamedTuple from the mypy plugin -class _Fields(Tuple[Attribute, ...]): - def __getattr__(self, name: str) -> Attribute: ... +class _Fields(Tuple[Attribute[Any], ...]): + def __getattr__(self, name: str) -> Attribute[Any]: ... def fields(cls: type) -> _Fields: ... -def fields_dict(cls: type) -> Dict[str, Attribute]: ... +def fields_dict(cls: type) -> Dict[str, Attribute[Any]]: ... def validate(inst: Any) -> None: ... # TODO: add support for returning a proper attrs class from the mypy plugin @@ -212,6 +214,7 @@ def make_class( auto_attribs: bool = ..., kw_only: bool = ..., cache_hash: bool = ..., + auto_exc: bool = ..., ) -> type: ... # _funcs -- @@ -223,7 +226,7 @@ def make_class( def asdict( inst: Any, recurse: bool = ..., - filter: Optional[_FilterType] = ..., + filter: Optional[_FilterType[Any]] = ..., dict_factory: Type[Mapping[Any, Any]] = ..., retain_collection_types: bool = ..., ) -> Dict[str, Any]: ... @@ -232,8 +235,8 @@ def asdict( def astuple( inst: Any, recurse: bool = ..., - filter: Optional[_FilterType] = ..., - tuple_factory: Type[Sequence] = ..., + filter: Optional[_FilterType[Any]] = ..., + tuple_factory: Type[Sequence[Any]] = ..., retain_collection_types: bool = ..., ) -> Tuple[Any, ...]: ... def has(cls: type) -> bool: ... diff --git a/pipenv/vendor/attr/_compat.py b/pipenv/vendor/attr/_compat.py index 5bb06593..9a99dcd9 100644 --- a/pipenv/vendor/attr/_compat.py +++ b/pipenv/vendor/attr/_compat.py @@ -20,6 +20,7 @@ else: if PY2: from UserDict import IterableUserDict + from collections import Mapping, Sequence # noqa # We 'bundle' isclass instead of using inspect as importing inspect is # fairly expensive (order of 10-15 ms for a modern machine in 2016) @@ -89,8 +90,27 @@ if PY2: res.data.update(d) # We blocked update, so we have to do it like this. return res + def just_warn(*args, **kw): # pragma: nocover + """ + We only warn on Python 3 because we are not aware of any concrete + consequences of not setting the cell on Python 2. + """ -else: + +else: # Python 3 and later. + from collections.abc import Mapping, Sequence # noqa + + def just_warn(*args, **kw): + """ + We only warn on Python 3 because we are not aware of any concrete + consequences of not setting the cell on Python 2. + """ + warnings.warn( + "Missing ctypes. Some features like bare super() or accessing " + "__class__ will not work with slotted classes.", + RuntimeWarning, + stacklevel=2, + ) def isclass(klass): return isinstance(klass, type) @@ -113,30 +133,6 @@ def import_ctypes(): return ctypes -if not PY2: - - def just_warn(*args, **kw): - """ - We only warn on Python 3 because we are not aware of any concrete - consequences of not setting the cell on Python 2. - """ - warnings.warn( - "Missing ctypes. Some features like bare super() or accessing " - "__class__ will not work with slots classes.", - RuntimeWarning, - stacklevel=2, - ) - - -else: - - def just_warn(*args, **kw): # pragma: nocover - """ - We only warn on Python 3 because we are not aware of any concrete - consequences of not setting the cell on Python 2. - """ - - def make_set_closure_cell(): """ Moved into a function for testability. diff --git a/pipenv/vendor/attr/_make.py b/pipenv/vendor/attr/_make.py index f7fd05e7..827175a4 100644 --- a/pipenv/vendor/attr/_make.py +++ b/pipenv/vendor/attr/_make.py @@ -409,12 +409,11 @@ def _transform_attrs(cls, these, auto_attribs, kw_only): a.kw_only is False ): had_default = True - if was_kw_only is True and a.kw_only is False: + if was_kw_only is True and a.kw_only is False and a.init is True: raise ValueError( "Non keyword-only attributes are not allowed after a " - "keyword-only attribute. Attribute in question: {a!r}".format( - a=a - ) + "keyword-only attribute (unless they are init=False). " + "Attribute in question: {a!r}".format(a=a) ) if was_kw_only is False and a.init is True and a.kw_only is True: was_kw_only = True @@ -454,6 +453,7 @@ class _ClassBuilder(object): "_has_post_init", "_delete_attribs", "_base_attr_map", + "_is_exc", ) def __init__( @@ -466,6 +466,7 @@ class _ClassBuilder(object): auto_attribs, kw_only, cache_hash, + is_exc, ): attrs, base_attrs, base_map = _transform_attrs( cls, these, auto_attribs, kw_only @@ -483,6 +484,7 @@ class _ClassBuilder(object): self._cache_hash = cache_hash self._has_post_init = bool(getattr(cls, "__attrs_post_init__", False)) self._delete_attribs = not bool(these) + self._is_exc = is_exc self._cls_dict["__attrs_attrs__"] = self._attrs @@ -530,6 +532,26 @@ class _ClassBuilder(object): for name, value in self._cls_dict.items(): setattr(cls, name, value) + # Attach __setstate__. This is necessary to clear the hash code + # cache on deserialization. See issue + # https://github.com/python-attrs/attrs/issues/482 . + # Note that this code only handles setstate for dict classes. + # For slotted classes, see similar code in _create_slots_class . + if self._cache_hash: + existing_set_state_method = getattr(cls, "__setstate__", None) + if existing_set_state_method: + raise NotImplementedError( + "Currently you cannot use hash caching if " + "you specify your own __setstate__ method." + "See https://github.com/python-attrs/attrs/issues/494 ." + ) + + def cache_hash_set_state(chss_self, _): + # clear hash code cache + setattr(chss_self, _hash_cache_field, None) + + setattr(cls, "__setstate__", cache_hash_set_state) + return cls def _create_slots_class(self): @@ -582,6 +604,8 @@ class _ClassBuilder(object): """ return tuple(getattr(self, name) for name in state_attr_names) + hash_caching_enabled = self._cache_hash + def slots_setstate(self, state): """ Automatically created by attrs. @@ -589,6 +613,13 @@ class _ClassBuilder(object): __bound_setattr = _obj_setattr.__get__(self, Attribute) for name, value in zip(state_attr_names, state): __bound_setattr(name, value) + # Clearing the hash code cache on deserialization is needed + # because hash codes can change from run to run. See issue + # https://github.com/python-attrs/attrs/issues/482 . + # Note that this code only handles setstate for slotted classes. + # For dict classes, see similar code in _patch_original_class . + if hash_caching_enabled: + __bound_setattr(_hash_cache_field, None) # slots and frozen require __getstate__/__setstate__ to work cd["__getstate__"] = slots_getstate @@ -660,6 +691,7 @@ class _ClassBuilder(object): self._slots, self._cache_hash, self._base_attr_map, + self._is_exc, ) ) @@ -710,6 +742,7 @@ def attrs( auto_attribs=False, kw_only=False, cache_hash=False, + auto_exc=False, ): r""" A class decorator that adds `dunder @@ -815,10 +848,23 @@ def attrs( :param bool cache_hash: Ensure that the object's hash code is computed only once and stored on the object. If this is set to ``True``, hashing must be either explicitly or implicitly enabled for this - class. If the hash code is cached, then no attributes of this - class which participate in hash code computation may be mutated - after object creation. + class. If the hash code is cached, avoid any reassignments of + fields involved in hash code computation or mutations of the objects + those fields point to after object creation. If such changes occur, + the behavior of the object's hash code is undefined. + :param bool auto_exc: If the class subclasses :class:`BaseException` + (which implicitly includes any subclass of any exception), the + following happens to behave like a well-behaved Python exceptions + class: + - the values for *cmp* and *hash* are ignored and the instances compare + and hash by the instance's ids (N.B. ``attrs`` will *not* remove + existing implementations of ``__hash__`` or the equality methods. It + just won't add own ones.), + - all attributes that are either passed into ``__init__`` or have a + default value are additionally available as a tuple in the ``args`` + attribute, + - the value of *str* is ignored leaving ``__str__`` to base classes. .. versionadded:: 16.0.0 *slots* .. versionadded:: 16.1.0 *frozen* @@ -838,12 +884,16 @@ def attrs( to each other. .. versionadded:: 18.2.0 *kw_only* .. versionadded:: 18.2.0 *cache_hash* + .. versionadded:: 19.1.0 *auto_exc* """ def wrap(cls): + if getattr(cls, "__class__", None) is None: raise TypeError("attrs only works with new-style classes.") + is_exc = auto_exc is True and issubclass(cls, BaseException) + builder = _ClassBuilder( cls, these, @@ -853,13 +903,14 @@ def attrs( auto_attribs, kw_only, cache_hash, + is_exc, ) if repr is True: builder.add_repr(repr_ns) if str is True: builder.add_str() - if cmp is True: + if cmp is True and not is_exc: builder.add_cmp() if hash is not True and hash is not False and hash is not None: @@ -874,7 +925,11 @@ def attrs( " hashing must be either explicitly or implicitly " "enabled." ) - elif hash is True or (hash is None and cmp is True and frozen is True): + elif ( + hash is True + or (hash is None and cmp is True and frozen is True) + and is_exc is False + ): builder.add_hash() else: if cache_hash: @@ -1213,7 +1268,9 @@ def _add_repr(cls, ns=None, attrs=None): return cls -def _make_init(attrs, post_init, frozen, slots, cache_hash, base_attr_map): +def _make_init( + attrs, post_init, frozen, slots, cache_hash, base_attr_map, is_exc +): attrs = [a for a in attrs if a.init or a.default is not NOTHING] # We cache the generated init methods for the same kinds of attributes. @@ -1222,16 +1279,18 @@ def _make_init(attrs, post_init, frozen, slots, cache_hash, base_attr_map): unique_filename = "".format(sha1.hexdigest()) script, globs, annotations = _attrs_to_init_script( - attrs, frozen, slots, post_init, cache_hash, base_attr_map + attrs, frozen, slots, post_init, cache_hash, base_attr_map, is_exc ) locs = {} bytecode = compile(script, unique_filename, "exec") attr_dict = dict((a.name, a) for a in attrs) globs.update({"NOTHING": NOTHING, "attr_dict": attr_dict}) + if frozen is True: # Save the lookup overhead in __init__ if we need to circumvent # immutability. globs["_cached_setattr"] = _obj_setattr + eval(bytecode, globs, locs) # In order of debuggers like PDB being able to step through the code, @@ -1245,24 +1304,10 @@ def _make_init(attrs, post_init, frozen, slots, cache_hash, base_attr_map): __init__ = locs["__init__"] __init__.__annotations__ = annotations + return __init__ -def _add_init(cls, frozen): - """ - Add a __init__ method to *cls*. If *frozen* is True, make it immutable. - """ - cls.__init__ = _make_init( - cls.__attrs_attrs__, - getattr(cls, "__attrs_post_init__", False), - frozen, - _is_slot_cls(cls), - cache_hash=False, - base_attr_map={}, - ) - return cls - - def fields(cls): """ Return the tuple of ``attrs`` attributes for a class. @@ -1348,7 +1393,7 @@ def _is_slot_attr(a_name, base_attr_map): def _attrs_to_init_script( - attrs, frozen, slots, post_init, cache_hash, base_attr_map + attrs, frozen, slots, post_init, cache_hash, base_attr_map, is_exc ): """ Return a script of an initializer for *attrs* and a dict of globals. @@ -1597,6 +1642,13 @@ def _attrs_to_init_script( init_hash_cache = "self.%s = %s" lines.append(init_hash_cache % (_hash_cache_field, "None")) + # For exceptions we rely on BaseException.__init__ for proper + # initialization. + if is_exc: + vals = ",".join("self." + a.name for a in attrs if a.init) + + lines.append("BaseException.__init__(self, %s)" % (vals,)) + args = ", ".join(args) if kw_only_args: if PY2: diff --git a/pipenv/vendor/attr/filters.pyi b/pipenv/vendor/attr/filters.pyi index a618140c..68368fe2 100644 --- a/pipenv/vendor/attr/filters.pyi +++ b/pipenv/vendor/attr/filters.pyi @@ -1,5 +1,5 @@ -from typing import Union +from typing import Union, Any from . import Attribute, _FilterType -def include(*what: Union[type, Attribute]) -> _FilterType: ... -def exclude(*what: Union[type, Attribute]) -> _FilterType: ... +def include(*what: Union[type, Attribute[Any]]) -> _FilterType[Any]: ... +def exclude(*what: Union[type, Attribute[Any]]) -> _FilterType[Any]: ... diff --git a/pipenv/vendor/attr/validators.py b/pipenv/vendor/attr/validators.py index f12d0aa5..7fc4446b 100644 --- a/pipenv/vendor/attr/validators.py +++ b/pipenv/vendor/attr/validators.py @@ -136,7 +136,7 @@ class _InValidator(object): def __call__(self, inst, attr, value): try: in_options = value in self.options - except TypeError as e: # e.g. `1 in "abc"` + except TypeError: # e.g. `1 in "abc"` in_options = False if not in_options: @@ -168,3 +168,115 @@ def in_(options): .. versionadded:: 17.1.0 """ return _InValidator(options) + + +@attrs(repr=False, slots=False, hash=True) +class _IsCallableValidator(object): + def __call__(self, inst, attr, value): + """ + We use a callable class to be able to change the ``__repr__``. + """ + if not callable(value): + raise TypeError("'{name}' must be callable".format(name=attr.name)) + + def __repr__(self): + return "" + + +def is_callable(): + """ + A validator that raises a :class:`TypeError` if the initializer is called + with a value for this particular attribute that is not callable. + + .. versionadded:: 19.1.0 + + :raises TypeError: With a human readable error message containing the + attribute (of type :class:`attr.Attribute`) name. + """ + return _IsCallableValidator() + + +@attrs(repr=False, slots=True, hash=True) +class _DeepIterable(object): + member_validator = attrib(validator=is_callable()) + iterable_validator = attrib( + default=None, validator=optional(is_callable()) + ) + + def __call__(self, inst, attr, value): + """ + We use a callable class to be able to change the ``__repr__``. + """ + if self.iterable_validator is not None: + self.iterable_validator(inst, attr, value) + + for member in value: + self.member_validator(inst, attr, member) + + def __repr__(self): + iterable_identifier = ( + "" + if self.iterable_validator is None + else " {iterable!r}".format(iterable=self.iterable_validator) + ) + return ( + "" + ).format( + iterable_identifier=iterable_identifier, + member=self.member_validator, + ) + + +def deep_iterable(member_validator, iterable_validator=None): + """ + A validator that performs deep validation of an iterable. + + :param member_validator: Validator to apply to iterable members + :param iterable_validator: Validator to apply to iterable itself + (optional) + + .. versionadded:: 19.1.0 + + :raises TypeError: if any sub-validators fail + """ + return _DeepIterable(member_validator, iterable_validator) + + +@attrs(repr=False, slots=True, hash=True) +class _DeepMapping(object): + key_validator = attrib(validator=is_callable()) + value_validator = attrib(validator=is_callable()) + mapping_validator = attrib(default=None, validator=optional(is_callable())) + + def __call__(self, inst, attr, value): + """ + We use a callable class to be able to change the ``__repr__``. + """ + if self.mapping_validator is not None: + self.mapping_validator(inst, attr, value) + + for key in value: + self.key_validator(inst, attr, key) + self.value_validator(inst, attr, value[key]) + + def __repr__(self): + return ( + "" + ).format(key=self.key_validator, value=self.value_validator) + + +def deep_mapping(key_validator, value_validator, mapping_validator=None): + """ + A validator that performs deep validation of a dictionary. + + :param key_validator: Validator to apply to dictionary keys + :param value_validator: Validator to apply to dictionary values + :param mapping_validator: Validator to apply to top-level mapping + attribute (optional) + + .. versionadded:: 19.1.0 + + :raises TypeError: if any sub-validators fail + """ + return _DeepMapping(key_validator, value_validator, mapping_validator) diff --git a/pipenv/vendor/attr/validators.pyi b/pipenv/vendor/attr/validators.pyi index abbaedf1..01af0684 100644 --- a/pipenv/vendor/attr/validators.pyi +++ b/pipenv/vendor/attr/validators.pyi @@ -12,3 +12,13 @@ def optional( ) -> _ValidatorType[Optional[_T]]: ... def in_(options: Container[_T]) -> _ValidatorType[_T]: ... def and_(*validators: _ValidatorType[_T]) -> _ValidatorType[_T]: ... +def deep_iterable( + member_validator: _ValidatorType[_T], + iterable_validator: Optional[_ValidatorType[_T]], +) -> _ValidatorType[_T]: ... +def deep_mapping( + key_validator: _ValidatorType[_T], + value_validator: _ValidatorType[_T], + mapping_validator: Optional[_ValidatorType[_T]], +) -> _ValidatorType[_T]: ... +def is_callable() -> _ValidatorType[_T]: ... diff --git a/pipenv/vendor/backports/__init__.py b/pipenv/vendor/backports/__init__.py index 0c64b4c1..e449e521 100644 --- a/pipenv/vendor/backports/__init__.py +++ b/pipenv/vendor/backports/__init__.py @@ -1,5 +1,5 @@ __path__ = __import__('pkgutil').extend_path(__path__, __name__) from . import weakref -from . import enum from . import shutil_get_terminal_size +from . import enum from . import functools_lru_cache diff --git a/pipenv/vendor/blindspin/__init__.py b/pipenv/vendor/blindspin/__init__.py deleted file mode 100644 index a1230e83..00000000 --- a/pipenv/vendor/blindspin/__init__.py +++ /dev/null @@ -1,73 +0,0 @@ -# -*- coding: utf-8 -*- - -import sys -import threading -import time -import itertools - - -class Spinner(object): - spinner_cycle = itertools.cycle(u'⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏') - - def __init__(self, beep=False, force=False): - self.beep = beep - self.force = force - self.stop_running = None - self.spin_thread = None - - def start(self): - if sys.stdout.isatty() or self.force: - self.stop_running = threading.Event() - self.spin_thread = threading.Thread(target=self.init_spin) - self.spin_thread.start() - - def stop(self): - if self.spin_thread: - self.stop_running.set() - self.spin_thread.join() - - def init_spin(self): - while not self.stop_running.is_set(): - next_val = next(self.spinner_cycle) - if sys.version_info[0] == 2: - next_val = next_val.encode('utf-8') - sys.stdout.write(next_val) - sys.stdout.flush() - time.sleep(0.07) - sys.stdout.write('\b') - - def __enter__(self): - self.start() - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self.stop() - if self.beep: - sys.stdout.write('\7') - sys.stdout.flush() - return False - - -def spinner(beep=False, force=False): - """This function creates a context manager that is used to display a - spinner on stdout as long as the context has not exited. - - The spinner is created only if stdout is not redirected, or if the spinner - is forced using the `force` parameter. - - Parameters - ---------- - beep : bool - Beep when spinner finishes. - force : bool - Force creation of spinner even when stdout is redirected. - - Example - ------- - - with spinner(): - do_something() - do_something_else() - - """ - return Spinner(beep, force) diff --git a/pipenv/vendor/cached_property.py b/pipenv/vendor/cached_property.py index a06be97a..125f6195 100644 --- a/pipenv/vendor/cached_property.py +++ b/pipenv/vendor/cached_property.py @@ -2,7 +2,7 @@ __author__ = "Daniel Greenfeld" __email__ = "pydanny@gmail.com" -__version__ = "1.4.3" +__version__ = "1.5.1" __license__ = "BSD" from time import time diff --git a/pipenv/vendor/cerberus/__init__.py b/pipenv/vendor/cerberus/__init__.py index 4b528cff..1e0f0d54 100644 --- a/pipenv/vendor/cerberus/__init__.py +++ b/pipenv/vendor/cerberus/__init__.py @@ -10,20 +10,23 @@ from __future__ import absolute_import +from pkg_resources import get_distribution, DistributionNotFound + from cerberus.validator import DocumentError, Validator -from cerberus.schema import (rules_set_registry, schema_registry, Registry, - SchemaError) +from cerberus.schema import rules_set_registry, schema_registry, SchemaError from cerberus.utils import TypeDefinition -__version__ = "1.2" +try: + __version__ = get_distribution("Cerberus").version +except DistributionNotFound: + __version__ = "unknown" __all__ = [ DocumentError.__name__, - Registry.__name__, SchemaError.__name__, TypeDefinition.__name__, Validator.__name__, - 'schema_registry', - 'rules_set_registry' + "schema_registry", + "rules_set_registry", ] diff --git a/pipenv/vendor/cerberus/errors.py b/pipenv/vendor/cerberus/errors.py index 4c497eeb..14e27eb8 100644 --- a/pipenv/vendor/cerberus/errors.py +++ b/pipenv/vendor/cerberus/errors.py @@ -3,12 +3,12 @@ from __future__ import absolute_import -from collections import defaultdict, namedtuple, MutableMapping +from collections import defaultdict, namedtuple from copy import copy, deepcopy from functools import wraps from pprint import pformat -from cerberus.platform import PYTHON_VERSION +from cerberus.platform import PYTHON_VERSION, MutableMapping from cerberus.utils import compare_paths_lt, quote_string @@ -54,6 +54,7 @@ UNALLOWED_VALUE = ErrorDefinition(0x44, 'allowed') UNALLOWED_VALUES = ErrorDefinition(0x45, 'allowed') FORBIDDEN_VALUE = ErrorDefinition(0x46, 'forbidden') FORBIDDEN_VALUES = ErrorDefinition(0x47, 'forbidden') +MISSING_MEMBERS = ErrorDefinition(0x48, 'contains') # other NORMALIZATION = ErrorDefinition(0x60, None) @@ -66,9 +67,10 @@ SETTING_DEFAULT_FAILED = ErrorDefinition(0x64, 'default_setter') ERROR_GROUP = ErrorDefinition(0x80, None) MAPPING_SCHEMA = ErrorDefinition(0x81, 'schema') SEQUENCE_SCHEMA = ErrorDefinition(0x82, 'schema') -KEYSCHEMA = ErrorDefinition(0x83, 'keyschema') -VALUESCHEMA = ErrorDefinition(0x84, 'valueschema') -BAD_ITEMS = ErrorDefinition(0x8f, 'items') +# TODO remove KEYSCHEMA AND VALUESCHEMA with next major release +KEYSRULES = KEYSCHEMA = ErrorDefinition(0x83, 'keysrules') +VALUESRULES = VALUESCHEMA = ErrorDefinition(0x84, 'valuesrules') +BAD_ITEMS = ErrorDefinition(0x8F, 'items') LOGICAL = ErrorDefinition(0x90, None) NONEOF = ErrorDefinition(0x91, 'noneof') @@ -79,8 +81,7 @@ ALLOF = ErrorDefinition(0x94, 'allof') """ SchemaError messages """ -SCHEMA_ERROR_DEFINITION_TYPE = \ - "schema definition for field '{0}' must be a dict" +SCHEMA_ERROR_DEFINITION_TYPE = "schema definition for field '{0}' must be a dict" SCHEMA_ERROR_MISSING = "validation schema missing" @@ -89,8 +90,8 @@ SCHEMA_ERROR_MISSING = "validation schema missing" class ValidationError(object): """ A simple class to store and query basic error information. """ - def __init__(self, document_path, schema_path, code, rule, constraint, - value, info): + + def __init__(self, document_path, schema_path, code, rule, constraint, value, info): self.document_path = document_path """ The path to the field within the document that caused the error. Type: :class:`tuple` """ @@ -115,8 +116,7 @@ class ValidationError(object): def __hash__(self): """ Expects that all other properties are transitively determined. """ - return hash(self.document_path) ^ hash(self.schema_path) \ - ^ hash(self.code) + return hash(self.document_path) ^ hash(self.schema_path) ^ hash(self.code) def __lt__(self, other): if self.document_path != other.document_path: @@ -125,20 +125,24 @@ class ValidationError(object): return compare_paths_lt(self.schema_path, other.schema_path) def __repr__(self): - return "{class_name} @ {memptr} ( " \ - "document_path={document_path}," \ - "schema_path={schema_path}," \ - "code={code}," \ - "constraint={constraint}," \ - "value={value}," \ - "info={info} )"\ - .format(class_name=self.__class__.__name__, memptr=hex(id(self)), # noqa: E501 - document_path=self.document_path, - schema_path=self.schema_path, - code=hex(self.code), - constraint=quote_string(self.constraint), - value=quote_string(self.value), - info=self.info) + return ( + "{class_name} @ {memptr} ( " + "document_path={document_path}," + "schema_path={schema_path}," + "code={code}," + "constraint={constraint}," + "value={value}," + "info={info} )".format( + class_name=self.__class__.__name__, + memptr=hex(id(self)), # noqa: E501 + document_path=self.document_path, + schema_path=self.schema_path, + code=hex(self.code), + constraint=quote_string(self.constraint), + value=quote_string(self.value), + info=self.info, + ) + ) @property def child_errors(self): @@ -190,11 +194,13 @@ class ErrorList(list): """ A list for :class:`~cerberus.errors.ValidationError` instances that can be queried with the ``in`` keyword for a particular :class:`~cerberus.errors.ErrorDefinition`. """ + def __contains__(self, error_definition): - for code in (x.code for x in self): - if code == error_definition.code: - return True - return False + if not isinstance(error_definition, ErrorDefinition): + raise TypeError + + wanted_code = error_definition.code + return any(x.code == wanted_code for x in self) class ErrorTreeNode(MutableMapping): @@ -203,14 +209,10 @@ class ErrorTreeNode(MutableMapping): def __init__(self, path, parent_node): self.parent_node = parent_node self.tree_root = self.parent_node.tree_root - self.path = path[:self.parent_node.depth + 1] + self.path = path[: self.parent_node.depth + 1] self.errors = ErrorList() self.descendants = {} - def __add__(self, error): - self.add(error) - return self - def __contains__(self, item): if isinstance(item, ErrorDefinition): return item in self.errors @@ -228,6 +230,7 @@ class ErrorTreeNode(MutableMapping): for error in self.errors: if item.code == error.code: return error + return None else: return self.descendants.get(item) @@ -258,14 +261,16 @@ class ErrorTreeNode(MutableMapping): if key not in self.descendants: self[key] = ErrorTreeNode(error_path, self) + node = self[key] + if len(error_path) == self.depth + 1: - self[key].errors.append(error) - self[key].errors.sort() + node.errors.append(error) + node.errors.sort() if error.is_group_error: for child_error in error.child_errors: - self.tree_root += child_error + self.tree_root.add(child_error) else: - self[key] += error + node.add(error) def _path_of_(self, error): return getattr(error, self.tree_type + '_path') @@ -274,14 +279,15 @@ class ErrorTreeNode(MutableMapping): class ErrorTree(ErrorTreeNode): """ Base class for :class:`~cerberus.errors.DocumentErrorTree` and :class:`~cerberus.errors.SchemaErrorTree`. """ - def __init__(self, errors=[]): + + def __init__(self, errors=()): self.parent_node = None self.tree_root = self self.path = () self.errors = ErrorList() self.descendants = {} for error in errors: - self += error + self.add(error) def add(self, error): """ Add an error to the tree. @@ -323,18 +329,21 @@ class ErrorTree(ErrorTreeNode): class DocumentErrorTree(ErrorTree): """ Implements a dict-like class to query errors by indexes following the structure of a validated document. """ + tree_type = 'document' class SchemaErrorTree(ErrorTree): """ Implements a dict-like class to query errors by indexes following the structure of the used schema. """ + tree_type = 'schema' class BaseErrorHandler(object): """ Base class for all error handlers. Subclasses are identified as error-handlers with an instance-test. """ + def __init__(self, *args, **kwargs): """ Optionally initialize a new instance. """ pass @@ -411,9 +420,9 @@ def encode_unicode(f): This decorator ensures that if legacy Python is used unicode strings are encoded before passing to a function. """ + @wraps(f) def wrapped(obj, error): - def _encode(value): """Helper encoding unicode strings into binary utf-8""" if isinstance(value, unicode): # noqa: F821 @@ -436,56 +445,52 @@ class BasicErrorHandler(BaseErrorHandler): through :class:`str` a pretty-formatted representation of that tree is returned. """ - messages = {0x00: "{0}", - 0x01: "document is missing", - 0x02: "required field", - 0x03: "unknown field", - 0x04: "field '{0}' is required", - 0x05: "depends on these values: {constraint}", - 0x06: "{0} must not be present with '{field}'", - - 0x21: "'{0}' is not a document, must be a dict", - 0x22: "empty values not allowed", - 0x23: "null value not allowed", - 0x24: "must be of {constraint} type", - 0x25: "must be of dict type", - 0x26: "length of list should be {constraint}, it is {0}", - 0x27: "min length is {constraint}", - 0x28: "max length is {constraint}", - - 0x41: "value does not match regex '{constraint}'", - 0x42: "min value is {constraint}", - 0x43: "max value is {constraint}", - 0x44: "unallowed value {value}", - 0x45: "unallowed values {0}", - 0x46: "unallowed value {value}", - 0x47: "unallowed values {0}", - - 0x61: "field '{field}' cannot be coerced: {0}", - 0x62: "field '{field}' cannot be renamed: {0}", - 0x63: "field is read-only", - 0x64: "default value for '{field}' cannot be set: {0}", - - 0x81: "mapping doesn't validate subschema: {0}", - 0x82: "one or more sequence-items don't validate: {0}", - 0x83: "one or more keys of a mapping don't validate: {0}", - 0x84: "one or more values in a mapping don't validate: {0}", - 0x85: "one or more sequence-items don't validate: {0}", - - 0x91: "one or more definitions validate", - 0x92: "none or more than one rule validate", - 0x93: "no definitions validate", - 0x94: "one or more definitions don't validate" - } + messages = { + 0x00: "{0}", + 0x01: "document is missing", + 0x02: "required field", + 0x03: "unknown field", + 0x04: "field '{0}' is required", + 0x05: "depends on these values: {constraint}", + 0x06: "{0} must not be present with '{field}'", + 0x21: "'{0}' is not a document, must be a dict", + 0x22: "empty values not allowed", + 0x23: "null value not allowed", + 0x24: "must be of {constraint} type", + 0x25: "must be of dict type", + 0x26: "length of list should be {constraint}, it is {0}", + 0x27: "min length is {constraint}", + 0x28: "max length is {constraint}", + 0x41: "value does not match regex '{constraint}'", + 0x42: "min value is {constraint}", + 0x43: "max value is {constraint}", + 0x44: "unallowed value {value}", + 0x45: "unallowed values {0}", + 0x46: "unallowed value {value}", + 0x47: "unallowed values {0}", + 0x48: "missing members {0}", + 0x61: "field '{field}' cannot be coerced: {0}", + 0x62: "field '{field}' cannot be renamed: {0}", + 0x63: "field is read-only", + 0x64: "default value for '{field}' cannot be set: {0}", + 0x81: "mapping doesn't validate subschema: {0}", + 0x82: "one or more sequence-items don't validate: {0}", + 0x83: "one or more keys of a mapping don't validate: {0}", + 0x84: "one or more values in a mapping don't validate: {0}", + 0x85: "one or more sequence-items don't validate: {0}", + 0x91: "one or more definitions validate", + 0x92: "none or more than one rule validate", + 0x93: "no definitions validate", + 0x94: "one or more definitions don't validate", + } def __init__(self, tree=None): self.tree = {} if tree is None else tree - def __call__(self, errors=None): - if errors is not None: - self.clear() - self.extend(errors) + def __call__(self, errors): + self.clear() + self.extend(errors) return self.pretty_tree def __str__(self): @@ -511,8 +516,9 @@ class BasicErrorHandler(BaseErrorHandler): elif error.is_group_error: self._insert_group_error(error) elif error.code in self.messages: - self._insert_error(error.document_path, - self._format_message(error.field, error)) + self._insert_error( + error.document_path, self._format_message(error.field, error) + ) def clear(self): self.tree = {} @@ -522,8 +528,8 @@ class BasicErrorHandler(BaseErrorHandler): def _format_message(self, field, error): return self.messages[error.code].format( - *error.info, constraint=error.constraint, - field=field, value=error.value) + *error.info, constraint=error.constraint, field=field, value=error.value + ) def _insert_error(self, path, node): """ Adds an error or sub-tree to :attr:tree. @@ -559,14 +565,14 @@ class BasicErrorHandler(BaseErrorHandler): elif child_error.is_group_error: self._insert_group_error(child_error) else: - self._insert_error(child_error.document_path, - self._format_message(child_error.field, - child_error)) + self._insert_error( + child_error.document_path, + self._format_message(child_error.field, child_error), + ) def _insert_logic_error(self, error): field = error.field - self._insert_error(error.document_path, - self._format_message(field, error)) + self._insert_error(error.document_path, self._format_message(field, error)) for definition_errors in error.definitions_errors.values(): for child_error in definition_errors: @@ -575,8 +581,10 @@ class BasicErrorHandler(BaseErrorHandler): elif child_error.is_group_error: self._insert_group_error(child_error) else: - self._insert_error(child_error.document_path, - self._format_message(field, child_error)) + self._insert_error( + child_error.document_path, + self._format_message(field, child_error), + ) def _purge_empty_dicts(self, error_list): subtree = error_list[-1] diff --git a/pipenv/vendor/cerberus/platform.py b/pipenv/vendor/cerberus/platform.py index eca9858d..66b1d5f0 100644 --- a/pipenv/vendor/cerberus/platform.py +++ b/pipenv/vendor/cerberus/platform.py @@ -12,3 +12,29 @@ if PYTHON_VERSION < 3: else: _str_type = str _int_types = (int,) + + +if PYTHON_VERSION < 3.3: + from collections import ( # noqa: F401 + Callable, + Container, + Hashable, + Iterable, + Mapping, + MutableMapping, + Sequence, + Set, + Sized, + ) +else: + from collections.abc import ( # noqa: F401 + Callable, + Container, + Hashable, + Iterable, + Mapping, + MutableMapping, + Sequence, + Set, + Sized, + ) diff --git a/pipenv/vendor/cerberus/schema.py b/pipenv/vendor/cerberus/schema.py index 3ddce172..305e59ff 100644 --- a/pipenv/vendor/cerberus/schema.py +++ b/pipenv/vendor/cerberus/schema.py @@ -1,13 +1,23 @@ from __future__ import absolute_import -from collections import (Callable, Hashable, Iterable, Mapping, - MutableMapping, Sequence) from copy import copy +from warnings import warn from cerberus import errors -from cerberus.platform import _str_type -from cerberus.utils import (get_Validator_class, validator_factory, - mapping_hash, TypeDefinition) +from cerberus.platform import ( + _str_type, + Callable, + Hashable, + Mapping, + MutableMapping, + Sequence, +) +from cerberus.utils import ( + get_Validator_class, + validator_factory, + mapping_hash, + TypeDefinition, +) class _Abort(Exception): @@ -17,6 +27,7 @@ class _Abort(Exception): class SchemaError(Exception): """ Raised when the validation schema is missing, has the wrong format or contains errors. """ + pass @@ -26,18 +37,19 @@ class DefinitionSchema(MutableMapping): def __new__(cls, *args, **kwargs): if 'SchemaValidator' not in globals(): global SchemaValidator - SchemaValidator = validator_factory('SchemaValidator', - SchemaValidatorMixin) + SchemaValidator = validator_factory('SchemaValidator', SchemaValidatorMixin) types_mapping = SchemaValidator.types_mapping.copy() - types_mapping.update({ - 'callable': TypeDefinition('callable', (Callable,), ()), - 'hashable': TypeDefinition('hashable', (Hashable,), ()) - }) + types_mapping.update( + { + 'callable': TypeDefinition('callable', (Callable,), ()), + 'hashable': TypeDefinition('hashable', (Hashable,), ()), + } + ) SchemaValidator.types_mapping = types_mapping return super(DefinitionSchema, cls).__new__(cls) - def __init__(self, validator, schema={}): + def __init__(self, validator, schema): """ :param validator: An instance of Validator-(sub-)class that uses this schema. @@ -45,8 +57,7 @@ class DefinitionSchema(MutableMapping): one. """ if not isinstance(validator, get_Validator_class()): - raise RuntimeError('validator argument must be a Validator-' - 'instance.') + raise RuntimeError('validator argument must be a Validator-' 'instance.') self.validator = validator if isinstance(schema, _str_type): @@ -56,14 +67,16 @@ class DefinitionSchema(MutableMapping): try: schema = dict(schema) except Exception: - raise SchemaError( - errors.SCHEMA_ERROR_DEFINITION_TYPE.format(schema)) + raise SchemaError(errors.SCHEMA_ERROR_DEFINITION_TYPE.format(schema)) self.validation_schema = SchemaValidationSchema(validator) self.schema_validator = SchemaValidator( - None, allow_unknown=self.validation_schema, + None, + allow_unknown=self.validation_schema, error_handler=errors.SchemaErrorHandler, - target_schema=schema, target_validator=validator) + target_schema=schema, + target_validator=validator, + ) schema = self.expand(schema) self.validate(schema) @@ -110,6 +123,10 @@ class DefinitionSchema(MutableMapping): schema = cls._expand_subschemas(schema) except Exception: pass + + # TODO remove this with the next major release + schema = cls._rename_deprecated_rulenames(schema) + return schema @classmethod @@ -119,13 +136,15 @@ class DefinitionSchema(MutableMapping): :param schema: The schema-definition to expand. :return: The expanded schema-definition. """ + def is_of_rule(x): - return isinstance(x, _str_type) and \ - x.startswith(('allof_', 'anyof_', 'noneof_', 'oneof_')) + return isinstance(x, _str_type) and x.startswith( + ('allof_', 'anyof_', 'noneof_', 'oneof_') + ) for field in schema: for of_rule in (x for x in schema[field] if is_of_rule(x)): - operator, rule = of_rule.split('_') + operator, rule = of_rule.split('_', 1) schema[field].update({operator: []}) for value in schema[field][of_rule]: schema[field][operator].append({rule: value}) @@ -135,15 +154,15 @@ class DefinitionSchema(MutableMapping): @classmethod def _expand_subschemas(cls, schema): def has_schema_rule(): - return isinstance(schema[field], Mapping) and \ - 'schema' in schema[field] + return isinstance(schema[field], Mapping) and 'schema' in schema[field] def has_mapping_schema(): """ Tries to determine heuristically if the schema-constraints are aimed to mappings. """ try: - return all(isinstance(x, Mapping) for x - in schema[field]['schema'].values()) + return all( + isinstance(x, Mapping) for x in schema[field]['schema'].values() + ) except TypeError: return False @@ -153,13 +172,12 @@ class DefinitionSchema(MutableMapping): elif has_mapping_schema(): schema[field]['schema'] = cls.expand(schema[field]['schema']) else: # assumes schema-constraints for a sequence - schema[field]['schema'] = \ - cls.expand({0: schema[field]['schema']})[0] + schema[field]['schema'] = cls.expand({0: schema[field]['schema']})[0] - for rule in ('keyschema', 'valueschema'): + # TODO remove the last two values in the tuple with the next major release + for rule in ('keysrules', 'valuesrules', 'keyschema', 'valueschema'): if rule in schema[field]: - schema[field][rule] = \ - cls.expand({0: schema[field][rule]})[0] + schema[field][rule] = cls.expand({0: schema[field][rule]})[0] for rule in ('allof', 'anyof', 'items', 'noneof', 'oneof'): if rule in schema[field]: @@ -171,6 +189,12 @@ class DefinitionSchema(MutableMapping): schema[field][rule] = new_rules_definition return schema + def get(self, item, default=None): + return self.schema.get(item, default) + + def items(self): + return self.schema.items() + def update(self, schema): try: schema = self.expand(schema) @@ -178,31 +202,64 @@ class DefinitionSchema(MutableMapping): _new_schema.update(schema) self.validate(_new_schema) except ValueError: - raise SchemaError(errors.SCHEMA_ERROR_DEFINITION_TYPE - .format(schema)) + raise SchemaError(errors.SCHEMA_ERROR_DEFINITION_TYPE.format(schema)) except Exception as e: raise e else: self.schema = _new_schema + # TODO remove with next major release + @staticmethod + def _rename_deprecated_rulenames(schema): + for field, rules in schema.items(): + + if isinstance(rules, str): # registry reference + continue + + for old, new in ( + ('keyschema', 'keysrules'), + ('validator', 'check_with'), + ('valueschema', 'valuesrules'), + ): + + if old not in rules: + continue + + if new in rules: + raise RuntimeError( + "The rule '{new}' is also present with its old " + "name '{old}' in the same set of rules." + ) + + warn( + "The rule '{old}' was renamed to '{new}'. The old name will " + "not be available in the next major release of " + "Cerberus.".format(old=old, new=new), + DeprecationWarning, + ) + schema[field][new] = schema[field][old] + schema[field].pop(old) + + return schema + def regenerate_validation_schema(self): self.validation_schema = SchemaValidationSchema(self.validator) def validate(self, schema=None): + """ Validates a schema that defines rules against supported rules. + + :param schema: The schema to be validated as a legal cerberus schema + according to the rules of the assigned Validator object. + Raises a :class:`~cerberus.base.SchemaError` when an invalid + schema is encountered. """ if schema is None: schema = self.schema - _hash = (mapping_hash(schema), - mapping_hash(self.validator.types_mapping)) + _hash = (mapping_hash(schema), mapping_hash(self.validator.types_mapping)) if _hash not in self.validator._valid_schemas: self._validate(schema) self.validator._valid_schemas.add(_hash) def _validate(self, schema): - """ Validates a schema that defines rules against supported rules. - - :param schema: The schema to be validated as a legal cerberus schema - according to the rules of this Validator object. - """ if isinstance(schema, _str_type): schema = self.validator.schema_registry.get(schema, schema) @@ -212,8 +269,7 @@ class DefinitionSchema(MutableMapping): schema = copy(schema) for field in schema: if isinstance(schema[field], _str_type): - schema[field] = rules_set_registry.get(schema[field], - schema[field]) + schema[field] = rules_set_registry.get(schema[field], schema[field]) if not self.schema_validator(schema, normalize=False): raise SchemaError(self.schema_validator.errors) @@ -236,31 +292,31 @@ class UnvalidatedSchema(DefinitionSchema): class SchemaValidationSchema(UnvalidatedSchema): def __init__(self, validator): - self.schema = {'allow_unknown': False, - 'schema': validator.rules, - 'type': 'dict'} + self.schema = { + 'allow_unknown': False, + 'schema': validator.rules, + 'type': 'dict', + } class SchemaValidatorMixin(object): - """ This validator is extended to validate schemas passed to a Cerberus + """ This validator mixin provides mechanics to validate schemas passed to a Cerberus validator. """ + + def __init__(self, *args, **kwargs): + kwargs.setdefault('known_rules_set_refs', set()) + kwargs.setdefault('known_schema_refs', set()) + super(SchemaValidatorMixin, self).__init__(*args, **kwargs) + @property def known_rules_set_refs(self): """ The encountered references to rules set registry items. """ - return self._config.get('known_rules_set_refs', ()) - - @known_rules_set_refs.setter - def known_rules_set_refs(self, value): - self._config['known_rules_set_refs'] = value + return self._config['known_rules_set_refs'] @property def known_schema_refs(self): """ The encountered references to schema registry items. """ - return self._config.get('known_schema_refs', ()) - - @known_schema_refs.setter - def known_schema_refs(self, value): - self._config['known_schema_refs'] = value + return self._config['known_schema_refs'] @property def target_schema(self): @@ -272,35 +328,13 @@ class SchemaValidatorMixin(object): """ The validator whose schema is being validated. """ return self._config['target_validator'] - def _validate_logical(self, rule, field, value): - """ {'allowed': ('allof', 'anyof', 'noneof', 'oneof')} """ - if not isinstance(value, Sequence): - self._error(field, errors.BAD_TYPE) - return - - validator = self._get_child_validator( - document_crumb=rule, allow_unknown=False, - schema=self.target_validator.validation_rules) - - for constraints in value: - _hash = (mapping_hash({'turing': constraints}), - mapping_hash(self.target_validator.types_mapping)) - if _hash in self.target_validator._valid_schemas: - continue - - validator(constraints, normalize=False) - if validator._errors: - self._error(validator._errors) - else: - self.target_validator._valid_schemas.add(_hash) - - def _validator_bulk_schema(self, field, value): + def _check_with_bulk_schema(self, field, value): # resolve schema registry reference if isinstance(value, _str_type): if value in self.known_rules_set_refs: return else: - self.known_rules_set_refs += (value,) + self.known_rules_set_refs.add(value) definition = self.target_validator.rules_set_registry.get(value) if definition is None: self._error(field, 'Rules set definition %s not found.' % value) @@ -308,28 +342,32 @@ class SchemaValidatorMixin(object): else: value = definition - _hash = (mapping_hash({'turing': value}), - mapping_hash(self.target_validator.types_mapping)) + _hash = ( + mapping_hash({'turing': value}), + mapping_hash(self.target_validator.types_mapping), + ) if _hash in self.target_validator._valid_schemas: return validator = self._get_child_validator( - document_crumb=field, allow_unknown=False, - schema=self.target_validator.rules) + document_crumb=field, + allow_unknown=False, + schema=self.target_validator.rules, + ) validator(value, normalize=False) if validator._errors: self._error(validator._errors) else: self.target_validator._valid_schemas.add(_hash) - def _validator_dependencies(self, field, value): + def _check_with_dependencies(self, field, value): if isinstance(value, _str_type): pass elif isinstance(value, Mapping): validator = self._get_child_validator( document_crumb=field, - schema={'valueschema': {'type': 'list'}}, - allow_unknown=True + schema={'valuesrules': {'type': 'list'}}, + allow_unknown=True, ) if not validator(value, normalize=False): self._error(validator._errors) @@ -338,54 +376,36 @@ class SchemaValidatorMixin(object): path = self.document_path + (field,) self._error(path, 'All dependencies must be a hashable type.') - def _validator_handler(self, field, value): - if isinstance(value, Callable): - return - if isinstance(value, _str_type): - if value not in self.target_validator.validators + \ - self.target_validator.coercers: - self._error(field, '%s is no valid coercer' % value) - elif isinstance(value, Iterable): - for handler in value: - self._validator_handler(field, handler) - - def _validator_items(self, field, value): + def _check_with_items(self, field, value): for i, schema in enumerate(value): - self._validator_bulk_schema((field, i), schema) + self._check_with_bulk_schema((field, i), schema) - def _validator_schema(self, field, value): + def _check_with_schema(self, field, value): try: value = self._handle_schema_reference_for_validator(field, value) except _Abort: return - _hash = (mapping_hash(value), - mapping_hash(self.target_validator.types_mapping)) + _hash = (mapping_hash(value), mapping_hash(self.target_validator.types_mapping)) if _hash in self.target_validator._valid_schemas: return validator = self._get_child_validator( - document_crumb=field, - schema=None, allow_unknown=self.root_allow_unknown) + document_crumb=field, schema=None, allow_unknown=self.root_allow_unknown + ) validator(self._expand_rules_set_refs(value), normalize=False) if validator._errors: self._error(validator._errors) else: self.target_validator._valid_schemas.add(_hash) - def _handle_schema_reference_for_validator(self, field, value): - if not isinstance(value, _str_type): - return value - if value in self.known_schema_refs: - raise _Abort - - self.known_schema_refs += (value,) - definition = self.target_validator.schema_registry.get(value) - if definition is None: - path = self.document_path + (field,) - self._error(path, 'Schema definition {} not found.'.format(value)) - raise _Abort - return definition + def _check_with_type(self, field, value): + value = set((value,)) if isinstance(value, _str_type) else set(value) + invalid_constraints = value - set(self.target_validator.types) + if invalid_constraints: + self._error( + field, 'Unsupported types: {}'.format(', '.join(invalid_constraints)) + ) def _expand_rules_set_refs(self, schema): result = {} @@ -396,15 +416,46 @@ class SchemaValidatorMixin(object): result[k] = v return result - def _validator_type(self, field, value): - value = (value,) if isinstance(value, _str_type) else value - invalid_constraints = () - for constraint in value: - if constraint not in self.target_validator.types: - invalid_constraints += (constraint,) - if invalid_constraints: + def _handle_schema_reference_for_validator(self, field, value): + if not isinstance(value, _str_type): + return value + if value in self.known_schema_refs: + raise _Abort + + self.known_schema_refs.add(value) + definition = self.target_validator.schema_registry.get(value) + if definition is None: path = self.document_path + (field,) - self._error(path, 'Unsupported types: %s' % invalid_constraints) + self._error(path, 'Schema definition {} not found.'.format(value)) + raise _Abort + return definition + + def _validate_logical(self, rule, field, value): + """ {'allowed': ('allof', 'anyof', 'noneof', 'oneof')} """ + if not isinstance(value, Sequence): + self._error(field, errors.BAD_TYPE) + return + + validator = self._get_child_validator( + document_crumb=rule, + allow_unknown=False, + schema=self.target_validator.validation_rules, + ) + + for constraints in value: + _hash = ( + mapping_hash({'turing': constraints}), + mapping_hash(self.target_validator.types_mapping), + ) + if _hash in self.target_validator._valid_schemas: + continue + + validator(constraints, normalize=False) + if validator._errors: + self._error(validator._errors) + else: + self.target_validator._valid_schemas.add(_hash) + #### diff --git a/pipenv/vendor/cerberus/tests/__init__.py b/pipenv/vendor/cerberus/tests/__init__.py index cc1c27dc..c014f3b1 100644 --- a/pipenv/vendor/cerberus/tests/__init__.py +++ b/pipenv/vendor/cerberus/tests/__init__.py @@ -1,22 +1,23 @@ # -*- coding: utf-8 -*- +import re + import pytest from cerberus import errors, Validator, SchemaError, DocumentError from cerberus.tests.conftest import sample_schema -def assert_exception(exception, document={}, schema=None, validator=None, - msg=None): +def assert_exception(exception, document={}, schema=None, validator=None, msg=None): """ Tests whether a specific exception is raised. Optionally also tests whether the exception message is as expected. """ if validator is None: validator = Validator() if msg is None: - with pytest.raises(exception) as excinfo: + with pytest.raises(exception): validator(document, schema) else: - with pytest.raises(exception, message=msg) as excinfo: # noqa: F841 + with pytest.raises(exception, match=re.escape(msg)): validator(document, schema) @@ -32,8 +33,15 @@ def assert_document_error(*args): assert_exception(DocumentError, *args) -def assert_fail(document, schema=None, validator=None, update=False, - error=None, errors=None, child_errors=None): +def assert_fail( + document, + schema=None, + validator=None, + update=False, + error=None, + errors=None, + child_errors=None, +): """ Tests whether a validation fails. """ if validator is None: validator = Validator(sample_schema) @@ -45,8 +53,7 @@ def assert_fail(document, schema=None, validator=None, update=False, assert not (error is not None and errors is not None) assert not (errors is not None and child_errors is not None), ( - 'child_errors can only be tested in ' - 'conjunction with the error parameter' + 'child_errors can only be tested in ' 'conjunction with the error parameter' ) assert not (child_errors is not None and error is None) if error is not None: @@ -99,7 +106,8 @@ def assert_has_error(_errors, d_path, s_path, error_def, constraint, info=()): else: break else: - raise AssertionError(""" + raise AssertionError( + """ Error with properties: document_path={doc_path} schema_path={schema_path} @@ -108,9 +116,15 @@ def assert_has_error(_errors, d_path, s_path, error_def, constraint, info=()): info={info} not found in errors: {errors} - """.format(doc_path=d_path, schema_path=s_path, - code=hex(error.code), info=info, - constraint=constraint, errors=_errors)) + """.format( + doc_path=d_path, + schema_path=s_path, + code=hex(error.code), + info=info, + constraint=constraint, + errors=_errors, + ) + ) return i @@ -133,8 +147,9 @@ def assert_not_has_error(_errors, *args, **kwargs): def assert_bad_type(field, data_type, value): - assert_fail({field: value}, - error=(field, (field, 'type'), errors.BAD_TYPE, data_type)) + assert_fail( + {field: value}, error=(field, (field, 'type'), errors.BAD_TYPE, data_type) + ) def assert_normalized(document, expected, schema=None, validator=None): diff --git a/pipenv/vendor/cerberus/tests/conftest.py b/pipenv/vendor/cerberus/tests/conftest.py index 3b4395ea..776c97bc 100644 --- a/pipenv/vendor/cerberus/tests/conftest.py +++ b/pipenv/vendor/cerberus/tests/conftest.py @@ -23,67 +23,27 @@ def validator(): sample_schema = { - 'a_string': { - 'type': 'string', - 'minlength': 2, - 'maxlength': 10 - }, - 'a_binary': { - 'type': 'binary', - 'minlength': 2, - 'maxlength': 10 - }, - 'a_nullable_integer': { - 'type': 'integer', - 'nullable': True - }, - 'an_integer': { - 'type': 'integer', - 'min': 1, - 'max': 100, - }, - 'a_restricted_integer': { - 'type': 'integer', - 'allowed': [-1, 0, 1], - }, - 'a_boolean': { - 'type': 'boolean', - }, - 'a_datetime': { - 'type': 'datetime', - }, - 'a_float': { - 'type': 'float', - 'min': 1, - 'max': 100, - }, - 'a_number': { - 'type': 'number', - 'min': 1, - 'max': 100, - }, - 'a_set': { - 'type': 'set', - }, - 'one_or_more_strings': { - 'type': ['string', 'list'], - 'schema': {'type': 'string'} - }, + 'a_string': {'type': 'string', 'minlength': 2, 'maxlength': 10}, + 'a_binary': {'type': 'binary', 'minlength': 2, 'maxlength': 10}, + 'a_nullable_integer': {'type': 'integer', 'nullable': True}, + 'an_integer': {'type': 'integer', 'min': 1, 'max': 100}, + 'a_restricted_integer': {'type': 'integer', 'allowed': [-1, 0, 1]}, + 'a_boolean': {'type': 'boolean', 'meta': 'can haz two distinct states'}, + 'a_datetime': {'type': 'datetime', 'meta': {'format': '%a, %d. %b %Y'}}, + 'a_float': {'type': 'float', 'min': 1, 'max': 100}, + 'a_number': {'type': 'number', 'min': 1, 'max': 100}, + 'a_set': {'type': 'set'}, + 'one_or_more_strings': {'type': ['string', 'list'], 'schema': {'type': 'string'}}, 'a_regex_email': { 'type': 'string', - 'regex': '^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$' + 'regex': r'^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$', }, - 'a_readonly_string': { - 'type': 'string', - 'readonly': True, - }, - 'a_restricted_string': { - 'type': 'string', - 'allowed': ["agent", "client", "vendor"], - }, - 'an_array': { + 'a_readonly_string': {'type': 'string', 'readonly': True}, + 'a_restricted_string': {'type': 'string', 'allowed': ['agent', 'client', 'vendor']}, + 'an_array': {'type': 'list', 'allowed': ['agent', 'client', 'vendor']}, + 'an_array_from_set': { 'type': 'list', - 'allowed': ["agent", "client", "vendor"], + 'allowed': set(['agent', 'client', 'vendor']), }, 'a_list_of_dicts': { 'type': 'list', @@ -97,38 +57,25 @@ sample_schema = { }, 'a_list_of_values': { 'type': 'list', - 'items': [{'type': 'string'}, {'type': 'integer'}, ] - }, - 'a_list_of_integers': { - 'type': 'list', - 'schema': {'type': 'integer'}, + 'items': [{'type': 'string'}, {'type': 'integer'}], }, + 'a_list_of_integers': {'type': 'list', 'schema': {'type': 'integer'}}, 'a_dict': { 'type': 'dict', 'schema': { 'address': {'type': 'string'}, - 'city': {'type': 'string', 'required': True} + 'city': {'type': 'string', 'required': True}, }, }, - 'a_dict_with_valueschema': { - 'type': 'dict', - 'valueschema': {'type': 'integer'} - }, - 'a_dict_with_keyschema': { - 'type': 'dict', - 'keyschema': {'type': 'string', 'regex': '[a-z]+'} - }, + 'a_dict_with_valuesrules': {'type': 'dict', 'valuesrules': {'type': 'integer'}}, 'a_list_length': { 'type': 'list', 'schema': {'type': 'integer'}, 'minlength': 2, 'maxlength': 5, }, - 'a_nullable_field_without_type': { - 'nullable': True - }, - 'a_not_nullable_field_without_type': { - }, + 'a_nullable_field_without_type': {'nullable': True}, + 'a_not_nullable_field_without_type': {}, } sample_document = {'name': 'john doe'} diff --git a/pipenv/vendor/cerberus/tests/test_assorted.py b/pipenv/vendor/cerberus/tests/test_assorted.py index 641adb7e..b84ef810 100644 --- a/pipenv/vendor/cerberus/tests/test_assorted.py +++ b/pipenv/vendor/cerberus/tests/test_assorted.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- from decimal import Decimal +from pkg_resources import Distribution, DistributionNotFound from pytest import mark @@ -8,6 +9,37 @@ from cerberus import TypeDefinition, Validator from cerberus.tests import assert_fail, assert_success from cerberus.utils import validator_factory from cerberus.validator import BareValidator +from cerberus.platform import PYTHON_VERSION + + +if PYTHON_VERSION > 3 and PYTHON_VERSION < 3.4: + from imp import reload +elif PYTHON_VERSION >= 3.4: + from importlib import reload +else: + pass # Python 2.x + + +def test_pkgresources_version(monkeypatch): + def create_fake_distribution(name): + return Distribution(project_name="cerberus", version="1.2.3") + + with monkeypatch.context() as m: + cerberus = __import__("cerberus") + m.setattr("pkg_resources.get_distribution", create_fake_distribution) + reload(cerberus) + assert cerberus.__version__ == "1.2.3" + + +def test_version_not_found(monkeypatch): + def raise_distribution_not_found(name): + raise DistributionNotFound("pkg_resources cannot get distribution") + + with monkeypatch.context() as m: + cerberus = __import__("cerberus") + m.setattr("pkg_resources.get_distribution", raise_distribution_not_found) + reload(cerberus) + assert cerberus.__version__ == "unknown" def test_clear_cache(validator): @@ -23,8 +55,11 @@ def test_docstring(validator): # Test that testing with the sample schema works as expected # as there might be rules with side-effects in it -@mark.parametrize('test,document', ((assert_fail, {'an_integer': 60}), - (assert_success, {'an_integer': 110}))) + +@mark.parametrize( + "test,document", + ((assert_fail, {"an_integer": 60}), (assert_success, {"an_integer": 110})), +) def test_that_test_fails(test, document): try: test(document) @@ -35,42 +70,42 @@ def test_that_test_fails(test, document): def test_dynamic_types(): - decimal_type = TypeDefinition('decimal', (Decimal,), ()) - document = {'measurement': Decimal(0)} - schema = {'measurement': {'type': 'decimal'}} + decimal_type = TypeDefinition("decimal", (Decimal,), ()) + document = {"measurement": Decimal(0)} + schema = {"measurement": {"type": "decimal"}} validator = Validator() - validator.types_mapping['decimal'] = decimal_type + validator.types_mapping["decimal"] = decimal_type assert_success(document, schema, validator) class MyValidator(Validator): types_mapping = Validator.types_mapping.copy() - types_mapping['decimal'] = decimal_type + types_mapping["decimal"] = decimal_type + validator = MyValidator() assert_success(document, schema, validator) def test_mro(): - assert Validator.__mro__ == (Validator, BareValidator, object), \ - Validator.__mro__ + assert Validator.__mro__ == (Validator, BareValidator, object), Validator.__mro__ def test_mixin_init(): class Mixin(object): def __init__(self, *args, **kwargs): - kwargs['test'] = True + kwargs["test"] = True super(Mixin, self).__init__(*args, **kwargs) - MyValidator = validator_factory('MyValidator', Mixin) + MyValidator = validator_factory("MyValidator", Mixin) validator = MyValidator() - assert validator._config['test'] + assert validator._config["test"] def test_sub_init(): class MyValidator(Validator): def __init__(self, *args, **kwargs): - kwargs['test'] = True + kwargs["test"] = True super(MyValidator, self).__init__(*args, **kwargs) validator = MyValidator() - assert validator._config['test'] + assert validator._config["test"] diff --git a/pipenv/vendor/cerberus/tests/test_customization.py b/pipenv/vendor/cerberus/tests/test_customization.py index 6055894d..8bc3f464 100644 --- a/pipenv/vendor/cerberus/tests/test_customization.py +++ b/pipenv/vendor/cerberus/tests/test_customization.py @@ -1,12 +1,13 @@ # -*- coding: utf-8 -*- +from pytest import mark + import cerberus from cerberus.tests import assert_fail, assert_success from cerberus.tests.conftest import sample_schema def test_contextual_data_preservation(): - class InheritedValidator(cerberus.Validator): def __init__(self, *args, **kwargs): if 'working_dir' in kwargs: @@ -18,9 +19,9 @@ def test_contextual_data_preservation(): return True assert 'test' in InheritedValidator.types - v = InheritedValidator({'test': {'type': 'list', - 'schema': {'type': 'test'}}}, - working_dir='/tmp') + v = InheritedValidator( + {'test': {'type': 'list', 'schema': {'type': 'test'}}}, working_dir='/tmp' + ) assert_success({'test': ['foo']}, validator=v) @@ -42,25 +43,47 @@ def test_docstring_parsing(): assert 'bar' in CustomValidator.validation_rules -def test_issue_265(): +# TODO remove 'validator' as rule parameter with the next major release +@mark.parametrize('rule', ('check_with', 'validator')) +def test_check_with_method(rule): + # https://github.com/pyeve/cerberus/issues/265 + class MyValidator(cerberus.Validator): + def _check_with_oddity(self, field, value): + if not value & 1: + self._error(field, "Must be an odd number") + + v = MyValidator(schema={'amount': {rule: 'oddity'}}) + assert_success(document={'amount': 1}, validator=v) + assert_fail( + document={'amount': 2}, + validator=v, + error=('amount', (), cerberus.errors.CUSTOM, None, ('Must be an odd number',)), + ) + + +# TODO remove test with the next major release +@mark.parametrize('rule', ('check_with', 'validator')) +def test_validator_method(rule): class MyValidator(cerberus.Validator): def _validator_oddity(self, field, value): if not value & 1: self._error(field, "Must be an odd number") - v = MyValidator(schema={'amount': {'validator': 'oddity'}}) + v = MyValidator(schema={'amount': {rule: 'oddity'}}) assert_success(document={'amount': 1}, validator=v) - assert_fail(document={'amount': 2}, validator=v, - error=('amount', (), cerberus.errors.CUSTOM, None, - ('Must be an odd number',))) + assert_fail( + document={'amount': 2}, + validator=v, + error=('amount', (), cerberus.errors.CUSTOM, None, ('Must be an odd number',)), + ) def test_schema_validation_can_be_disabled_in_schema_setter(): - class NonvalidatingValidator(cerberus.Validator): """ Skips schema validation to speed up initialization """ + @cerberus.Validator.schema.setter def schema(self, schema): if schema is None: diff --git a/pipenv/vendor/cerberus/tests/test_errors.py b/pipenv/vendor/cerberus/tests/test_errors.py index df33964f..e4d9b37a 100644 --- a/pipenv/vendor/cerberus/tests/test_errors.py +++ b/pipenv/vendor/cerberus/tests/test_errors.py @@ -24,14 +24,14 @@ def test__error_1(): def test__error_2(): - v = Validator(schema={'foo': {'keyschema': {'type': 'integer'}}}) + v = Validator(schema={'foo': {'keysrules': {'type': 'integer'}}}) v.document = {'foo': {'0': 'bar'}} - v._error('foo', errors.KEYSCHEMA, ()) + v._error('foo', errors.KEYSRULES, ()) error = v._errors[0] assert error.document_path == ('foo',) - assert error.schema_path == ('foo', 'keyschema') + assert error.schema_path == ('foo', 'keysrules') assert error.code == 0x83 - assert error.rule == 'keyschema' + assert error.rule == 'keysrules' assert error.constraint == {'type': 'integer'} assert error.value == {'0': 'bar'} assert error.info == ((),) @@ -40,8 +40,10 @@ def test__error_2(): def test__error_3(): - valids = [{'type': 'string', 'regex': '0x[0-9a-f]{2}'}, - {'type': 'integer', 'min': 0, 'max': 255}] + valids = [ + {'type': 'string', 'regex': '0x[0-9a-f]{2}'}, + {'type': 'integer', 'min': 0, 'max': 255}, + ] v = Validator(schema={'foo': {'oneof': valids}}) v.document = {'foo': '0x100'} v._error('foo', errors.ONEOF, (), 0, 2) @@ -77,8 +79,9 @@ def test_error_tree_from_subschema(validator): assert 'bar' in s_error_tree['foo']['schema'] assert 'type' in s_error_tree['foo']['schema']['bar'] assert s_error_tree['foo']['schema']['bar']['type'].errors[0].value == 0 - assert s_error_tree.fetch_errors_from( - ('foo', 'schema', 'bar', 'type'))[0].value == 0 + assert ( + s_error_tree.fetch_errors_from(('foo', 'schema', 'bar', 'type'))[0].value == 0 + ) def test_error_tree_from_anyof(validator): @@ -98,12 +101,17 @@ def test_error_tree_from_anyof(validator): def test_nested_error_paths(validator): - schema = {'a_dict': {'keyschema': {'type': 'integer'}, - 'valueschema': {'regex': '[a-z]*'}}, - 'a_list': {'schema': {'type': 'string', - 'oneof_regex': ['[a-z]*$', '[A-Z]*']}}} - document = {'a_dict': {0: 'abc', 'one': 'abc', 2: 'aBc', 'three': 'abC'}, - 'a_list': [0, 'abc', 'abC']} + schema = { + 'a_dict': { + 'keysrules': {'type': 'integer'}, + 'valuesrules': {'regex': '[a-z]*'}, + }, + 'a_list': {'schema': {'type': 'string', 'oneof_regex': ['[a-z]*$', '[A-Z]*']}}, + } + document = { + 'a_dict': {0: 'abc', 'one': 'abc', 2: 'aBc', 'three': 'abC'}, + 'a_list': [0, 'abc', 'abC'], + } assert_fail(document, schema, validator=validator) _det = validator.document_error_tree @@ -120,35 +128,59 @@ def test_nested_error_paths(validator): assert len(_det['a_dict'][2].errors) == 1 assert len(_det['a_dict']['three'].errors) == 2 - assert len(_set['a_dict']['keyschema'].errors) == 1 - assert len(_set['a_dict']['valueschema'].errors) == 1 + assert len(_set['a_dict']['keysrules'].errors) == 1 + assert len(_set['a_dict']['valuesrules'].errors) == 1 - assert len(_set['a_dict']['keyschema']['type'].errors) == 2 - assert len(_set['a_dict']['valueschema']['regex'].errors) == 2 + assert len(_set['a_dict']['keysrules']['type'].errors) == 2 + assert len(_set['a_dict']['valuesrules']['regex'].errors) == 2 _ref_err = ValidationError( - ('a_dict', 'one'), ('a_dict', 'keyschema', 'type'), - errors.BAD_TYPE.code, 'type', 'integer', 'one', ()) + ('a_dict', 'one'), + ('a_dict', 'keysrules', 'type'), + errors.BAD_TYPE.code, + 'type', + 'integer', + 'one', + (), + ) assert _det['a_dict']['one'].errors[0] == _ref_err - assert _set['a_dict']['keyschema']['type'].errors[0] == _ref_err + assert _set['a_dict']['keysrules']['type'].errors[0] == _ref_err _ref_err = ValidationError( - ('a_dict', 2), ('a_dict', 'valueschema', 'regex'), - errors.REGEX_MISMATCH.code, 'regex', '[a-z]*$', 'aBc', ()) + ('a_dict', 2), + ('a_dict', 'valuesrules', 'regex'), + errors.REGEX_MISMATCH.code, + 'regex', + '[a-z]*$', + 'aBc', + (), + ) assert _det['a_dict'][2].errors[0] == _ref_err - assert _set['a_dict']['valueschema']['regex'].errors[0] == _ref_err + assert _set['a_dict']['valuesrules']['regex'].errors[0] == _ref_err _ref_err = ValidationError( - ('a_dict', 'three'), ('a_dict', 'keyschema', 'type'), - errors.BAD_TYPE.code, 'type', 'integer', 'three', ()) + ('a_dict', 'three'), + ('a_dict', 'keysrules', 'type'), + errors.BAD_TYPE.code, + 'type', + 'integer', + 'three', + (), + ) assert _det['a_dict']['three'].errors[0] == _ref_err - assert _set['a_dict']['keyschema']['type'].errors[1] == _ref_err + assert _set['a_dict']['keysrules']['type'].errors[1] == _ref_err _ref_err = ValidationError( - ('a_dict', 'three'), ('a_dict', 'valueschema', 'regex'), - errors.REGEX_MISMATCH.code, 'regex', '[a-z]*$', 'abC', ()) + ('a_dict', 'three'), + ('a_dict', 'valuesrules', 'regex'), + errors.REGEX_MISMATCH.code, + 'regex', + '[a-z]*$', + 'abC', + (), + ) assert _det['a_dict']['three'].errors[1] == _ref_err - assert _set['a_dict']['valueschema']['regex'].errors[1] == _ref_err + assert _set['a_dict']['valuesrules']['regex'].errors[1] == _ref_err assert len(_det['a_list'].errors) == 1 assert len(_det['a_list'][0].errors) == 1 @@ -161,34 +193,56 @@ def test_nested_error_paths(validator): assert len(_set['a_list']['schema']['oneof'][1]['regex'].errors) == 1 _ref_err = ValidationError( - ('a_list', 0), ('a_list', 'schema', 'type'), errors.BAD_TYPE.code, - 'type', 'string', 0, ()) + ('a_list', 0), + ('a_list', 'schema', 'type'), + errors.BAD_TYPE.code, + 'type', + 'string', + 0, + (), + ) assert _det['a_list'][0].errors[0] == _ref_err assert _set['a_list']['schema']['type'].errors[0] == _ref_err _ref_err = ValidationError( - ('a_list', 2), ('a_list', 'schema', 'oneof'), errors.ONEOF.code, - 'oneof', 'irrelevant_at_this_point', 'abC', ()) + ('a_list', 2), + ('a_list', 'schema', 'oneof'), + errors.ONEOF.code, + 'oneof', + 'irrelevant_at_this_point', + 'abC', + (), + ) assert _det['a_list'][2].errors[0] == _ref_err assert _set['a_list']['schema']['oneof'].errors[0] == _ref_err _ref_err = ValidationError( - ('a_list', 2), ('a_list', 'schema', 'oneof', 0, 'regex'), - errors.REGEX_MISMATCH.code, 'regex', '[a-z]*$', 'abC', ()) + ('a_list', 2), + ('a_list', 'schema', 'oneof', 0, 'regex'), + errors.REGEX_MISMATCH.code, + 'regex', + '[a-z]*$', + 'abC', + (), + ) assert _det['a_list'][2].errors[1] == _ref_err assert _set['a_list']['schema']['oneof'][0]['regex'].errors[0] == _ref_err _ref_err = ValidationError( - ('a_list', 2), ('a_list', 'schema', 'oneof', 1, 'regex'), - errors.REGEX_MISMATCH.code, 'regex', '[a-z]*$', 'abC', ()) + ('a_list', 2), + ('a_list', 'schema', 'oneof', 1, 'regex'), + errors.REGEX_MISMATCH.code, + 'regex', + '[a-z]*$', + 'abC', + (), + ) assert _det['a_list'][2].errors[2] == _ref_err assert _set['a_list']['schema']['oneof'][1]['regex'].errors[0] == _ref_err def test_queries(): - schema = {'foo': {'type': 'dict', - 'schema': - {'bar': {'type': 'number'}}}} + schema = {'foo': {'type': 'dict', 'schema': {'bar': {'type': 'number'}}}} document = {'foo': {'bar': 'zero'}} validator = Validator(schema) validator(document) @@ -202,59 +256,68 @@ def test_queries(): assert errors.MAPPING_SCHEMA in validator.document_error_tree['foo'] assert errors.BAD_TYPE in validator.document_error_tree['foo']['bar'] assert errors.MAPPING_SCHEMA in validator.schema_error_tree['foo']['schema'] - assert errors.BAD_TYPE in \ - validator.schema_error_tree['foo']['schema']['bar']['type'] + assert ( + errors.BAD_TYPE in validator.schema_error_tree['foo']['schema']['bar']['type'] + ) - assert (validator.document_error_tree['foo'][errors.MAPPING_SCHEMA] - .child_errors[0].code == errors.BAD_TYPE.code) + assert ( + validator.document_error_tree['foo'][errors.MAPPING_SCHEMA].child_errors[0].code + == errors.BAD_TYPE.code + ) def test_basic_error_handler(): handler = errors.BasicErrorHandler() _errors, ref = [], {} - _errors.append(ValidationError( - ['foo'], ['foo'], 0x63, 'readonly', True, None, ())) + _errors.append(ValidationError(['foo'], ['foo'], 0x63, 'readonly', True, None, ())) ref.update({'foo': [handler.messages[0x63]]}) assert handler(_errors) == ref - _errors.append(ValidationError( - ['bar'], ['foo'], 0x42, 'min', 1, 2, ())) + _errors.append(ValidationError(['bar'], ['foo'], 0x42, 'min', 1, 2, ())) ref.update({'bar': [handler.messages[0x42].format(constraint=1)]}) assert handler(_errors) == ref - _errors.append(ValidationError( - ['zap', 'foo'], ['zap', 'schema', 'foo'], 0x24, 'type', 'string', - True, ())) - ref.update({'zap': [{'foo': [handler.messages[0x24].format( - constraint='string')]}]}) + _errors.append( + ValidationError( + ['zap', 'foo'], ['zap', 'schema', 'foo'], 0x24, 'type', 'string', True, () + ) + ) + ref.update({'zap': [{'foo': [handler.messages[0x24].format(constraint='string')]}]}) assert handler(_errors) == ref - _errors.append(ValidationError( - ['zap', 'foo'], ['zap', 'schema', 'foo'], 0x41, 'regex', - '^p[äe]ng$', 'boom', ())) - ref['zap'][0]['foo'].append( - handler.messages[0x41].format(constraint='^p[äe]ng$')) + _errors.append( + ValidationError( + ['zap', 'foo'], + ['zap', 'schema', 'foo'], + 0x41, + 'regex', + '^p[äe]ng$', + 'boom', + (), + ) + ) + ref['zap'][0]['foo'].append(handler.messages[0x41].format(constraint='^p[äe]ng$')) assert handler(_errors) == ref def test_basic_error_of_errors(validator): - schema = {'foo': {'oneof': [ - {'type': 'integer'}, - {'type': 'string'} - ]}} + schema = {'foo': {'oneof': [{'type': 'integer'}, {'type': 'string'}]}} document = {'foo': 23.42} - error = ('foo', ('foo', 'oneof'), errors.ONEOF, - schema['foo']['oneof'], ()) + error = ('foo', ('foo', 'oneof'), errors.ONEOF, schema['foo']['oneof'], ()) child_errors = [ (error[0], error[1] + (0, 'type'), errors.BAD_TYPE, 'integer'), - (error[0], error[1] + (1, 'type'), errors.BAD_TYPE, 'string') + (error[0], error[1] + (1, 'type'), errors.BAD_TYPE, 'string'), ] - assert_fail(document, schema, validator=validator, - error=error, child_errors=child_errors) + assert_fail( + document, schema, validator=validator, error=error, child_errors=child_errors + ) assert validator.errors == { - 'foo': [errors.BasicErrorHandler.messages[0x92], - {'oneof definition 0': ['must be of integer type'], - 'oneof definition 1': ['must be of string type']} - ] + 'foo': [ + errors.BasicErrorHandler.messages[0x92], + { + 'oneof definition 0': ['must be of integer type'], + 'oneof definition 1': ['must be of string type'], + }, + ] } diff --git a/pipenv/vendor/cerberus/tests/test_normalization.py b/pipenv/vendor/cerberus/tests/test_normalization.py index 6e06f553..adc281ef 100644 --- a/pipenv/vendor/cerberus/tests/test_normalization.py +++ b/pipenv/vendor/cerberus/tests/test_normalization.py @@ -1,10 +1,21 @@ # -*- coding: utf-8 -*- +from copy import deepcopy from tempfile import NamedTemporaryFile +from pytest import mark + from cerberus import Validator, errors -from cerberus.tests import (assert_fail, assert_has_error, assert_normalized, - assert_success) +from cerberus.tests import ( + assert_fail, + assert_has_error, + assert_normalized, + assert_success, +) + + +def must_not_be_called(*args, **kwargs): + raise RuntimeError('This shall not be called.') def test_coerce(): @@ -15,21 +26,31 @@ def test_coerce(): def test_coerce_in_dictschema(): - schema = {'thing': {'type': 'dict', - 'schema': {'amount': {'coerce': int}}}} + schema = {'thing': {'type': 'dict', 'schema': {'amount': {'coerce': int}}}} document = {'thing': {'amount': '2'}} expected = {'thing': {'amount': 2}} assert_normalized(document, expected, schema) def test_coerce_in_listschema(): - schema = {'things': {'type': 'list', - 'schema': {'coerce': int}}} + schema = {'things': {'type': 'list', 'schema': {'coerce': int}}} document = {'things': ['1', '2', '3']} expected = {'things': [1, 2, 3]} assert_normalized(document, expected, schema) +def test_coerce_in_listitems(): + schema = {'things': {'type': 'list', 'items': [{'coerce': int}, {'coerce': str}]}} + document = {'things': ['1', 2]} + expected = {'things': [1, '2']} + assert_normalized(document, expected, schema) + + validator = Validator(schema) + document['things'].append(3) + assert not validator(document) + assert validator.document['things'] == document['things'] + + def test_coerce_in_dictschema_in_listschema(): item_schema = {'type': 'dict', 'schema': {'amount': {'coerce': int}}} schema = {'things': {'type': 'list', 'schema': item_schema}} @@ -39,9 +60,7 @@ def test_coerce_in_dictschema_in_listschema(): def test_coerce_not_destructive(): - schema = { - 'amount': {'coerce': int} - } + schema = {'amount': {'coerce': int}} v = Validator(schema) doc = {'amount': '1'} v.validate(doc) @@ -52,16 +71,48 @@ def test_coerce_catches_ValueError(): schema = {'amount': {'coerce': int}} _errors = assert_fail({'amount': 'not_a_number'}, schema) _errors[0].info = () # ignore exception message here - assert_has_error(_errors, 'amount', ('amount', 'coerce'), - errors.COERCION_FAILED, int) + assert_has_error( + _errors, 'amount', ('amount', 'coerce'), errors.COERCION_FAILED, int + ) + + +def test_coerce_in_listitems_catches_ValueError(): + schema = {'things': {'type': 'list', 'items': [{'coerce': int}, {'coerce': str}]}} + document = {'things': ['not_a_number', 2]} + _errors = assert_fail(document, schema) + _errors[0].info = () # ignore exception message here + assert_has_error( + _errors, + ('things', 0), + ('things', 'items', 'coerce'), + errors.COERCION_FAILED, + int, + ) def test_coerce_catches_TypeError(): schema = {'name': {'coerce': str.lower}} _errors = assert_fail({'name': 1234}, schema) _errors[0].info = () # ignore exception message here - assert_has_error(_errors, 'name', ('name', 'coerce'), - errors.COERCION_FAILED, str.lower) + assert_has_error( + _errors, 'name', ('name', 'coerce'), errors.COERCION_FAILED, str.lower + ) + + +def test_coerce_in_listitems_catches_TypeError(): + schema = { + 'things': {'type': 'list', 'items': [{'coerce': int}, {'coerce': str.lower}]} + } + document = {'things': ['1', 2]} + _errors = assert_fail(document, schema) + _errors[0].info = () # ignore exception message here + assert_has_error( + _errors, + ('things', 1), + ('things', 'items', 'coerce'), + errors.COERCION_FAILED, + str.lower, + ) def test_coerce_unknown(): @@ -88,16 +139,16 @@ def test_custom_coerce_and_rename(): def test_coerce_chain(): - drop_prefix = lambda x: x[2:] - upper = lambda x: x.upper() + drop_prefix = lambda x: x[2:] # noqa: E731 + upper = lambda x: x.upper() # noqa: E731 schema = {'foo': {'coerce': [hex, drop_prefix, upper]}} assert_normalized({'foo': 15}, {'foo': 'F'}, schema) def test_coerce_chain_aborts(validator): def dont_do_me(value): - raise AssertionError('The coercion chain did not abort after an ' - 'error.') + raise AssertionError('The coercion chain did not abort after an ' 'error.') + schema = {'foo': {'coerce': [hex, dont_do_me]}} validator({'foo': '0'}, schema) assert errors.COERCION_FAILED in validator._errors @@ -105,12 +156,12 @@ def test_coerce_chain_aborts(validator): def test_coerce_non_digit_in_sequence(validator): # https://github.com/pyeve/cerberus/issues/211 - schema = {'data': {'type': 'list', - 'schema': {'type': 'integer', 'coerce': int}}} + schema = {'data': {'type': 'list', 'schema': {'type': 'integer', 'coerce': int}}} document = {'data': ['q']} assert validator.validated(document, schema) is None - assert (validator.validated(document, schema, always_return_document=True) - == document) # noqa: W503 + assert ( + validator.validated(document, schema, always_return_document=True) == document + ) # noqa: W503 def test_nullables_dont_fail_coerce(): @@ -119,6 +170,18 @@ def test_nullables_dont_fail_coerce(): assert_normalized(document, document, schema) +def test_nullables_fail_coerce_on_non_null_values(validator): + def failing_coercion(value): + raise Exception("expected to fail") + + schema = {'foo': {'coerce': failing_coercion, 'nullable': True, 'type': 'integer'}} + document = {'foo': None} + assert_normalized(document, document, schema) + + validator({'foo': 2}, schema) + assert errors.COERCION_FAILED in validator._errors + + def test_normalized(): schema = {'amount': {'coerce': int}} document = {'amount': '2'} @@ -154,9 +217,13 @@ def test_purge_unknown(): def test_purge_unknown_in_subschema(): - schema = {'foo': {'type': 'dict', - 'schema': {'foo': {'type': 'string'}}, - 'purge_unknown': True}} + schema = { + 'foo': { + 'type': 'dict', + 'schema': {'foo': {'type': 'string'}}, + 'purge_unknown': True, + } + } document = {'foo': {'bar': ''}} expected = {'foo': {}} assert_normalized(document, expected, schema) @@ -175,8 +242,7 @@ def test_issue_147_complex(): def test_issue_147_nested_dict(): - schema = {'thing': {'type': 'dict', - 'schema': {'amount': {'coerce': int}}}} + schema = {'thing': {'type': 'dict', 'schema': {'amount': {'coerce': int}}}} ref_obj = '2' document = {'thing': {'amount': ref_obj}} normalized = Validator(schema).normalized(document) @@ -186,20 +252,21 @@ def test_issue_147_nested_dict(): assert document['thing']['amount'] is ref_obj -def test_coerce_in_valueschema(): +def test_coerce_in_valuesrules(): # https://github.com/pyeve/cerberus/issues/155 - schema = {'thing': {'type': 'dict', - 'valueschema': {'coerce': int, - 'type': 'integer'}}} + schema = { + 'thing': {'type': 'dict', 'valuesrules': {'coerce': int, 'type': 'integer'}} + } document = {'thing': {'amount': '2'}} expected = {'thing': {'amount': 2}} assert_normalized(document, expected, schema) -def test_coerce_in_keyschema(): +def test_coerce_in_keysrules(): # https://github.com/pyeve/cerberus/issues/155 - schema = {'thing': {'type': 'dict', - 'keyschema': {'coerce': int, 'type': 'integer'}}} + schema = { + 'thing': {'type': 'dict', 'keysrules': {'coerce': int, 'type': 'integer'}} + } document = {'thing': {'5': 'foo'}} expected = {'thing': {5: 'foo'}} assert_normalized(document, expected, schema) @@ -207,8 +274,7 @@ def test_coerce_in_keyschema(): def test_coercion_of_sequence_items(validator): # https://github.com/pyeve/cerberus/issues/161 - schema = {'a_list': {'type': 'list', 'schema': {'type': 'float', - 'coerce': float}}} + schema = {'a_list': {'type': 'list', 'schema': {'type': 'float', 'coerce': float}}} document = {'a_list': [3, 4, 5]} expected = {'a_list': [3.0, 4.0, 5.0]} assert_normalized(document, expected, schema, validator) @@ -216,110 +282,76 @@ def test_coercion_of_sequence_items(validator): assert isinstance(x, float) -def test_default_missing(): - _test_default_missing({'default': 'bar_value'}) - - -def test_default_setter_missing(): - _test_default_missing({'default_setter': lambda doc: 'bar_value'}) - - -def _test_default_missing(default): +@mark.parametrize( + 'default', ({'default': 'bar_value'}, {'default_setter': lambda doc: 'bar_value'}) +) +def test_default_missing(default): bar_schema = {'type': 'string'} bar_schema.update(default) - schema = {'foo': {'type': 'string'}, - 'bar': bar_schema} + schema = {'foo': {'type': 'string'}, 'bar': bar_schema} document = {'foo': 'foo_value'} expected = {'foo': 'foo_value', 'bar': 'bar_value'} assert_normalized(document, expected, schema) -def test_default_existent(): - _test_default_existent({'default': 'bar_value'}) - - -def test_default_setter_existent(): - def raise_error(doc): - raise RuntimeError('should not be called') - _test_default_existent({'default_setter': raise_error}) - - -def _test_default_existent(default): +@mark.parametrize( + 'default', ({'default': 'bar_value'}, {'default_setter': must_not_be_called}) +) +def test_default_existent(default): bar_schema = {'type': 'string'} bar_schema.update(default) - schema = {'foo': {'type': 'string'}, - 'bar': bar_schema} + schema = {'foo': {'type': 'string'}, 'bar': bar_schema} document = {'foo': 'foo_value', 'bar': 'non_default'} assert_normalized(document, document.copy(), schema) -def test_default_none_nullable(): - _test_default_none_nullable({'default': 'bar_value'}) - - -def test_default_setter_none_nullable(): - def raise_error(doc): - raise RuntimeError('should not be called') - _test_default_none_nullable({'default_setter': raise_error}) - - -def _test_default_none_nullable(default): - bar_schema = {'type': 'string', - 'nullable': True} +@mark.parametrize( + 'default', ({'default': 'bar_value'}, {'default_setter': must_not_be_called}) +) +def test_default_none_nullable(default): + bar_schema = {'type': 'string', 'nullable': True} bar_schema.update(default) - schema = {'foo': {'type': 'string'}, - 'bar': bar_schema} + schema = {'foo': {'type': 'string'}, 'bar': bar_schema} document = {'foo': 'foo_value', 'bar': None} assert_normalized(document, document.copy(), schema) -def test_default_none_nonnullable(): - _test_default_none_nullable({'default': 'bar_value'}) - - -def test_default_setter_none_nonnullable(): - _test_default_none_nullable( - {'default_setter': lambda doc: 'bar_value'}) - - -def _test_default_none_nonnullable(default): - bar_schema = {'type': 'string', - 'nullable': False} +@mark.parametrize( + 'default', ({'default': 'bar_value'}, {'default_setter': lambda doc: 'bar_value'}) +) +def test_default_none_nonnullable(default): + bar_schema = {'type': 'string', 'nullable': False} bar_schema.update(default) - schema = {'foo': {'type': 'string'}, - 'bar': bar_schema} - document = {'foo': 'foo_value', 'bar': 'bar_value'} - assert_normalized(document, document.copy(), schema) + schema = {'foo': {'type': 'string'}, 'bar': bar_schema} + document = {'foo': 'foo_value', 'bar': None} + expected = {'foo': 'foo_value', 'bar': 'bar_value'} + assert_normalized(document, expected, schema) def test_default_none_default_value(): - schema = {'foo': {'type': 'string'}, - 'bar': {'type': 'string', - 'nullable': True, - 'default': None}} + schema = { + 'foo': {'type': 'string'}, + 'bar': {'type': 'string', 'nullable': True, 'default': None}, + } document = {'foo': 'foo_value'} expected = {'foo': 'foo_value', 'bar': None} assert_normalized(document, expected, schema) -def test_default_missing_in_subschema(): - _test_default_missing_in_subschema({'default': 'bar_value'}) - - -def test_default_setter_missing_in_subschema(): - _test_default_missing_in_subschema( - {'default_setter': lambda doc: 'bar_value'}) - - -def _test_default_missing_in_subschema(default): +@mark.parametrize( + 'default', ({'default': 'bar_value'}, {'default_setter': lambda doc: 'bar_value'}) +) +def test_default_missing_in_subschema(default): bar_schema = {'type': 'string'} bar_schema.update(default) - schema = {'thing': {'type': 'dict', - 'schema': {'foo': {'type': 'string'}, - 'bar': bar_schema}}} + schema = { + 'thing': { + 'type': 'dict', + 'schema': {'foo': {'type': 'string'}, 'bar': bar_schema}, + } + } document = {'thing': {'foo': 'foo_value'}} - expected = {'thing': {'foo': 'foo_value', - 'bar': 'bar_value'}} + expected = {'thing': {'foo': 'foo_value', 'bar': 'bar_value'}} assert_normalized(document, expected, schema) @@ -328,8 +360,7 @@ def test_depending_default_setters(): 'a': {'type': 'integer'}, 'b': {'type': 'integer', 'default_setter': lambda d: d['a'] + 1}, 'c': {'type': 'integer', 'default_setter': lambda d: d['b'] * 2}, - 'd': {'type': 'integer', - 'default_setter': lambda d: d['b'] + d['c']} + 'd': {'type': 'integer', 'default_setter': lambda d: d['b'] + d['c']}, } document = {'a': 1} expected = {'a': 1, 'b': 2, 'c': 4, 'd': 6} @@ -339,7 +370,7 @@ def test_depending_default_setters(): def test_circular_depending_default_setters(validator): schema = { 'a': {'type': 'integer', 'default_setter': lambda d: d['b'] + 1}, - 'b': {'type': 'integer', 'default_setter': lambda d: d['a'] + 1} + 'b': {'type': 'integer', 'default_setter': lambda d: d['a'] + 1}, } validator({}, schema) assert errors.SETTING_DEFAULT_FAILED in validator._errors @@ -353,14 +384,16 @@ def test_issue_250(): 'schema': { 'type': 'dict', 'allow_unknown': True, - 'schema': {'a': {'type': 'string'}} - } + 'schema': {'a': {'type': 'string'}}, + }, } } document = {'list': {'is_a': 'mapping'}} - assert_fail(document, schema, - error=('list', ('list', 'type'), errors.BAD_TYPE, - schema['list']['type'])) + assert_fail( + document, + schema, + error=('list', ('list', 'type'), errors.BAD_TYPE, schema['list']['type']), + ) def test_issue_250_no_type_pass_on_list(): @@ -370,7 +403,7 @@ def test_issue_250_no_type_pass_on_list(): 'schema': { 'allow_unknown': True, 'type': 'dict', - 'schema': {'a': {'type': 'string'}} + 'schema': {'a': {'type': 'string'}}, } } } @@ -381,28 +414,25 @@ def test_issue_250_no_type_pass_on_list(): def test_issue_250_no_type_fail_on_dict(): # https://github.com/pyeve/cerberus/issues/250 schema = { - 'list': { - 'schema': { - 'allow_unknown': True, - 'schema': {'a': {'type': 'string'}} - } - } + 'list': {'schema': {'allow_unknown': True, 'schema': {'a': {'type': 'string'}}}} } document = {'list': {'a': {'a': 'known'}}} - assert_fail(document, schema, - error=('list', ('list', 'schema'), errors.BAD_TYPE_FOR_SCHEMA, - schema['list']['schema'])) + assert_fail( + document, + schema, + error=( + 'list', + ('list', 'schema'), + errors.BAD_TYPE_FOR_SCHEMA, + schema['list']['schema'], + ), + ) def test_issue_250_no_type_fail_pass_on_other(): # https://github.com/pyeve/cerberus/issues/250 schema = { - 'list': { - 'schema': { - 'allow_unknown': True, - 'schema': {'a': {'type': 'string'}} - } - } + 'list': {'schema': {'allow_unknown': True, 'schema': {'a': {'type': 'string'}}}} } document = {'list': 1} assert_normalized(document, document, schema) @@ -416,21 +446,20 @@ def test_allow_unknown_with_of_rules(): { 'type': 'dict', 'allow_unknown': True, - 'schema': {'known': {'type': 'string'}} - }, - { - 'type': 'dict', - 'schema': {'known': {'type': 'string'}} + 'schema': {'known': {'type': 'string'}}, }, + {'type': 'dict', 'schema': {'known': {'type': 'string'}}}, ] } } # check regression and that allow unknown does not cause any different # than expected behaviour for one-of. document = {'test': {'known': 's'}} - assert_fail(document, schema, - error=('test', ('test', 'oneof'), - errors.ONEOF, schema['test']['oneof'])) + assert_fail( + document, + schema, + error=('test', ('test', 'oneof'), errors.ONEOF, schema['test']['oneof']), + ) # check that allow_unknown is actually applied document = {'test': {'known': 's', 'unknown': 'asd'}} assert_success(document, schema) @@ -439,18 +468,20 @@ def test_allow_unknown_with_of_rules(): def test_271_normalising_tuples(): # https://github.com/pyeve/cerberus/issues/271 schema = { - 'my_field': { - 'type': 'list', - 'schema': {'type': ('string', 'number', 'dict')} - } + 'my_field': {'type': 'list', 'schema': {'type': ('string', 'number', 'dict')}} } - document = {'my_field': ('foo', 'bar', 42, 'albert', - 'kandinsky', {'items': 23})} + document = {'my_field': ('foo', 'bar', 42, 'albert', 'kandinsky', {'items': 23})} assert_success(document, schema) normalized = Validator(schema).normalized(document) - assert normalized['my_field'] == ('foo', 'bar', 42, 'albert', - 'kandinsky', {'items': 23}) + assert normalized['my_field'] == ( + 'foo', + 'bar', + 42, + 'albert', + 'kandinsky', + {'items': 23}, + ) def test_allow_unknown_wo_schema(): @@ -472,14 +503,41 @@ def test_allow_unknown_with_purge_unknown_subdocument(): schema = { 'foo': { 'type': 'dict', - 'schema': { - 'bar': { - 'type': 'string' - } - }, - 'allow_unknown': True + 'schema': {'bar': {'type': 'string'}}, + 'allow_unknown': True, } } document = {'foo': {'bar': 'baz', 'corge': False}, 'thud': 'xyzzy'} expected = {'foo': {'bar': 'baz', 'corge': False}} assert_normalized(document, expected, schema, validator) + + +def test_purge_readonly(): + schema = { + 'description': {'type': 'string', 'maxlength': 500}, + 'last_updated': {'readonly': True}, + } + validator = Validator(schema=schema, purge_readonly=True) + document = {'description': 'it is a thing'} + expected = deepcopy(document) + document['last_updated'] = 'future' + assert_normalized(document, expected, validator=validator) + + +def test_defaults_in_allow_unknown_schema(): + schema = {'meta': {'type': 'dict'}, 'version': {'type': 'string'}} + allow_unknown = { + 'type': 'dict', + 'schema': { + 'cfg_path': {'type': 'string', 'default': 'cfg.yaml'}, + 'package': {'type': 'string'}, + }, + } + validator = Validator(schema=schema, allow_unknown=allow_unknown) + + document = {'version': '1.2.3', 'plugin_foo': {'package': 'foo'}} + expected = { + 'version': '1.2.3', + 'plugin_foo': {'package': 'foo', 'cfg_path': 'cfg.yaml'}, + } + assert_normalized(document, expected, schema, validator) diff --git a/pipenv/vendor/cerberus/tests/test_registries.py b/pipenv/vendor/cerberus/tests/test_registries.py index 05f01c52..b628952d 100644 --- a/pipenv/vendor/cerberus/tests/test_registries.py +++ b/pipenv/vendor/cerberus/tests/test_registries.py @@ -1,14 +1,17 @@ # -*- coding: utf-8 -*- from cerberus import schema_registry, rules_set_registry, Validator -from cerberus.tests import (assert_fail, assert_normalized, - assert_schema_error, assert_success) +from cerberus.tests import ( + assert_fail, + assert_normalized, + assert_schema_error, + assert_success, +) def test_schema_registry_simple(): schema_registry.add('foo', {'bar': {'type': 'string'}}) - schema = {'a': {'schema': 'foo'}, - 'b': {'schema': 'foo'}} + schema = {'a': {'schema': 'foo'}, 'b': {'schema': 'foo'}} document = {'a': {'bar': 'a'}, 'b': {'bar': 'b'}} assert_success(document, schema) @@ -33,23 +36,22 @@ def test_allow_unknown_as_reference(): def test_recursion(): - rules_set_registry.add('self', - {'type': 'dict', 'allow_unknown': 'self'}) + rules_set_registry.add('self', {'type': 'dict', 'allow_unknown': 'self'}) v = Validator(allow_unknown='self') assert_success({0: {1: {2: {}}}}, {}, v) def test_references_remain_unresolved(validator): - rules_set_registry.extend((('boolean', {'type': 'boolean'}), - ('booleans', {'valueschema': 'boolean'}))) + rules_set_registry.extend( + (('boolean', {'type': 'boolean'}), ('booleans', {'valuesrules': 'boolean'})) + ) validator.schema = {'foo': 'booleans'} assert 'booleans' == validator.schema['foo'] - assert 'boolean' == rules_set_registry._storage['booleans']['valueschema'] + assert 'boolean' == rules_set_registry._storage['booleans']['valuesrules'] def test_rules_registry_with_anyof_type(): - rules_set_registry.add('string_or_integer', - {'anyof_type': ['string', 'integer']}) + rules_set_registry.add('string_or_integer', {'anyof_type': ['string', 'integer']}) schema = {'soi': 'string_or_integer'} assert_success({'soi': 'hello'}, schema) diff --git a/pipenv/vendor/cerberus/tests/test_schema.py b/pipenv/vendor/cerberus/tests/test_schema.py index 1776cae3..84e50946 100644 --- a/pipenv/vendor/cerberus/tests/test_schema.py +++ b/pipenv/vendor/cerberus/tests/test_schema.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- +import re + import pytest from cerberus import Validator, errors, SchemaError @@ -9,14 +11,14 @@ from cerberus.tests import assert_schema_error def test_empty_schema(): validator = Validator() - with pytest.raises(SchemaError, message=errors.SCHEMA_ERROR_MISSING): + with pytest.raises(SchemaError, match=errors.SCHEMA_ERROR_MISSING): validator({}, schema=None) def test_bad_schema_type(validator): schema = "this string should really be dict" - exp_msg = errors.SCHEMA_ERROR_DEFINITION_TYPE.format(schema) - with pytest.raises(SchemaError, message=exp_msg): + msg = errors.SCHEMA_ERROR_DEFINITION_TYPE.format(schema) + with pytest.raises(SchemaError, match=msg): validator.schema = schema @@ -28,23 +30,21 @@ def test_bad_schema_type_field(validator): def test_unknown_rule(validator): - message = "{'foo': [{'unknown': ['unknown rule']}]}" - with pytest.raises(SchemaError, message=message): + msg = "{'foo': [{'unknown': ['unknown rule']}]}" + with pytest.raises(SchemaError, match=re.escape(msg)): validator.schema = {'foo': {'unknown': 'rule'}} def test_unknown_type(validator): - field = 'name' - value = 'catch_me' - message = str({field: [{'type': ['unallowed value %s' % value]}]}) - with pytest.raises(SchemaError, message=message): - validator.schema = {'foo': {'unknown': 'rule'}} + msg = str({'foo': [{'type': ['Unsupported types: unknown']}]}) + with pytest.raises(SchemaError, match=re.escape(msg)): + validator.schema = {'foo': {'type': 'unknown'}} def test_bad_schema_definition(validator): field = 'name' - message = str({field: ['must be of dict type']}) - with pytest.raises(SchemaError, message=message): + msg = str({field: ['must be of dict type']}) + with pytest.raises(SchemaError, match=re.escape(msg)): validator.schema = {field: 'this should really be a dict'} @@ -61,14 +61,14 @@ def test_normalization_rules_are_invalid_in_of_rules(): def test_anyof_allof_schema_validate(): # make sure schema with 'anyof' and 'allof' constraints are checked # correctly - schema = {'doc': {'type': 'dict', - 'anyof': [ - {'schema': [{'param': {'type': 'number'}}]}]}} + schema = { + 'doc': {'type': 'dict', 'anyof': [{'schema': [{'param': {'type': 'number'}}]}]} + } assert_schema_error({'doc': 'this is my document'}, schema) - schema = {'doc': {'type': 'dict', - 'allof': [ - {'schema': [{'param': {'type': 'number'}}]}]}} + schema = { + 'doc': {'type': 'dict', 'allof': [{'schema': [{'param': {'type': 'number'}}]}]} + } assert_schema_error({'doc': 'this is my document'}, schema) @@ -88,24 +88,87 @@ def test_validated_schema_cache(): v = Validator({'foozifix': {'coerce': int}}) assert len(v._valid_schemas) == cache_size - max_cache_size = 147 - assert cache_size <= max_cache_size, \ - "There's an unexpected high amount (%s) of cached valid " \ - "definition schemas. Unless you added further tests, " \ - "there are good chances that something is wrong. " \ - "If you added tests with new schemas, you can try to " \ - "adjust the variable `max_cache_size` according to " \ + max_cache_size = 160 + assert cache_size <= max_cache_size, ( + "There's an unexpected high amount (%s) of cached valid " + "definition schemas. Unless you added further tests, " + "there are good chances that something is wrong. " + "If you added tests with new schemas, you can try to " + "adjust the variable `max_cache_size` according to " "the added schemas." % cache_size + ) def test_expansion_in_nested_schema(): schema = {'detroit': {'schema': {'anyof_regex': ['^Aladdin', 'Sane$']}}} v = Validator(schema) - assert (v.schema['detroit']['schema'] == - {'anyof': [{'regex': '^Aladdin'}, {'regex': 'Sane$'}]}) + assert v.schema['detroit']['schema'] == { + 'anyof': [{'regex': '^Aladdin'}, {'regex': 'Sane$'}] + } def test_unvalidated_schema_can_be_copied(): schema = UnvalidatedSchema() schema_copy = schema.copy() assert schema_copy == schema + + +# TODO remove with next major release +def test_deprecated_rule_names_in_valueschema(): + def check_with(field, value, error): + pass + + schema = { + "field_1": { + "type": "dict", + "valueschema": { + "type": "dict", + "keyschema": {"type": "string"}, + "valueschema": {"type": "string"}, + }, + }, + "field_2": { + "type": "list", + "items": [ + {"keyschema": {}}, + {"validator": check_with}, + {"valueschema": {}}, + ], + }, + } + + validator = Validator(schema) + + assert validator.schema == { + "field_1": { + "type": "dict", + "valuesrules": { + "type": "dict", + "keysrules": {"type": "string"}, + "valuesrules": {"type": "string"}, + }, + }, + "field_2": { + "type": "list", + "items": [ + {"keysrules": {}}, + {"check_with": check_with}, + {"valuesrules": {}}, + ], + }, + } + + +def test_anyof_check_with(): + def foo(field, value, error): + pass + + def bar(field, value, error): + pass + + schema = {'field': {'anyof_check_with': [foo, bar]}} + validator = Validator(schema) + + assert validator.schema == { + 'field': {'anyof': [{'check_with': foo}, {'check_with': bar}]} + } diff --git a/pipenv/vendor/cerberus/tests/test_utils.py b/pipenv/vendor/cerberus/tests/test_utils.py new file mode 100644 index 00000000..6ab38790 --- /dev/null +++ b/pipenv/vendor/cerberus/tests/test_utils.py @@ -0,0 +1,11 @@ +from cerberus.utils import compare_paths_lt + + +def test_compare_paths(): + lesser = ('a_dict', 'keysrules') + greater = ('a_dict', 'valuesrules') + assert compare_paths_lt(lesser, greater) + + lesser += ('type',) + greater += ('regex',) + assert compare_paths_lt(lesser, greater) diff --git a/pipenv/vendor/cerberus/tests/test_validation.py b/pipenv/vendor/cerberus/tests/test_validation.py index 1f828fac..ead79517 100644 --- a/pipenv/vendor/cerberus/tests/test_validation.py +++ b/pipenv/vendor/cerberus/tests/test_validation.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- +import itertools import re import sys from datetime import datetime, date @@ -10,29 +11,34 @@ from pytest import mark from cerberus import errors, Validator from cerberus.tests import ( - assert_bad_type, assert_document_error, assert_fail, assert_has_error, - assert_not_has_error, assert_success + assert_bad_type, + assert_document_error, + assert_fail, + assert_has_error, + assert_not_has_error, + assert_success, ) from cerberus.tests.conftest import sample_schema def test_empty_document(): - assert_document_error(None, sample_schema, None, - errors.DOCUMENT_MISSING) + assert_document_error(None, sample_schema, None, errors.DOCUMENT_MISSING) def test_bad_document_type(): document = "not a dict" assert_document_error( - document, sample_schema, None, - errors.DOCUMENT_FORMAT.format(document) + document, sample_schema, None, errors.DOCUMENT_FORMAT.format(document) ) def test_unknown_field(validator): field = 'surname' - assert_fail({field: 'doe'}, validator=validator, - error=(field, (), errors.UNKNOWN_FIELD, None)) + assert_fail( + {field: 'doe'}, + validator=validator, + error=(field, (), errors.UNKNOWN_FIELD, None), + ) assert validator.errors == {field: ['unknown field']} @@ -45,14 +51,19 @@ def test_empty_field_definition(document): def test_required_field(schema): field = 'a_required_string' required_string_extension = { - 'a_required_string': {'type': 'string', - 'minlength': 2, - 'maxlength': 10, - 'required': True}} + 'a_required_string': { + 'type': 'string', + 'minlength': 2, + 'maxlength': 10, + 'required': True, + } + } schema.update(required_string_extension) - assert_fail({'an_integer': 1}, schema, - error=(field, (field, 'required'), errors.REQUIRED_FIELD, - True)) + assert_fail( + {'an_integer': 1}, + schema, + error=(field, (field, 'required'), errors.REQUIRED_FIELD, True), + ) def test_nullable_field(): @@ -64,22 +75,23 @@ def test_nullable_field(): assert_fail({'a_not_nullable_field_without_type': None}) +def test_nullable_skips_allowed(): + schema = {'role': {'allowed': ['agent', 'client', 'supplier'], 'nullable': True}} + assert_success({'role': None}, schema) + + def test_readonly_field(): field = 'a_readonly_string' - assert_fail({field: 'update me if you can'}, - error=(field, (field, 'readonly'), errors.READONLY_FIELD, True)) + assert_fail( + {field: 'update me if you can'}, + error=(field, (field, 'readonly'), errors.READONLY_FIELD, True), + ) def test_readonly_field_first_rule(): # test that readonly rule is checked before any other rule, and blocks. # See #63. - schema = { - 'a_readonly_number': { - 'type': 'integer', - 'readonly': True, - 'max': 1 - } - } + schema = {'a_readonly_number': {'type': 'integer', 'readonly': True, 'max': 1}} v = Validator(schema) v.validate({'a_readonly_number': 2}) # it would be a list if there's more than one error; we get a dict @@ -89,28 +101,34 @@ def test_readonly_field_first_rule(): def test_readonly_field_with_default_value(): schema = { - 'created': { - 'type': 'string', - 'readonly': True, - 'default': 'today' - }, + 'created': {'type': 'string', 'readonly': True, 'default': 'today'}, 'modified': { 'type': 'string', 'readonly': True, - 'default_setter': lambda d: d['created'] - } + 'default_setter': lambda d: d['created'], + }, } assert_success({}, schema) - expected_errors = [('created', ('created', 'readonly'), - errors.READONLY_FIELD, - schema['created']['readonly']), - ('modified', ('modified', 'readonly'), - errors.READONLY_FIELD, - schema['modified']['readonly'])] - assert_fail({'created': 'tomorrow', 'modified': 'today'}, - schema, errors=expected_errors) - assert_fail({'created': 'today', 'modified': 'today'}, - schema, errors=expected_errors) + expected_errors = [ + ( + 'created', + ('created', 'readonly'), + errors.READONLY_FIELD, + schema['created']['readonly'], + ), + ( + 'modified', + ('modified', 'readonly'), + errors.READONLY_FIELD, + schema['modified']['readonly'], + ), + ] + assert_fail( + {'created': 'tomorrow', 'modified': 'today'}, schema, errors=expected_errors + ) + assert_fail( + {'created': 'today', 'modified': 'today'}, schema, errors=expected_errors + ) def test_nested_readonly_field_with_default_value(): @@ -118,33 +136,40 @@ def test_nested_readonly_field_with_default_value(): 'some_field': { 'type': 'dict', 'schema': { - 'created': { - 'type': 'string', - 'readonly': True, - 'default': 'today' - }, + 'created': {'type': 'string', 'readonly': True, 'default': 'today'}, 'modified': { 'type': 'string', 'readonly': True, - 'default_setter': lambda d: d['created'] - } - } + 'default_setter': lambda d: d['created'], + }, + }, } } assert_success({'some_field': {}}, schema) expected_errors = [ - (('some_field', 'created'), - ('some_field', 'schema', 'created', 'readonly'), - errors.READONLY_FIELD, - schema['some_field']['schema']['created']['readonly']), - (('some_field', 'modified'), - ('some_field', 'schema', 'modified', 'readonly'), - errors.READONLY_FIELD, - schema['some_field']['schema']['modified']['readonly'])] - assert_fail({'some_field': {'created': 'tomorrow', 'modified': 'now'}}, - schema, errors=expected_errors) - assert_fail({'some_field': {'created': 'today', 'modified': 'today'}}, - schema, errors=expected_errors) + ( + ('some_field', 'created'), + ('some_field', 'schema', 'created', 'readonly'), + errors.READONLY_FIELD, + schema['some_field']['schema']['created']['readonly'], + ), + ( + ('some_field', 'modified'), + ('some_field', 'schema', 'modified', 'readonly'), + errors.READONLY_FIELD, + schema['some_field']['schema']['modified']['readonly'], + ), + ] + assert_fail( + {'some_field': {'created': 'tomorrow', 'modified': 'now'}}, + schema, + errors=expected_errors, + ) + assert_fail( + {'some_field': {'created': 'today', 'modified': 'today'}}, + schema, + errors=expected_errors, + ) def test_repeated_readonly(validator): @@ -195,44 +220,73 @@ def test_bad_max_length(schema): field = 'a_string' max_length = schema[field]['maxlength'] value = "".join(choice(ascii_lowercase) for i in range(max_length + 1)) - assert_fail({field: value}, - error=(field, (field, 'maxlength'), errors.MAX_LENGTH, - max_length, (len(value),))) + assert_fail( + {field: value}, + error=( + field, + (field, 'maxlength'), + errors.MAX_LENGTH, + max_length, + (len(value),), + ), + ) def test_bad_max_length_binary(schema): field = 'a_binary' max_length = schema[field]['maxlength'] value = b'\x00' * (max_length + 1) - assert_fail({field: value}, - error=(field, (field, 'maxlength'), errors.MAX_LENGTH, - max_length, (len(value),))) + assert_fail( + {field: value}, + error=( + field, + (field, 'maxlength'), + errors.MAX_LENGTH, + max_length, + (len(value),), + ), + ) def test_bad_min_length(schema): field = 'a_string' min_length = schema[field]['minlength'] value = "".join(choice(ascii_lowercase) for i in range(min_length - 1)) - assert_fail({field: value}, - error=(field, (field, 'minlength'), errors.MIN_LENGTH, - min_length, (len(value),))) + assert_fail( + {field: value}, + error=( + field, + (field, 'minlength'), + errors.MIN_LENGTH, + min_length, + (len(value),), + ), + ) def test_bad_min_length_binary(schema): field = 'a_binary' min_length = schema[field]['minlength'] value = b'\x00' * (min_length - 1) - assert_fail({field: value}, - error=(field, (field, 'minlength'), errors.MIN_LENGTH, - min_length, (len(value),))) + assert_fail( + {field: value}, + error=( + field, + (field, 'minlength'), + errors.MIN_LENGTH, + min_length, + (len(value),), + ), + ) def test_bad_max_value(schema): def assert_bad_max_value(field, inc): max_value = schema[field]['max'] value = max_value + inc - assert_fail({field: value}, - error=(field, (field, 'max'), errors.MAX_VALUE, max_value)) + assert_fail( + {field: value}, error=(field, (field, 'max'), errors.MAX_VALUE, max_value) + ) field = 'an_integer' assert_bad_max_value(field, 1) @@ -246,9 +300,9 @@ def test_bad_min_value(schema): def assert_bad_min_value(field, inc): min_value = schema[field]['min'] value = min_value - inc - assert_fail({field: value}, - error=(field, (field, 'min'), - errors.MIN_VALUE, min_value)) + assert_fail( + {field: value}, error=(field, (field, 'min'), errors.MIN_VALUE, min_value) + ) field = 'an_integer' assert_bad_min_value(field, 1) @@ -261,65 +315,112 @@ def test_bad_min_value(schema): def test_bad_schema(): field = 'a_dict' subschema_field = 'address' - schema = {field: {'type': 'dict', - 'schema': {subschema_field: {'type': 'string'}, - 'city': {'type': 'string', 'required': True}} - }} + schema = { + field: { + 'type': 'dict', + 'schema': { + subschema_field: {'type': 'string'}, + 'city': {'type': 'string', 'required': True}, + }, + } + } document = {field: {subschema_field: 34}} validator = Validator(schema) assert_fail( - document, validator=validator, - error=(field, (field, 'schema'), errors.MAPPING_SCHEMA, - validator.schema['a_dict']['schema']), + document, + validator=validator, + error=( + field, + (field, 'schema'), + errors.MAPPING_SCHEMA, + validator.schema['a_dict']['schema'], + ), child_errors=[ - ((field, subschema_field), - (field, 'schema', subschema_field, 'type'), - errors.BAD_TYPE, 'string'), - ((field, 'city'), (field, 'schema', 'city', 'required'), - errors.REQUIRED_FIELD, True)] + ( + (field, subschema_field), + (field, 'schema', subschema_field, 'type'), + errors.BAD_TYPE, + 'string', + ), + ( + (field, 'city'), + (field, 'schema', 'city', 'required'), + errors.REQUIRED_FIELD, + True, + ), + ], ) handler = errors.BasicErrorHandler assert field in validator.errors assert subschema_field in validator.errors[field][-1] - assert handler.messages[errors.BAD_TYPE.code].format(constraint='string') \ + assert ( + handler.messages[errors.BAD_TYPE.code].format(constraint='string') in validator.errors[field][-1][subschema_field] + ) assert 'city' in validator.errors[field][-1] - assert (handler.messages[errors.REQUIRED_FIELD.code] - in validator.errors[field][-1]['city']) + assert ( + handler.messages[errors.REQUIRED_FIELD.code] + in validator.errors[field][-1]['city'] + ) -def test_bad_valueschema(): - field = 'a_dict_with_valueschema' +def test_bad_valuesrules(): + field = 'a_dict_with_valuesrules' schema_field = 'a_string' value = {schema_field: 'not an integer'} exp_child_errors = [ - ((field, schema_field), (field, 'valueschema', 'type'), errors.BAD_TYPE, - 'integer')] - assert_fail({field: value}, - error=(field, (field, 'valueschema'), errors.VALUESCHEMA, - {'type': 'integer'}), child_errors=exp_child_errors) + ( + (field, schema_field), + (field, 'valuesrules', 'type'), + errors.BAD_TYPE, + 'integer', + ) + ] + assert_fail( + {field: value}, + error=(field, (field, 'valuesrules'), errors.VALUESRULES, {'type': 'integer'}), + child_errors=exp_child_errors, + ) def test_bad_list_of_values(validator): field = 'a_list_of_values' value = ['a string', 'not an integer'] - assert_fail({field: value}, validator=validator, - error=(field, (field, 'items'), errors.BAD_ITEMS, - [{'type': 'string'}, {'type': 'integer'}]), - child_errors=[((field, 1), (field, 'items', 1, 'type'), - errors.BAD_TYPE, 'integer')]) + assert_fail( + {field: value}, + validator=validator, + error=( + field, + (field, 'items'), + errors.BAD_ITEMS, + [{'type': 'string'}, {'type': 'integer'}], + ), + child_errors=[ + ((field, 1), (field, 'items', 1, 'type'), errors.BAD_TYPE, 'integer') + ], + ) - assert (errors.BasicErrorHandler.messages[errors.BAD_TYPE.code]. - format(constraint='integer') - in validator.errors[field][-1][1]) + assert ( + errors.BasicErrorHandler.messages[errors.BAD_TYPE.code].format( + constraint='integer' + ) + in validator.errors[field][-1][1] + ) value = ['a string', 10, 'an extra item'] - assert_fail({field: value}, - error=(field, (field, 'items'), errors.ITEMS_LENGTH, - [{'type': 'string'}, {'type': 'integer'}], (2, 3))) + assert_fail( + {field: value}, + error=( + field, + (field, 'items'), + errors.ITEMS_LENGTH, + [{'type': 'string'}, {'type': 'integer'}], + (2, 3), + ), + ) def test_bad_list_of_integers(): @@ -330,58 +431,81 @@ def test_bad_list_of_integers(): def test_bad_list_of_dicts(): field = 'a_list_of_dicts' - map_schema = {'sku': {'type': 'string'}, - 'price': {'type': 'integer', 'required': True}} + map_schema = { + 'sku': {'type': 'string'}, + 'price': {'type': 'integer', 'required': True}, + } seq_schema = {'type': 'dict', 'schema': map_schema} schema = {field: {'type': 'list', 'schema': seq_schema}} validator = Validator(schema) value = [{'sku': 'KT123', 'price': '100'}] document = {field: value} - assert_fail(document, validator=validator, - error=(field, (field, 'schema'), errors.SEQUENCE_SCHEMA, - seq_schema), - child_errors=[((field, 0), (field, 'schema', 'schema'), - errors.MAPPING_SCHEMA, map_schema)]) + assert_fail( + document, + validator=validator, + error=(field, (field, 'schema'), errors.SEQUENCE_SCHEMA, seq_schema), + child_errors=[ + ((field, 0), (field, 'schema', 'schema'), errors.MAPPING_SCHEMA, map_schema) + ], + ) assert field in validator.errors assert 0 in validator.errors[field][-1] assert 'price' in validator.errors[field][-1][0][-1] - exp_msg = errors.BasicErrorHandler.messages[errors.BAD_TYPE.code] \ - .format(constraint='integer') + exp_msg = errors.BasicErrorHandler.messages[errors.BAD_TYPE.code].format( + constraint='integer' + ) assert exp_msg in validator.errors[field][-1][0][-1]['price'] value = ["not a dict"] - exp_child_errors = [((field, 0), (field, 'schema', 'type'), - errors.BAD_TYPE, 'dict', ())] - assert_fail({field: value}, - error=(field, (field, 'schema'), errors.SEQUENCE_SCHEMA, - seq_schema), - child_errors=exp_child_errors) + exp_child_errors = [ + ((field, 0), (field, 'schema', 'type'), errors.BAD_TYPE, 'dict', ()) + ] + assert_fail( + {field: value}, + error=(field, (field, 'schema'), errors.SEQUENCE_SCHEMA, seq_schema), + child_errors=exp_child_errors, + ) def test_array_unallowed(): field = 'an_array' value = ['agent', 'client', 'profit'] - assert_fail({field: value}, - error=(field, (field, 'allowed'), errors.UNALLOWED_VALUES, - ['agent', 'client', 'vendor'], ['profit'])) + assert_fail( + {field: value}, + error=( + field, + (field, 'allowed'), + errors.UNALLOWED_VALUES, + ['agent', 'client', 'vendor'], + ['profit'], + ), + ) def test_string_unallowed(): field = 'a_restricted_string' value = 'profit' - assert_fail({field: value}, - error=(field, (field, 'allowed'), errors.UNALLOWED_VALUE, - ['agent', 'client', 'vendor'], value)) + assert_fail( + {field: value}, + error=( + field, + (field, 'allowed'), + errors.UNALLOWED_VALUE, + ['agent', 'client', 'vendor'], + value, + ), + ) def test_integer_unallowed(): field = 'a_restricted_integer' value = 2 - assert_fail({field: value}, - error=(field, (field, 'allowed'), errors.UNALLOWED_VALUE, - [-1, 0, 1], value)) + assert_fail( + {field: value}, + error=(field, (field, 'allowed'), errors.UNALLOWED_VALUE, [-1, 0, 1], value), + ) def test_integer_allowed(): @@ -389,10 +513,14 @@ def test_integer_allowed(): def test_validate_update(): - assert_success({'an_integer': 100, - 'a_dict': {'address': 'adr'}, - 'a_list_of_dicts': [{'sku': 'let'}] - }, update=True) + assert_success( + { + 'an_integer': 100, + 'a_dict': {'address': 'adr'}, + 'a_list_of_dicts': [{'sku': 'let'}], + }, + update=True, + ) def test_string(): @@ -437,24 +565,35 @@ def test_one_of_two_types(validator): field = 'one_or_more_strings' assert_success({field: 'foo'}) assert_success({field: ['foo', 'bar']}) - exp_child_errors = [((field, 1), (field, 'schema', 'type'), - errors.BAD_TYPE, 'string')] - assert_fail({field: ['foo', 23]}, validator=validator, - error=(field, (field, 'schema'), errors.SEQUENCE_SCHEMA, - {'type': 'string'}), - child_errors=exp_child_errors) - assert_fail({field: 23}, - error=((field,), (field, 'type'), errors.BAD_TYPE, - ['string', 'list'])) + exp_child_errors = [ + ((field, 1), (field, 'schema', 'type'), errors.BAD_TYPE, 'string') + ] + assert_fail( + {field: ['foo', 23]}, + validator=validator, + error=(field, (field, 'schema'), errors.SEQUENCE_SCHEMA, {'type': 'string'}), + child_errors=exp_child_errors, + ) + assert_fail( + {field: 23}, + error=((field,), (field, 'type'), errors.BAD_TYPE, ['string', 'list']), + ) assert validator.errors == {field: [{1: ['must be of string type']}]} def test_regex(validator): field = 'a_regex_email' assert_success({field: 'valid.email@gmail.com'}, validator=validator) - assert_fail({field: 'invalid'}, update=True, - error=(field, (field, 'regex'), errors.REGEX_MISMATCH, - '^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$')) + assert_fail( + {field: 'invalid'}, + update=True, + error=( + field, + (field, 'regex'), + errors.REGEX_MISMATCH, + r'^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$', + ), + ) def test_a_list_of_dicts(): @@ -462,7 +601,7 @@ def test_a_list_of_dicts(): { 'a_list_of_dicts': [ {'sku': 'AK345', 'price': 100}, - {'sku': 'YZ069', 'price': 25} + {'sku': 'YZ069', 'price': 25}, ] } ) @@ -472,50 +611,84 @@ def test_a_list_of_values(): assert_success({'a_list_of_values': ['hello', 100]}) +def test_an_array_from_set(): + assert_success({'an_array_from_set': ['agent', 'client']}) + + def test_a_list_of_integers(): assert_success({'a_list_of_integers': [99, 100]}) def test_a_dict(schema): - assert_success({'a_dict': {'address': 'i live here', - 'city': 'in my own town'}}) + assert_success({'a_dict': {'address': 'i live here', 'city': 'in my own town'}}) assert_fail( {'a_dict': {'address': 8545}}, - error=('a_dict', ('a_dict', 'schema'), errors.MAPPING_SCHEMA, - schema['a_dict']['schema']), - child_errors=[(('a_dict', 'address'), - ('a_dict', 'schema', 'address', 'type'), - errors.BAD_TYPE, 'string'), - (('a_dict', 'city'), - ('a_dict', 'schema', 'city', 'required'), - errors.REQUIRED_FIELD, True)] + error=( + 'a_dict', + ('a_dict', 'schema'), + errors.MAPPING_SCHEMA, + schema['a_dict']['schema'], + ), + child_errors=[ + ( + ('a_dict', 'address'), + ('a_dict', 'schema', 'address', 'type'), + errors.BAD_TYPE, + 'string', + ), + ( + ('a_dict', 'city'), + ('a_dict', 'schema', 'city', 'required'), + errors.REQUIRED_FIELD, + True, + ), + ], ) -def test_a_dict_with_valueschema(validator): - assert_success({'a_dict_with_valueschema': - {'an integer': 99, 'another integer': 100}}) +def test_a_dict_with_valuesrules(validator): + assert_success( + {'a_dict_with_valuesrules': {'an integer': 99, 'another integer': 100}} + ) error = ( - 'a_dict_with_valueschema', ('a_dict_with_valueschema', 'valueschema'), - errors.VALUESCHEMA, {'type': 'integer'}) + 'a_dict_with_valuesrules', + ('a_dict_with_valuesrules', 'valuesrules'), + errors.VALUESRULES, + {'type': 'integer'}, + ) child_errors = [ - (('a_dict_with_valueschema', 'a string'), - ('a_dict_with_valueschema', 'valueschema', 'type'), - errors.BAD_TYPE, 'integer')] + ( + ('a_dict_with_valuesrules', 'a string'), + ('a_dict_with_valuesrules', 'valuesrules', 'type'), + errors.BAD_TYPE, + 'integer', + ) + ] - assert_fail({'a_dict_with_valueschema': {'a string': '99'}}, - validator=validator, error=error, child_errors=child_errors) + assert_fail( + {'a_dict_with_valuesrules': {'a string': '99'}}, + validator=validator, + error=error, + child_errors=child_errors, + ) - assert 'valueschema' in \ - validator.schema_error_tree['a_dict_with_valueschema'] + assert 'valuesrules' in validator.schema_error_tree['a_dict_with_valuesrules'] v = validator.schema_error_tree - assert len(v['a_dict_with_valueschema']['valueschema'].descendants) == 1 + assert len(v['a_dict_with_valuesrules']['valuesrules'].descendants) == 1 -def test_a_dict_with_keyschema(): - assert_success({'a_dict_with_keyschema': {'key': 'value'}}) - assert_fail({'a_dict_with_keyschema': {'KEY': 'value'}}) +# TODO remove 'keyschema' as rule with the next major release +@mark.parametrize('rule', ('keysrules', 'keyschema')) +def test_keysrules(rule): + schema = { + 'a_dict_with_keysrules': { + 'type': 'dict', + rule: {'type': 'string', 'regex': '[a-z]+'}, + } + } + assert_success({'a_dict_with_keysrules': {'key': 'value'}}, schema=schema) + assert_fail({'a_dict_with_keysrules': {'KEY': 'value'}}, schema=schema) def test_a_list_length(schema): @@ -523,17 +696,31 @@ def test_a_list_length(schema): min_length = schema[field]['minlength'] max_length = schema[field]['maxlength'] - assert_fail({field: [1] * (min_length - 1)}, - error=(field, (field, 'minlength'), errors.MIN_LENGTH, - min_length, (min_length - 1,))) + assert_fail( + {field: [1] * (min_length - 1)}, + error=( + field, + (field, 'minlength'), + errors.MIN_LENGTH, + min_length, + (min_length - 1,), + ), + ) for i in range(min_length, max_length): value = [1] * i assert_success({field: value}) - assert_fail({field: [1] * (max_length + 1)}, - error=(field, (field, 'maxlength'), errors.MAX_LENGTH, - max_length, (max_length + 1,))) + assert_fail( + {field: [1] * (max_length + 1)}, + error=( + field, + (field, 'maxlength'), + errors.MAX_LENGTH, + max_length, + (max_length + 1,), + ), + ) def test_custom_datatype(): @@ -544,11 +731,12 @@ def test_custom_datatype(): schema = {'test_field': {'type': 'objectid'}} validator = MyValidator(schema) - assert_success({'test_field': '50ad188438345b1049c88a28'}, - validator=validator) - assert_fail({'test_field': 'hello'}, validator=validator, - error=('test_field', ('test_field', 'type'), errors.BAD_TYPE, - 'objectid')) + assert_success({'test_field': '50ad188438345b1049c88a28'}, validator=validator) + assert_fail( + {'test_field': 'hello'}, + validator=validator, + error=('test_field', ('test_field', 'type'), errors.BAD_TYPE, 'objectid'), + ) def test_custom_datatype_rule(): @@ -565,12 +753,16 @@ def test_custom_datatype_rule(): schema = {'test_field': {'min_number': 1, 'type': 'number'}} validator = MyValidator(schema) - assert_fail({'test_field': '0'}, validator=validator, - error=('test_field', ('test_field', 'type'), errors.BAD_TYPE, - 'number')) - assert_fail({'test_field': 0}, validator=validator, - error=('test_field', (), errors.CUSTOM, None, - ('Below the min',))) + assert_fail( + {'test_field': '0'}, + validator=validator, + error=('test_field', ('test_field', 'type'), errors.BAD_TYPE, 'number'), + ) + assert_fail( + {'test_field': 0}, + validator=validator, + error=('test_field', (), errors.CUSTOM, None, ('Below the min',)), + ) assert validator.errors == {'test_field': ['Below the min']} @@ -584,14 +776,17 @@ def test_custom_validator(): schema = {'test_field': {'isodd': True}} validator = MyValidator(schema) assert_success({'test_field': 7}, validator=validator) - assert_fail({'test_field': 6}, validator=validator, - error=('test_field', (), errors.CUSTOM, None, - ('Not an odd number',))) + assert_fail( + {'test_field': 6}, + validator=validator, + error=('test_field', (), errors.CUSTOM, None, ('Not an odd number',)), + ) assert validator.errors == {'test_field': ['Not an odd number']} -@mark.parametrize('value, _type', - (('', 'string'), ((), 'list'), ({}, 'dict'), ([], 'list'))) +@mark.parametrize( + 'value, _type', (('', 'string'), ((), 'list'), ({}, 'dict'), ([], 'list')) +) def test_empty_values(value, _type): field = 'test' schema = {field: {'type': _type}} @@ -600,17 +795,18 @@ def test_empty_values(value, _type): assert_success(document, schema) schema[field]['empty'] = False - assert_fail(document, schema, - error=(field, (field, 'empty'), - errors.EMPTY_NOT_ALLOWED, False)) + assert_fail( + document, + schema, + error=(field, (field, 'empty'), errors.EMPTY_NOT_ALLOWED, False), + ) schema[field]['empty'] = True assert_success(document, schema) def test_empty_skips_regex(validator): - schema = {'foo': {'empty': True, 'regex': r'\d?\d\.\d\d', - 'type': 'string'}} + schema = {'foo': {'empty': True, 'regex': r'\d?\d\.\d\d', 'type': 'string'}} assert validator({'foo': ''}, schema) @@ -625,8 +821,9 @@ def test_ignore_none_values(): validator.schema[field]['required'] = True validator.schema.validate() _errors = assert_fail(document, validator=validator) - assert_not_has_error(_errors, field, (field, 'required'), - errors.REQUIRED_FIELD, True) + assert_not_has_error( + _errors, field, (field, 'required'), errors.REQUIRED_FIELD, True + ) # Test ignore None behaviour validator = Validator(schema, ignore_none_values=True) @@ -635,10 +832,8 @@ def test_ignore_none_values(): assert_success(document, validator=validator) validator.schema[field]['required'] = True _errors = assert_fail(schema=schema, document=document, validator=validator) - assert_has_error(_errors, field, (field, 'required'), errors.REQUIRED_FIELD, - True) - assert_not_has_error(_errors, field, (field, 'type'), errors.BAD_TYPE, - 'string') + assert_has_error(_errors, field, (field, 'required'), errors.REQUIRED_FIELD, True) + assert_not_has_error(_errors, field, (field, 'type'), errors.BAD_TYPE, 'string') def test_unknown_keys(): @@ -677,8 +872,7 @@ def test_unknown_keys_list_of_dicts(validator): # test that allow_unknown is honored even for subdicts in lists. # https://github.com/pyeve/cerberus/issues/67. validator.allow_unknown = True - document = {'a_list_of_dicts': [{'sku': 'YZ069', 'price': 25, - 'extra': True}]} + document = {'a_list_of_dicts': [{'sku': 'YZ069', 'price': 25, 'extra': True}]} assert_success(document, validator=validator) @@ -692,8 +886,7 @@ def test_unknown_keys_retain_custom_rules(): validator = CustomValidator({}) validator.allow_unknown = {"type": "foo"} - assert_success(document={"fred": "foo", "barney": "foo"}, - validator=validator) + assert_success(document={"fred": "foo", "barney": "foo"}, validator=validator) def test_nested_unknown_keys(): @@ -701,16 +894,10 @@ def test_nested_unknown_keys(): 'field1': { 'type': 'dict', 'allow_unknown': True, - 'schema': {'nested1': {'type': 'string'}} - } - } - document = { - 'field1': { - 'nested1': 'foo', - 'arb1': 'bar', - 'arb2': 42 + 'schema': {'nested1': {'type': 'string'}}, } } + document = {'field1': {'nested1': 'foo', 'arb1': 'bar', 'arb2': 42}} assert_success(document=document, schema=schema) schema['field1']['allow_unknown'] = {'type': 'string'} @@ -739,8 +926,7 @@ def test_callable_validator(): def test_dependencies_field(): - schema = {'test_field': {'dependencies': 'foo'}, - 'foo': {'type': 'string'}} + schema = {'test_field': {'dependencies': 'foo'}, 'foo': {'type': 'string'}} assert_success({'test_field': 'foobar', 'foo': 'bar'}, schema) assert_fail({'test_field': 'foobar'}, schema) @@ -749,10 +935,9 @@ def test_dependencies_list(): schema = { 'test_field': {'dependencies': ['foo', 'bar']}, 'foo': {'type': 'string'}, - 'bar': {'type': 'string'} + 'bar': {'type': 'string'}, } - assert_success({'test_field': 'foobar', 'foo': 'bar', 'bar': 'foo'}, - schema) + assert_success({'test_field': 'foobar', 'foo': 'bar', 'bar': 'foo'}, schema) assert_fail({'test_field': 'foobar', 'foo': 'bar'}, schema) @@ -760,7 +945,7 @@ def test_dependencies_list_with_required_field(): schema = { 'test_field': {'required': True, 'dependencies': ['foo', 'bar']}, 'foo': {'type': 'string'}, - 'bar': {'type': 'string'} + 'bar': {'type': 'string'}, } # False: all dependencies missing assert_fail({'test_field': 'foobar'}, schema) @@ -784,27 +969,23 @@ def test_dependencies_list_with_subodcuments_fields(): 'test_field': {'dependencies': ['a_dict.foo', 'a_dict.bar']}, 'a_dict': { 'type': 'dict', - 'schema': { - 'foo': {'type': 'string'}, - 'bar': {'type': 'string'} - } - } + 'schema': {'foo': {'type': 'string'}, 'bar': {'type': 'string'}}, + }, } - assert_success({'test_field': 'foobar', - 'a_dict': {'foo': 'foo', 'bar': 'bar'}}, schema) + assert_success( + {'test_field': 'foobar', 'a_dict': {'foo': 'foo', 'bar': 'bar'}}, schema + ) assert_fail({'test_field': 'foobar', 'a_dict': {}}, schema) - assert_fail({'test_field': 'foobar', - 'a_dict': {'foo': 'foo'}}, schema) + assert_fail({'test_field': 'foobar', 'a_dict': {'foo': 'foo'}}, schema) def test_dependencies_dict(): schema = { 'test_field': {'dependencies': {'foo': 'foo', 'bar': 'bar'}}, 'foo': {'type': 'string'}, - 'bar': {'type': 'string'} + 'bar': {'type': 'string'}, } - assert_success({'test_field': 'foobar', 'foo': 'foo', 'bar': 'bar'}, - schema) + assert_success({'test_field': 'foobar', 'foo': 'foo', 'bar': 'bar'}, schema) assert_fail({'test_field': 'foobar', 'foo': 'foo'}, schema) assert_fail({'test_field': 'foobar', 'foo': 'bar'}, schema) assert_fail({'test_field': 'foobar', 'bar': 'bar'}, schema) @@ -814,12 +995,9 @@ def test_dependencies_dict(): def test_dependencies_dict_with_required_field(): schema = { - 'test_field': { - 'required': True, - 'dependencies': {'foo': 'foo', 'bar': 'bar'} - }, + 'test_field': {'required': True, 'dependencies': {'foo': 'foo', 'bar': 'bar'}}, 'foo': {'type': 'string'}, - 'bar': {'type': 'string'} + 'bar': {'type': 'string'}, } # False: all dependencies missing assert_fail({'test_field': 'foobar'}, schema) @@ -833,8 +1011,7 @@ def test_dependencies_dict_with_required_field(): # False: dependency missing assert_fail({'foo': 'bar'}, schema) - assert_success({'test_field': 'foobar', 'foo': 'foo', 'bar': 'bar'}, - schema) + assert_success({'test_field': 'foobar', 'foo': 'foo', 'bar': 'bar'}, schema) # True: dependencies are validated but field is not required schema['test_field']['required'] = False @@ -843,10 +1020,7 @@ def test_dependencies_dict_with_required_field(): def test_dependencies_field_satisfy_nullable_field(): # https://github.com/pyeve/cerberus/issues/305 - schema = { - 'foo': {'nullable': True}, - 'bar': {'dependencies': 'foo'} - } + schema = {'foo': {'nullable': True}, 'bar': {'dependencies': 'foo'}} assert_success({'foo': None, 'bar': 1}, schema) assert_success({'foo': None}, schema) @@ -857,7 +1031,7 @@ def test_dependencies_field_with_mutually_dependent_nullable_fields(): # https://github.com/pyeve/cerberus/pull/306 schema = { 'foo': {'dependencies': 'bar', 'nullable': True}, - 'bar': {'dependencies': 'foo', 'nullable': True} + 'bar': {'dependencies': 'foo', 'nullable': True}, } assert_success({'foo': None, 'bar': None}, schema) assert_success({'foo': 1, 'bar': 1}, schema) @@ -868,63 +1042,75 @@ def test_dependencies_field_with_mutually_dependent_nullable_fields(): def test_dependencies_dict_with_subdocuments_fields(): schema = { - 'test_field': {'dependencies': {'a_dict.foo': ['foo', 'bar'], - 'a_dict.bar': 'bar'}}, + 'test_field': { + 'dependencies': {'a_dict.foo': ['foo', 'bar'], 'a_dict.bar': 'bar'} + }, 'a_dict': { 'type': 'dict', - 'schema': { - 'foo': {'type': 'string'}, - 'bar': {'type': 'string'} - } - } + 'schema': {'foo': {'type': 'string'}, 'bar': {'type': 'string'}}, + }, } - assert_success({'test_field': 'foobar', - 'a_dict': {'foo': 'foo', 'bar': 'bar'}}, schema) - assert_success({'test_field': 'foobar', - 'a_dict': {'foo': 'bar', 'bar': 'bar'}}, schema) + assert_success( + {'test_field': 'foobar', 'a_dict': {'foo': 'foo', 'bar': 'bar'}}, schema + ) + assert_success( + {'test_field': 'foobar', 'a_dict': {'foo': 'bar', 'bar': 'bar'}}, schema + ) assert_fail({'test_field': 'foobar', 'a_dict': {}}, schema) - assert_fail({'test_field': 'foobar', - 'a_dict': {'foo': 'foo', 'bar': 'foo'}}, schema) - assert_fail({'test_field': 'foobar', 'a_dict': {'bar': 'foo'}}, - schema) - assert_fail({'test_field': 'foobar', 'a_dict': {'bar': 'bar'}}, - schema) + assert_fail( + {'test_field': 'foobar', 'a_dict': {'foo': 'foo', 'bar': 'foo'}}, schema + ) + assert_fail({'test_field': 'foobar', 'a_dict': {'bar': 'foo'}}, schema) + assert_fail({'test_field': 'foobar', 'a_dict': {'bar': 'bar'}}, schema) def test_root_relative_dependencies(): # https://github.com/pyeve/cerberus/issues/288 subschema = {'version': {'dependencies': '^repo'}} - schema = {'package': {'allow_unknown': True, 'schema': subschema}, - 'repo': {}} + schema = {'package': {'allow_unknown': True, 'schema': subschema}, 'repo': {}} assert_fail( - {'package': {'repo': 'somewhere', 'version': 0}}, schema, - error=('package', ('package', 'schema'), - errors.MAPPING_SCHEMA, subschema), - child_errors=[( - ('package', 'version'), - ('package', 'schema', 'version', 'dependencies'), - errors.DEPENDENCIES_FIELD, '^repo', ('^repo',) - )] + {'package': {'repo': 'somewhere', 'version': 0}}, + schema, + error=('package', ('package', 'schema'), errors.MAPPING_SCHEMA, subschema), + child_errors=[ + ( + ('package', 'version'), + ('package', 'schema', 'version', 'dependencies'), + errors.DEPENDENCIES_FIELD, + '^repo', + ('^repo',), + ) + ], ) assert_success({'repo': 'somewhere', 'package': {'version': 1}}, schema) def test_dependencies_errors(): - v = Validator({'field1': {'required': False}, - 'field2': {'required': True, - 'dependencies': {'field1': ['one', 'two']}}}) - assert_fail({'field1': 'three', 'field2': 7}, validator=v, - error=('field2', ('field2', 'dependencies'), - errors.DEPENDENCIES_FIELD_VALUE, - {'field1': ['one', 'two']}, ({'field1': 'three'},))) + v = Validator( + { + 'field1': {'required': False}, + 'field2': {'required': True, 'dependencies': {'field1': ['one', 'two']}}, + } + ) + assert_fail( + {'field1': 'three', 'field2': 7}, + validator=v, + error=( + 'field2', + ('field2', 'dependencies'), + errors.DEPENDENCIES_FIELD_VALUE, + {'field1': ['one', 'two']}, + ({'field1': 'three'},), + ), + ) def test_options_passed_to_nested_validators(validator): - validator.schema = {'sub_dict': {'type': 'dict', - 'schema': {'foo': {'type': 'string'}}}} + validator.schema = { + 'sub_dict': {'type': 'dict', 'schema': {'foo': {'type': 'string'}}} + } validator.allow_unknown = True - assert_success({'sub_dict': {'foo': 'bar', 'unknown': True}}, - validator=validator) + assert_success({'sub_dict': {'foo': 'bar', 'unknown': True}}, validator=validator) def test_self_root_document(): @@ -937,8 +1123,7 @@ def test_self_root_document(): class MyValidator(Validator): def _validate_root_doc(self, root_doc, field, value): """ {'type': 'boolean'} """ - if ('sub' not in self.root_document or - len(self.root_document['sub']) != 2): + if 'sub' not in self.root_document or len(self.root_document['sub']) != 2: self._error(field, 'self.context is not the root doc!') schema = { @@ -947,17 +1132,13 @@ def test_self_root_document(): 'root_doc': True, 'schema': { 'type': 'dict', - 'schema': { - 'foo': { - 'type': 'string', - 'root_doc': True - } - } - } + 'schema': {'foo': {'type': 'string', 'root_doc': True}}, + }, } } - assert_success({'sub': [{'foo': 'bar'}, {'foo': 'baz'}]}, - validator=MyValidator(schema)) + assert_success( + {'sub': [{'foo': 'bar'}, {'foo': 'baz'}]}, validator=MyValidator(schema) + ) def test_validator_rule(validator): @@ -967,11 +1148,14 @@ def test_validator_rule(validator): validator.schema = { 'name': {'validator': validate_name}, - 'age': {'type': 'integer'} + 'age': {'type': 'integer'}, } - assert_fail({'name': 'ItsMe', 'age': 2}, validator=validator, - error=('name', (), errors.CUSTOM, None, ('must be lowercase',))) + assert_fail( + {'name': 'ItsMe', 'age': 2}, + validator=validator, + error=('name', (), errors.CUSTOM, None, ('must be lowercase',)), + ) assert validator.errors == {'name': ['must be lowercase']} assert_success({'name': 'itsme', 'age': 2}, validator=validator) @@ -992,23 +1176,20 @@ def test_anyof(): assert_success(doc, schema) # prop1 must be either a number between 0 and 10 or 100 and 110 - schema = {'prop1': {'anyof': - [{'min': 0, 'max': 10}, {'min': 100, 'max': 110}]}} + schema = {'prop1': {'anyof': [{'min': 0, 'max': 10}, {'min': 100, 'max': 110}]}} doc = {'prop1': 105} assert_success(doc, schema) # prop1 must be either a number between 0 and 10 or 100 and 110 - schema = {'prop1': {'anyof': - [{'min': 0, 'max': 10}, {'min': 100, 'max': 110}]}} + schema = {'prop1': {'anyof': [{'min': 0, 'max': 10}, {'min': 100, 'max': 110}]}} doc = {'prop1': 50} assert_fail(doc, schema) # prop1 must be an integer that is either be # greater than or equal to 0, or greater than or equal to 10 - schema = {'prop1': {'type': 'integer', - 'anyof': [{'min': 0}, {'min': 10}]}} + schema = {'prop1': {'type': 'integer', 'anyof': [{'min': 0}, {'min': 10}]}} assert_success({'prop1': 10}, schema) # test that intermediate schemas do not sustain assert 'type' not in schema['prop1']['anyof'][0] @@ -1019,12 +1200,14 @@ def test_anyof(): exp_child_errors = [ (('prop1',), ('prop1', 'anyof', 0, 'min'), errors.MIN_VALUE, 0), - (('prop1',), ('prop1', 'anyof', 1, 'min'), errors.MIN_VALUE, 10) + (('prop1',), ('prop1', 'anyof', 1, 'min'), errors.MIN_VALUE, 10), ] - assert_fail({'prop1': -1}, schema, - error=(('prop1',), ('prop1', 'anyof'), errors.ANYOF, - [{'min': 0}, {'min': 10}]), - child_errors=exp_child_errors) + assert_fail( + {'prop1': -1}, + schema, + error=(('prop1',), ('prop1', 'anyof'), errors.ANYOF, [{'min': 0}, {'min': 10}]), + child_errors=exp_child_errors, + ) doc = {'prop1': 5.5} assert_fail(doc, schema) doc = {'prop1': '5.5'} @@ -1033,8 +1216,7 @@ def test_anyof(): def test_allof(): # prop1 has to be a float between 0 and 10 - schema = {'prop1': {'allof': [ - {'type': 'float'}, {'min': 0}, {'max': 10}]}} + schema = {'prop1': {'allof': [{'type': 'float'}, {'min': 0}, {'max': 10}]}} doc = {'prop1': -1} assert_fail(doc, schema) doc = {'prop1': 5} @@ -1067,8 +1249,7 @@ def test_unicode_allowed(): assert_success(doc, schema) -@mark.skipif(sys.version_info[0] < 3, - reason='requires python 3.x') +@mark.skipif(sys.version_info[0] < 3, reason='requires python 3.x') def test_unicode_allowed_py3(): """ All strings are unicode in Python 3.x. Input doc and schema have equal strings and validation yield success.""" @@ -1079,8 +1260,7 @@ def test_unicode_allowed_py3(): assert_success(doc, schema) -@mark.skipif(sys.version_info[0] > 2, - reason='requires python 2.x') +@mark.skipif(sys.version_info[0] > 2, reason='requires python 2.x') def test_unicode_allowed_py2(): """ Python 2.x encodes value of allowed using default encoding if the string includes characters outside ASCII range. Produced string @@ -1098,10 +1278,12 @@ def test_oneof(): # - greater than 0 # - equal to -5, 5, or 15 - schema = {'prop1': {'type': 'integer', 'oneof': [ - {'min': 0}, - {'min': 10}, - {'allowed': [-5, 5, 15]}]}} + schema = { + 'prop1': { + 'type': 'integer', + 'oneof': [{'min': 0}, {'min': 10}, {'allowed': [-5, 5, 15]}], + } + } # document is not valid # prop1 not greater than 0, 10 or equal to -5 @@ -1144,10 +1326,12 @@ def test_noneof(): # - greater than 0 # - equal to -5, 5, or 15 - schema = {'prop1': {'type': 'integer', 'noneof': [ - {'min': 0}, - {'min': 10}, - {'allowed': [-5, 5, 15]}]}} + schema = { + 'prop1': { + 'type': 'integer', + 'noneof': [{'min': 0}, {'min': 10}, {'allowed': [-5, 5, 15]}], + } + } # document is valid doc = {'prop1': -1} @@ -1179,11 +1363,14 @@ def test_noneof(): def test_anyof_allof(): # prop1 can be any number outside of [0-10] - schema = {'prop1': {'allof': [{'anyof': [{'type': 'float'}, - {'type': 'integer'}]}, - {'anyof': [{'min': 10}, - {'max': 0}]} - ]}} + schema = { + 'prop1': { + 'allof': [ + {'anyof': [{'type': 'float'}, {'type': 'integer'}]}, + {'anyof': [{'min': 10}, {'max': 0}]}, + ] + } + } doc = {'prop1': 11} assert_success(doc, schema) @@ -1206,15 +1393,19 @@ def test_anyof_allof(): def test_anyof_schema(validator): # test that a list of schemas can be specified. - valid_parts = [{'schema': {'model number': {'type': 'string'}, - 'count': {'type': 'integer'}}}, - {'schema': {'serial number': {'type': 'string'}, - 'count': {'type': 'integer'}}}] + valid_parts = [ + {'schema': {'model number': {'type': 'string'}, 'count': {'type': 'integer'}}}, + {'schema': {'serial number': {'type': 'string'}, 'count': {'type': 'integer'}}}, + ] valid_item = {'type': ['dict', 'string'], 'anyof': valid_parts} schema = {'parts': {'type': 'list', 'schema': valid_item}} - document = {'parts': [{'model number': 'MX-009', 'count': 100}, - {'serial number': '898-001'}, - 'misc']} + document = { + 'parts': [ + {'model number': 'MX-009', 'count': 100}, + {'serial number': '898-001'}, + 'misc', + ] + } # document is valid. each entry in 'parts' matches a type or schema assert_success(document, schema, validator=validator) @@ -1232,18 +1423,25 @@ def test_anyof_schema(validator): # and invalid. numbers are not allowed. exp_child_errors = [ - (('parts', 3), ('parts', 'schema', 'anyof'), errors.ANYOF, - valid_parts), - (('parts', 4), ('parts', 'schema', 'type'), errors.BAD_TYPE, - ['dict', 'string']) + (('parts', 3), ('parts', 'schema', 'anyof'), errors.ANYOF, valid_parts), + ( + ('parts', 4), + ('parts', 'schema', 'type'), + errors.BAD_TYPE, + ['dict', 'string'], + ), ] - _errors = assert_fail(document, schema, validator=validator, - error=('parts', ('parts', 'schema'), - errors.SEQUENCE_SCHEMA, valid_item), - child_errors=exp_child_errors) - assert_not_has_error(_errors, ('parts', 4), ('parts', 'schema', 'anyof'), - errors.ANYOF, valid_parts) + _errors = assert_fail( + document, + schema, + validator=validator, + error=('parts', ('parts', 'schema'), errors.SEQUENCE_SCHEMA, valid_item), + child_errors=exp_child_errors, + ) + assert_not_has_error( + _errors, ('parts', 4), ('parts', 'schema', 'anyof'), errors.ANYOF, valid_parts + ) # tests errors.BasicErrorHandler's tree representation v_errors = validator.errors @@ -1260,15 +1458,23 @@ def test_anyof_schema(validator): def test_anyof_2(): # these two schema should be the same - schema1 = {'prop': {'anyof': [{'type': 'dict', - 'schema': { - 'val': {'type': 'integer'}}}, - {'type': 'dict', - 'schema': { - 'val': {'type': 'string'}}}]}} - schema2 = {'prop': {'type': 'dict', 'anyof': [ - {'schema': {'val': {'type': 'integer'}}}, - {'schema': {'val': {'type': 'string'}}}]}} + schema1 = { + 'prop': { + 'anyof': [ + {'type': 'dict', 'schema': {'val': {'type': 'integer'}}}, + {'type': 'dict', 'schema': {'val': {'type': 'string'}}}, + ] + } + } + schema2 = { + 'prop': { + 'type': 'dict', + 'anyof': [ + {'schema': {'val': {'type': 'integer'}}}, + {'schema': {'val': {'type': 'string'}}}, + ], + } + } doc = {'prop': {'val': 0}} assert_success(doc, schema1) @@ -1290,47 +1496,69 @@ def test_anyof_type(): def test_oneof_schema(): - schema = {'oneof_schema': {'type': 'dict', - 'oneof_schema': - [{'digits': {'type': 'integer', - 'min': 0, 'max': 99}}, - {'text': {'type': 'string', - 'regex': '^[0-9]{2}$'}}]}} + schema = { + 'oneof_schema': { + 'type': 'dict', + 'oneof_schema': [ + {'digits': {'type': 'integer', 'min': 0, 'max': 99}}, + {'text': {'type': 'string', 'regex': '^[0-9]{2}$'}}, + ], + } + } assert_success({'oneof_schema': {'digits': 19}}, schema) assert_success({'oneof_schema': {'text': '84'}}, schema) assert_fail({'oneof_schema': {'digits': 19, 'text': '84'}}, schema) def test_nested_oneof_type(): - schema = {'nested_oneof_type': - {'valueschema': {'oneof_type': ['string', 'integer']}}} + schema = { + 'nested_oneof_type': {'valuesrules': {'oneof_type': ['string', 'integer']}} + } assert_success({'nested_oneof_type': {'foo': 'a'}}, schema) assert_success({'nested_oneof_type': {'bar': 3}}, schema) def test_nested_oneofs(validator): - validator.schema = {'abc': { - 'type': 'dict', - 'oneof_schema': [ - {'foo': { - 'type': 'dict', - 'schema': {'bar': {'oneof_type': ['integer', 'float']}} - }}, - {'baz': {'type': 'string'}} - ]}} + validator.schema = { + 'abc': { + 'type': 'dict', + 'oneof_schema': [ + { + 'foo': { + 'type': 'dict', + 'schema': {'bar': {'oneof_type': ['integer', 'float']}}, + } + }, + {'baz': {'type': 'string'}}, + ], + } + } document = {'abc': {'foo': {'bar': 'bad'}}} expected_errors = { 'abc': [ 'none or more than one rule validate', - {'oneof definition 0': [ - {'foo': [{'bar': [ - 'none or more than one rule validate', - {'oneof definition 0': ['must be of integer type'], - 'oneof definition 1': ['must be of float type']} - ]}]}], - 'oneof definition 1': [{'foo': ['unknown field']}]} + { + 'oneof definition 0': [ + { + 'foo': [ + { + 'bar': [ + 'none or more than one rule validate', + { + 'oneof definition 0': [ + 'must be of integer type' + ], + 'oneof definition 1': ['must be of float type'], + }, + ] + } + ] + } + ], + 'oneof definition 1': [{'foo': ['unknown field']}], + }, ] } @@ -1339,21 +1567,23 @@ def test_nested_oneofs(validator): def test_no_of_validation_if_type_fails(validator): - valid_parts = [{'schema': {'model number': {'type': 'string'}, - 'count': {'type': 'integer'}}}, - {'schema': {'serial number': {'type': 'string'}, - 'count': {'type': 'integer'}}}] - validator.schema = {'part': {'type': ['dict', 'string'], - 'anyof': valid_parts}} + valid_parts = [ + {'schema': {'model number': {'type': 'string'}, 'count': {'type': 'integer'}}}, + {'schema': {'serial number': {'type': 'string'}, 'count': {'type': 'integer'}}}, + ] + validator.schema = {'part': {'type': ['dict', 'string'], 'anyof': valid_parts}} document = {'part': 10} _errors = assert_fail(document, validator=validator) assert len(_errors) == 1 def test_issue_107(validator): - schema = {'info': {'type': 'dict', - 'schema': {'name': {'type': 'string', - 'required': True}}}} + schema = { + 'info': { + 'type': 'dict', + 'schema': {'name': {'type': 'string', 'required': True}}, + } + } document = {'info': {'name': 'my name'}} assert_success(document, schema, validator=validator) @@ -1369,20 +1599,23 @@ def test_dont_type_validate_nulled_values(validator): def test_dependencies_error(validator): - schema = {'field1': {'required': False}, - 'field2': {'required': True, - 'dependencies': {'field1': ['one', 'two']}}} + schema = { + 'field1': {'required': False}, + 'field2': {'required': True, 'dependencies': {'field1': ['one', 'two']}}, + } validator.validate({'field2': 7}, schema) - exp_msg = errors.BasicErrorHandler \ - .messages[errors.DEPENDENCIES_FIELD_VALUE.code] \ - .format(field='field2', constraint={'field1': ['one', 'two']}) + exp_msg = errors.BasicErrorHandler.messages[ + errors.DEPENDENCIES_FIELD_VALUE.code + ].format(field='field2', constraint={'field1': ['one', 'two']}) assert validator.errors == {'field2': [exp_msg]} def test_dependencies_on_boolean_field_with_one_value(): # https://github.com/pyeve/cerberus/issues/138 - schema = {'deleted': {'type': 'boolean'}, - 'text': {'dependencies': {'deleted': False}}} + schema = { + 'deleted': {'type': 'boolean'}, + 'text': {'dependencies': {'deleted': False}}, + } try: assert_success({'text': 'foo', 'deleted': False}, schema) assert_fail({'text': 'foo', 'deleted': True}, schema) @@ -1392,15 +1625,18 @@ def test_dependencies_on_boolean_field_with_one_value(): raise AssertionError( "Bug #138 still exists, couldn't use boolean in dependency " "without putting it in a list.\n" - "'some_field': True vs 'some_field: [True]") + "'some_field': True vs 'some_field: [True]" + ) else: raise def test_dependencies_on_boolean_field_with_value_in_list(): # https://github.com/pyeve/cerberus/issues/138 - schema = {'deleted': {'type': 'boolean'}, - 'text': {'dependencies': {'deleted': [False]}}} + schema = { + 'deleted': {'type': 'boolean'}, + 'text': {'dependencies': {'deleted': [False]}}, + } assert_success({'text': 'foo', 'deleted': False}, schema) assert_fail({'text': 'foo', 'deleted': True}, schema) @@ -1423,9 +1659,10 @@ def test_document_path(): def test_excludes(): - schema = {'this_field': {'type': 'dict', - 'excludes': 'that_field'}, - 'that_field': {'type': 'dict'}} + schema = { + 'this_field': {'type': 'dict', 'excludes': 'that_field'}, + 'that_field': {'type': 'dict'}, + } assert_success({'this_field': {}}, schema) assert_success({'that_field': {}}, schema) assert_success({}, schema) @@ -1433,10 +1670,10 @@ def test_excludes(): def test_mutual_excludes(): - schema = {'this_field': {'type': 'dict', - 'excludes': 'that_field'}, - 'that_field': {'type': 'dict', - 'excludes': 'this_field'}} + schema = { + 'this_field': {'type': 'dict', 'excludes': 'that_field'}, + 'that_field': {'type': 'dict', 'excludes': 'this_field'}, + } assert_success({'this_field': {}}, schema) assert_success({'that_field': {}}, schema) assert_success({}, schema) @@ -1444,12 +1681,10 @@ def test_mutual_excludes(): def test_required_excludes(): - schema = {'this_field': {'type': 'dict', - 'excludes': 'that_field', - 'required': True}, - 'that_field': {'type': 'dict', - 'excludes': 'this_field', - 'required': True}} + schema = { + 'this_field': {'type': 'dict', 'excludes': 'that_field', 'required': True}, + 'that_field': {'type': 'dict', 'excludes': 'this_field', 'required': True}, + } assert_success({'this_field': {}}, schema, update=False) assert_success({'that_field': {}}, schema, update=False) assert_fail({}, schema) @@ -1457,11 +1692,11 @@ def test_required_excludes(): def test_multiples_exclusions(): - schema = {'this_field': {'type': 'dict', - 'excludes': ['that_field', 'bazo_field']}, - 'that_field': {'type': 'dict', - 'excludes': 'this_field'}, - 'bazo_field': {'type': 'dict'}} + schema = { + 'this_field': {'type': 'dict', 'excludes': ['that_field', 'bazo_field']}, + 'that_field': {'type': 'dict', 'excludes': 'this_field'}, + 'bazo_field': {'type': 'dict'}, + } assert_success({'this_field': {}}, schema) assert_success({'that_field': {}}, schema) assert_fail({'this_field': {}, 'that_field': {}}, schema) @@ -1471,21 +1706,28 @@ def test_multiples_exclusions(): def test_bad_excludes_fields(validator): - validator.schema = {'this_field': {'type': 'dict', - 'excludes': ['that_field', 'bazo_field'], - 'required': True}, - 'that_field': {'type': 'dict', - 'excludes': 'this_field', - 'required': True}} + validator.schema = { + 'this_field': { + 'type': 'dict', + 'excludes': ['that_field', 'bazo_field'], + 'required': True, + }, + 'that_field': {'type': 'dict', 'excludes': 'this_field', 'required': True}, + } assert_fail({'that_field': {}, 'this_field': {}}, validator=validator) handler = errors.BasicErrorHandler - assert (validator.errors == - {'that_field': - [handler.messages[errors.EXCLUDES_FIELD.code].format( - "'this_field'", field="that_field")], - 'this_field': - [handler.messages[errors.EXCLUDES_FIELD.code].format( - "'that_field', 'bazo_field'", field="this_field")]}) + assert validator.errors == { + 'that_field': [ + handler.messages[errors.EXCLUDES_FIELD.code].format( + "'this_field'", field="that_field" + ) + ], + 'this_field': [ + handler.messages[errors.EXCLUDES_FIELD.code].format( + "'that_field', 'bazo_field'", field="this_field" + ) + ], + } def test_boolean_is_not_a_number(): @@ -1511,17 +1753,29 @@ def test_forbidden(): assert_success({'user': 'alice'}, schema) +def test_forbidden_number(): + schema = {'amount': {'forbidden': (0, 0.0)}} + assert_fail({'amount': 0}, schema) + assert_fail({'amount': 0.0}, schema) + + def test_mapping_with_sequence_schema(): schema = {'list': {'schema': {'allowed': ['a', 'b', 'c']}}} document = {'list': {'is_a': 'mapping'}} - assert_fail(document, schema, - error=('list', ('list', 'schema'), errors.BAD_TYPE_FOR_SCHEMA, - schema['list']['schema'])) + assert_fail( + document, + schema, + error=( + 'list', + ('list', 'schema'), + errors.BAD_TYPE_FOR_SCHEMA, + schema['list']['schema'], + ), + ) def test_sequence_with_mapping_schema(): - schema = {'list': {'schema': {'foo': {'allowed': ['a', 'b', 'c']}}, - 'type': 'dict'}} + schema = {'list': {'schema': {'foo': {'allowed': ['a', 'b', 'c']}}, 'type': 'dict'}} document = {'list': ['a', 'b', 'c']} assert_fail(document, schema) @@ -1529,19 +1783,24 @@ def test_sequence_with_mapping_schema(): def test_type_error_aborts_validation(): schema = {'foo': {'type': 'string', 'allowed': ['a']}} document = {'foo': 0} - assert_fail(document, schema, - error=('foo', ('foo', 'type'), errors.BAD_TYPE, 'string')) + assert_fail( + document, schema, error=('foo', ('foo', 'type'), errors.BAD_TYPE, 'string') + ) def test_dependencies_in_oneof(): # https://github.com/pyeve/cerberus/issues/241 - schema = {'a': {'type': 'integer', - 'oneof': [ - {'allowed': [1], 'dependencies': 'b'}, - {'allowed': [2], 'dependencies': 'c'} - ]}, - 'b': {}, - 'c': {}} + schema = { + 'a': { + 'type': 'integer', + 'oneof': [ + {'allowed': [1], 'dependencies': 'b'}, + {'allowed': [2], 'dependencies': 'c'}, + ], + }, + 'b': {}, + 'c': {}, + } assert_success({'a': 1, 'b': 'foo'}, schema) assert_success({'a': 2, 'c': 'bar'}, schema) assert_fail({'a': 1, 'c': 'foo'}, schema) @@ -1556,12 +1815,9 @@ def test_allow_unknown_with_oneof_rules(validator): { 'type': 'dict', 'allow_unknown': True, - 'schema': {'known': {'type': 'string'}} - }, - { - 'type': 'dict', - 'schema': {'known': {'type': 'string'}} + 'schema': {'known': {'type': 'string'}}, }, + {'type': 'dict', 'schema': {'known': {'type': 'string'}}}, ] } } @@ -1571,9 +1827,122 @@ def test_allow_unknown_with_oneof_rules(validator): validator(document, schema) _errors = validator._errors assert len(_errors) == 1 - assert_has_error(_errors, 'test', ('test', 'oneof'), - errors.ONEOF, schema['test']['oneof']) + assert_has_error( + _errors, 'test', ('test', 'oneof'), errors.ONEOF, schema['test']['oneof'] + ) assert len(_errors[0].child_errors) == 0 # check that allow_unknown is actually applied document = {'test': {'known': 's', 'unknown': 'asd'}} assert_success(document, validator=validator) + + +@mark.parametrize('constraint', (('Graham Chapman', 'Eric Idle'), 'Terry Gilliam')) +def test_contains(constraint): + validator = Validator({'actors': {'contains': constraint}}) + + document = {'actors': ('Graham Chapman', 'Eric Idle', 'Terry Gilliam')} + assert validator(document) + + document = {'actors': ('Eric idle', 'Terry Jones', 'John Cleese', 'Michael Palin')} + assert not validator(document) + assert errors.MISSING_MEMBERS in validator.document_error_tree['actors'] + missing_actors = validator.document_error_tree['actors'][ + errors.MISSING_MEMBERS + ].info[0] + assert any(x in missing_actors for x in ('Eric Idle', 'Terry Gilliam')) + + +def test_require_all_simple(): + schema = {'foo': {'type': 'string'}} + validator = Validator(require_all=True) + assert_fail( + {}, + schema, + validator, + error=('foo', '__require_all__', errors.REQUIRED_FIELD, True), + ) + assert_success({'foo': 'bar'}, schema, validator) + validator.require_all = False + assert_success({}, schema, validator) + assert_success({'foo': 'bar'}, schema, validator) + + +def test_require_all_override_by_required(): + schema = {'foo': {'type': 'string', 'required': False}} + validator = Validator(require_all=True) + assert_success({}, schema, validator) + assert_success({'foo': 'bar'}, schema, validator) + validator.require_all = False + assert_success({}, schema, validator) + assert_success({'foo': 'bar'}, schema, validator) + + schema = {'foo': {'type': 'string', 'required': True}} + validator.require_all = True + assert_fail( + {}, + schema, + validator, + error=('foo', ('foo', 'required'), errors.REQUIRED_FIELD, True), + ) + assert_success({'foo': 'bar'}, schema, validator) + validator.require_all = False + assert_fail( + {}, + schema, + validator, + error=('foo', ('foo', 'required'), errors.REQUIRED_FIELD, True), + ) + assert_success({'foo': 'bar'}, schema, validator) + + +@mark.parametrize( + "validator_require_all, sub_doc_require_all", + list(itertools.product([True, False], repeat=2)), +) +def test_require_all_override_by_subdoc_require_all( + validator_require_all, sub_doc_require_all +): + sub_schema = {"bar": {"type": "string"}} + schema = { + "foo": { + "type": "dict", + "require_all": sub_doc_require_all, + "schema": sub_schema, + } + } + validator = Validator(require_all=validator_require_all) + + assert_success({"foo": {"bar": "baz"}}, schema, validator) + if validator_require_all: + assert_fail({}, schema, validator) + else: + assert_success({}, schema, validator) + if sub_doc_require_all: + assert_fail({"foo": {}}, schema, validator) + else: + assert_success({"foo": {}}, schema, validator) + + +def test_require_all_and_exclude(): + schema = { + 'foo': {'type': 'string', 'excludes': 'bar'}, + 'bar': {'type': 'string', 'excludes': 'foo'}, + } + validator = Validator(require_all=True) + assert_fail( + {}, + schema, + validator, + errors=[ + ('foo', '__require_all__', errors.REQUIRED_FIELD, True), + ('bar', '__require_all__', errors.REQUIRED_FIELD, True), + ], + ) + assert_success({'foo': 'value'}, schema, validator) + assert_success({'bar': 'value'}, schema, validator) + assert_fail({'foo': 'value', 'bar': 'value'}, schema, validator) + validator.require_all = False + assert_success({}, schema, validator) + assert_success({'foo': 'value'}, schema, validator) + assert_success({'bar': 'value'}, schema, validator) + assert_fail({'foo': 'value', 'bar': 'value'}, schema, validator) diff --git a/pipenv/vendor/cerberus/utils.py b/pipenv/vendor/cerberus/utils.py index f10d3976..5a015d64 100644 --- a/pipenv/vendor/cerberus/utils.py +++ b/pipenv/vendor/cerberus/utils.py @@ -1,12 +1,11 @@ from __future__ import absolute_import -from collections import Mapping, namedtuple, Sequence +from collections import namedtuple -from cerberus.platform import _int_types, _str_type +from cerberus.platform import _int_types, _str_type, Mapping, Sequence, Set -TypeDefinition = namedtuple('TypeDefinition', - 'name,included_types,excluded_types') +TypeDefinition = namedtuple('TypeDefinition', 'name,included_types,excluded_types') """ This class is used to define types that can be used as value in the :attr:`~cerberus.Validator.types_mapping` property. @@ -19,19 +18,33 @@ contained in ``excluded_types``. def compare_paths_lt(x, y): - for i in range(min(len(x), len(y))): - if isinstance(x[i], type(y[i])): - if x[i] != y[i]: - return x[i] < y[i] - elif isinstance(x[i], _int_types): + min_length = min(len(x), len(y)) + + if x[:min_length] == y[:min_length]: + return len(x) == min_length + + for i in range(min_length): + a, b = x[i], y[i] + + for _type in (_int_types, _str_type, tuple): + if isinstance(a, _type): + if isinstance(b, _type): + break + else: + return True + + if a == b: + continue + elif a < b: return True - elif isinstance(y[i], _int_types): + else: return False - return len(x) < len(y) + + raise RuntimeError def drop_item_from_tuple(t, i): - return t[:i] + t[i + 1:] + return t[:i] + t[i + 1 :] def get_Validator_class(): @@ -50,26 +63,24 @@ def mapping_to_frozenset(mapping): equal. As it is used to identify equality of schemas, this can be considered okay as definitions are semantically equal regardless the container type. """ - mapping = mapping.copy() + + aggregation = {} + for key, value in mapping.items(): if isinstance(value, Mapping): - mapping[key] = mapping_to_frozenset(value) + aggregation[key] = mapping_to_frozenset(value) elif isinstance(value, Sequence): value = list(value) for i, item in enumerate(value): if isinstance(item, Mapping): value[i] = mapping_to_frozenset(item) - mapping[key] = tuple(value) - return frozenset(mapping.items()) + aggregation[key] = tuple(value) + elif isinstance(value, Set): + aggregation[key] = frozenset(value) + else: + aggregation[key] = value - -def isclass(obj): - try: - issubclass(obj, object) - except TypeError: - return False - else: - return True + return frozenset(aggregation.items()) def quote_string(value): diff --git a/pipenv/vendor/cerberus/validator.py b/pipenv/vendor/cerberus/validator.py index 27a29053..ed1c1536 100644 --- a/pipenv/vendor/cerberus/validator.py +++ b/pipenv/vendor/cerberus/validator.py @@ -11,28 +11,41 @@ from __future__ import absolute_import from ast import literal_eval -from collections import Hashable, Iterable, Mapping, Sequence from copy import copy from datetime import date, datetime import re from warnings import warn from cerberus import errors -from cerberus.platform import _int_types, _str_type -from cerberus.schema import (schema_registry, rules_set_registry, - DefinitionSchema, SchemaError) -from cerberus.utils import (drop_item_from_tuple, isclass, - readonly_classproperty, TypeDefinition) - +from cerberus.platform import ( + _int_types, + _str_type, + Container, + Hashable, + Iterable, + Mapping, + Sequence, + Sized, +) +from cerberus.schema import ( + schema_registry, + rules_set_registry, + DefinitionSchema, + SchemaError, +) +from cerberus.utils import drop_item_from_tuple, readonly_classproperty, TypeDefinition toy_error_handler = errors.ToyErrorHandler() def dummy_for_rule_validation(rule_constraints): def dummy(self, constraint, field, value): - raise RuntimeError('Dummy method called. Its purpose is to hold just' - 'validation constraints for a rule in its ' - 'docstring.') + raise RuntimeError( + 'Dummy method called. Its purpose is to hold just' + 'validation constraints for a rule in its ' + 'docstring.' + ) + f = dummy f.__doc__ = rule_constraints return f @@ -40,12 +53,14 @@ def dummy_for_rule_validation(rule_constraints): class DocumentError(Exception): """ Raised when the target document is missing or has the wrong format """ + pass class _SchemaRuleTypeError(Exception): """ Raised when a schema (list) validation encounters a mapping. Not supposed to be used outside this module. """ + pass @@ -76,9 +91,15 @@ class BareValidator(object): :param allow_unknown: See :attr:`~cerberus.Validator.allow_unknown`. Defaults to ``False``. :type allow_unknown: :class:`bool` or any :term:`mapping` + :param require_all: See :attr:`~cerberus.Validator.require_all`. + Defaults to ``False``. + :type require_all: :class:`bool` :param purge_unknown: See :attr:`~cerberus.Validator.purge_unknown`. Defaults to to ``False``. :type purge_unknown: :class:`bool` + :param purge_readonly: Removes all fields that are defined as ``readonly`` in the + normalization phase. + :type purge_readonly: :class:`bool` :param error_handler: The error handler that formats the result of :attr:`~cerberus.Validator.errors`. When given as two-value tuple with an error-handler @@ -98,28 +119,18 @@ class BareValidator(object): """ Rules that will be processed in that order before any other. Type: :class:`tuple` """ types_mapping = { - 'binary': - TypeDefinition('binary', (bytes, bytearray), ()), - 'boolean': - TypeDefinition('boolean', (bool,), ()), - 'date': - TypeDefinition('date', (date,), ()), - 'datetime': - TypeDefinition('datetime', (datetime,), ()), - 'dict': - TypeDefinition('dict', (Mapping,), ()), - 'float': - TypeDefinition('float', (float, _int_types), ()), - 'integer': - TypeDefinition('integer', (_int_types,), ()), - 'list': - TypeDefinition('list', (Sequence,), (_str_type,)), - 'number': - TypeDefinition('number', (_int_types, float), (bool,)), - 'set': - TypeDefinition('set', (set,), ()), - 'string': - TypeDefinition('string', (_str_type), ()) + 'binary': TypeDefinition('binary', (bytes, bytearray), ()), + 'boolean': TypeDefinition('boolean', (bool,), ()), + 'container': TypeDefinition('container', (Container,), (_str_type,)), + 'date': TypeDefinition('date', (date,), ()), + 'datetime': TypeDefinition('datetime', (datetime,), ()), + 'dict': TypeDefinition('dict', (Mapping,), ()), + 'float': TypeDefinition('float', (float, _int_types), ()), + 'integer': TypeDefinition('integer', (_int_types,), ()), + 'list': TypeDefinition('list', (Sequence,), (_str_type,)), + 'number': TypeDefinition('number', (_int_types, float), (bool,)), + 'set': TypeDefinition('set', (set,), ()), + 'string': TypeDefinition('string', (_str_type), ()), } """ This mapping holds all available constraints for the type rule and their assigned :class:`~cerberus.TypeDefinition`. """ @@ -131,7 +142,8 @@ class BareValidator(object): """ The arguments will be treated as with this signature: __init__(self, schema=None, ignore_none_values=False, - allow_unknown=False, purge_unknown=False, + allow_unknown=False, require_all=False, + purge_unknown=False, purge_readonly=False, error_handler=errors.BasicErrorHandler) """ @@ -168,6 +180,7 @@ class BareValidator(object): self.__store_config(args, kwargs) self.schema = kwargs.get('schema', None) self.allow_unknown = kwargs.get('allow_unknown', False) + self.require_all = kwargs.get('require_all', False) self._remaining_rules = [] """ Keeps track of the rules that are next in line to be evaluated during the validation of a field. @@ -182,8 +195,9 @@ class BareValidator(object): error_handler, eh_config = error_handler else: eh_config = {} - if isclass(error_handler) and \ - issubclass(error_handler, errors.BaseErrorHandler): + if isinstance(error_handler, type) and issubclass( + error_handler, errors.BaseErrorHandler + ): return error_handler(**eh_config) elif isinstance(error_handler, errors.BaseErrorHandler): return error_handler @@ -192,12 +206,17 @@ class BareValidator(object): def __store_config(self, args, kwargs): """ Assign args to kwargs and store configuration. """ - signature = ('schema', 'ignore_none_values', 'allow_unknown', - 'purge_unknown') - for i, p in enumerate(signature[:len(args)]): + signature = ( + 'schema', + 'ignore_none_values', + 'allow_unknown', + 'require_all', + 'purge_unknown', + 'purge_readonly', + ) + for i, p in enumerate(signature[: len(args)]): if p in kwargs: - raise TypeError("__init__ got multiple values for argument " - "'%s'" % p) + raise TypeError("__init__ got multiple values for argument " "'%s'" % p) else: kwargs[p] = args[i] self._config = kwargs @@ -251,8 +270,8 @@ class BareValidator(object): self._errors.extend(args[0]) self._errors.sort() for error in args[0]: - self.document_error_tree += error - self.schema_error_tree += error + self.document_error_tree.add(error) + self.schema_error_tree.add(error) self.error_handler.emit(error) elif len(args) == 2 and isinstance(args[1], _str_type): self._error(args[0], errors.CUSTOM, args[1]) @@ -262,7 +281,7 @@ class BareValidator(object): rule = args[1].rule info = args[2:] - document_path = self.document_path + (field, ) + document_path = self.document_path + (field,) schema_path = self.schema_path if code != errors.UNKNOWN_FIELD.code and rule is not None: @@ -274,6 +293,10 @@ class BareValidator(object): field_definitions = self._resolve_rules_set(self.schema[field]) if rule == 'nullable': constraint = field_definitions.get(rule, False) + elif rule == 'required': + constraint = field_definitions.get(rule, self.require_all) + if rule not in field_definitions: + schema_path = "__require_all__" else: constraint = field_definitions[rule] @@ -284,8 +307,7 @@ class BareValidator(object): ) self._error([self.recent_error]) - def _get_child_validator(self, document_crumb=None, schema_crumb=None, - **kwargs): + def _get_child_validator(self, document_crumb=None, schema_crumb=None, **kwargs): """ Creates a new instance of Validator-(sub-)class. All initial parameters of the parent are passed to the initialization, unless a parameter is given as an explicit *keyword*-parameter. @@ -309,6 +331,7 @@ class BareValidator(object): child_config['is_child'] = True child_config['error_handler'] = toy_error_handler child_config['root_allow_unknown'] = self.allow_unknown + child_config['root_require_all'] = self.require_all child_config['root_document'] = self.document child_config['root_schema'] = self.schema @@ -318,14 +341,14 @@ class BareValidator(object): child_validator.document_path = self.document_path else: if not isinstance(document_crumb, tuple): - document_crumb = (document_crumb, ) + document_crumb = (document_crumb,) child_validator.document_path = self.document_path + document_crumb if schema_crumb is None: child_validator.schema_path = self.schema_path else: if not isinstance(schema_crumb, tuple): - schema_crumb = (schema_crumb, ) + schema_crumb = (schema_crumb,) child_validator.schema_path = self.schema_path + schema_crumb return child_validator @@ -334,8 +357,10 @@ class BareValidator(object): methodname = '_{0}_{1}'.format(domain, rule.replace(' ', '_')) result = getattr(self, methodname, None) if result is None: - raise RuntimeError("There's no handler for '{}' in the '{}' " - "domain.".format(rule, domain)) + raise RuntimeError( + "There's no handler for '{}' in the '{}' " + "domain.".format(rule, domain) + ) return result def _drop_nodes_from_errorpaths(self, _errors, dp_items, sp_items): @@ -351,14 +376,15 @@ class BareValidator(object): sp_basedepth = len(self.schema_path) for error in _errors: for i in sorted(dp_items, reverse=True): - error.document_path = \ - drop_item_from_tuple(error.document_path, dp_basedepth + i) + error.document_path = drop_item_from_tuple( + error.document_path, dp_basedepth + i + ) for i in sorted(sp_items, reverse=True): - error.schema_path = \ - drop_item_from_tuple(error.schema_path, sp_basedepth + i) + error.schema_path = drop_item_from_tuple( + error.schema_path, sp_basedepth + i + ) if error.child_errors: - self._drop_nodes_from_errorpaths(error.child_errors, - dp_items, sp_items) + self._drop_nodes_from_errorpaths(error.child_errors, dp_items, sp_items) def _lookup_field(self, path): """ Searches for a field as defined by path. This method is used by the @@ -377,8 +403,7 @@ class BareValidator(object): """ if path.startswith('^'): path = path[1:] - context = self.document if path.startswith('^') \ - else self.root_document + context = self.document if path.startswith('^') else self.root_document else: context = self.document @@ -386,7 +411,7 @@ class BareValidator(object): for part in parts: if part not in context: return None, None - context = context.get(part) + context = context.get(part, {}) return parts[-1], context @@ -421,6 +446,17 @@ class BareValidator(object): DefinitionSchema(self, {'allow_unknown': value}) self._config['allow_unknown'] = value + @property + def require_all(self): + """ If ``True`` known fields that are defined in the schema will + be required. + Type: :class:`bool` """ + return self._config.get('require_all', False) + + @require_all.setter + def require_all(self, value): + self._config['require_all'] = value + @property def errors(self): """ The errors of the last processing formatted by the handler that is @@ -455,7 +491,7 @@ class BareValidator(object): @property def purge_unknown(self): - """ If ``True`` unknown fields will be deleted from the document + """ If ``True``, unknown fields will be deleted from the document unless a validation is called with disabled normalization. Also see :ref:`purging-unknown-fields`. Type: :class:`bool` """ return self._config.get('purge_unknown', False) @@ -464,12 +500,29 @@ class BareValidator(object): def purge_unknown(self, value): self._config['purge_unknown'] = value + @property + def purge_readonly(self): + """ If ``True``, fields declared as readonly will be deleted from the + document unless a validation is called with disabled normalization. + Type: :class:`bool` """ + return self._config.get('purge_readonly', False) + + @purge_readonly.setter + def purge_readonly(self, value): + self._config['purge_readonly'] = value + @property def root_allow_unknown(self): """ The :attr:`~cerberus.Validator.allow_unknown` attribute of the first level ancestor of a child validator. """ return self._config.get('root_allow_unknown', self.allow_unknown) + @property + def root_require_all(self): + """ The :attr:`~cerberus.Validator.require_all` attribute of + the first level ancestor of a child validator. """ + return self._config.get('root_require_all', self.require_all) + @property def root_document(self): """ The :attr:`~cerberus.Validator.document` attribute of the @@ -524,12 +577,12 @@ class BareValidator(object): def types(cls): """ The constraints that can be used for the 'type' rule. Type: A tuple of strings. """ - redundant_types = \ - set(cls.types_mapping) & set(cls._types_from_methods) + redundant_types = set(cls.types_mapping) & set(cls._types_from_methods) if redundant_types: - warn("These types are defined both with a method and in the" - "'types_mapping' property of this validator: %s" - % redundant_types) + warn( + "These types are defined both with a method and in the" + "'types_mapping' property of this validator: %s" % redundant_types + ) return tuple(cls.types_mapping) + cls._types_from_methods @@ -554,8 +607,7 @@ class BareValidator(object): if document is None: raise DocumentError(errors.DOCUMENT_MISSING) if not isinstance(document, Mapping): - raise DocumentError( - errors.DOCUMENT_FORMAT.format(document)) + raise DocumentError(errors.DOCUMENT_FORMAT.format(document)) self.error_handler.start(self) def _drop_remaining_rules(self, *rules): @@ -608,6 +660,8 @@ class BareValidator(object): self.__normalize_rename_fields(mapping, schema) if self.purge_unknown and not self.allow_unknown: self._normalize_purge_unknown(mapping, schema) + if self.purge_readonly: + self.__normalize_purge_readonly(mapping, schema) # Check `readonly` fields before applying default values because # a field's schema definition might contain both `readonly` and # `default`. @@ -631,13 +685,23 @@ class BareValidator(object): for field in mapping: if field in schema and 'coerce' in schema[field]: mapping[field] = self.__normalize_coerce( - schema[field]['coerce'], field, mapping[field], - schema[field].get('nullable', False), error) - elif isinstance(self.allow_unknown, Mapping) and \ - 'coerce' in self.allow_unknown: + schema[field]['coerce'], + field, + mapping[field], + schema[field].get('nullable', False), + error, + ) + elif ( + isinstance(self.allow_unknown, Mapping) + and 'coerce' in self.allow_unknown + ): mapping[field] = self.__normalize_coerce( - self.allow_unknown['coerce'], field, mapping[field], - self.allow_unknown.get('nullable', False), error) + self.allow_unknown['coerce'], + field, + mapping[field], + self.allow_unknown.get('nullable', False), + error, + ) def __normalize_coerce(self, processor, field, value, nullable, error): if isinstance(processor, _str_type): @@ -646,52 +710,60 @@ class BareValidator(object): elif isinstance(processor, Iterable): result = value for p in processor: - result = self.__normalize_coerce(p, field, result, - nullable, error) - if errors.COERCION_FAILED in \ - self.document_error_tree.fetch_errors_from( - self.document_path + (field,)): + result = self.__normalize_coerce(p, field, result, nullable, error) + if ( + errors.COERCION_FAILED + in self.document_error_tree.fetch_errors_from( + self.document_path + (field,) + ) + ): break return result try: return processor(value) except Exception as e: - if not nullable and e is not TypeError: + if not (nullable and value is None): self._error(field, error, str(e)) return value def __normalize_containers(self, mapping, schema): for field in mapping: - if field not in schema: - continue + rules = set(schema.get(field, ())) + # TODO: This check conflates validation and normalization if isinstance(mapping[field], Mapping): - if 'keyschema' in schema[field]: - self.__normalize_mapping_per_keyschema( - field, mapping, schema[field]['keyschema']) - if 'valueschema' in schema[field]: - self.__normalize_mapping_per_valueschema( - field, mapping, schema[field]['valueschema']) - if set(schema[field]) & set(('allow_unknown', 'purge_unknown', - 'schema')): + if 'keysrules' in rules: + self.__normalize_mapping_per_keysrules( + field, mapping, schema[field]['keysrules'] + ) + if 'valuesrules' in rules: + self.__normalize_mapping_per_valuesrules( + field, mapping, schema[field]['valuesrules'] + ) + if rules & set( + ('allow_unknown', 'purge_unknown', 'schema') + ) or isinstance(self.allow_unknown, Mapping): try: - self.__normalize_mapping_per_schema( - field, mapping, schema) + self.__normalize_mapping_per_schema(field, mapping, schema) except _SchemaRuleTypeError: pass + elif isinstance(mapping[field], _str_type): continue - elif isinstance(mapping[field], Sequence) and \ - 'schema' in schema[field]: - self.__normalize_sequence(field, mapping, schema) - def __normalize_mapping_per_keyschema(self, field, mapping, property_rules): + elif isinstance(mapping[field], Sequence): + if 'schema' in rules: + self.__normalize_sequence_per_schema(field, mapping, schema) + elif 'items' in rules: + self.__normalize_sequence_per_items(field, mapping, schema) + + def __normalize_mapping_per_keysrules(self, field, mapping, property_rules): schema = dict(((k, property_rules) for k in mapping[field])) document = dict(((k, k) for k in mapping[field])) validator = self._get_child_validator( - document_crumb=field, schema_crumb=(field, 'keyschema'), - schema=schema) + document_crumb=field, schema_crumb=(field, 'keysrules'), schema=schema + ) result = validator.normalized(document, always_return_document=True) if validator._errors: self._drop_nodes_from_errorpaths(validator._errors, [], [2, 4]) @@ -700,46 +772,72 @@ class BareValidator(object): if k == result[k]: continue if result[k] in mapping[field]: - warn("Normalizing keys of {path}: {key} already exists, " - "its value is replaced." - .format(path='.'.join(self.document_path + (field,)), - key=k)) + warn( + "Normalizing keys of {path}: {key} already exists, " + "its value is replaced.".format( + path='.'.join(str(x) for x in self.document_path + (field,)), + key=k, + ) + ) mapping[field][result[k]] = mapping[field][k] else: mapping[field][result[k]] = mapping[field][k] del mapping[field][k] - def __normalize_mapping_per_valueschema(self, field, mapping, value_rules): + def __normalize_mapping_per_valuesrules(self, field, mapping, value_rules): schema = dict(((k, value_rules) for k in mapping[field])) validator = self._get_child_validator( - document_crumb=field, schema_crumb=(field, 'valueschema'), - schema=schema) - mapping[field] = validator.normalized(mapping[field], - always_return_document=True) + document_crumb=field, schema_crumb=(field, 'valuesrules'), schema=schema + ) + mapping[field] = validator.normalized( + mapping[field], always_return_document=True + ) if validator._errors: self._drop_nodes_from_errorpaths(validator._errors, [], [2]) self._error(validator._errors) def __normalize_mapping_per_schema(self, field, mapping, schema): + rules = schema.get(field, {}) + if not rules and isinstance(self.allow_unknown, Mapping): + rules = self.allow_unknown validator = self._get_child_validator( - document_crumb=field, schema_crumb=(field, 'schema'), - schema=schema[field].get('schema', {}), - allow_unknown=schema[field].get('allow_unknown', self.allow_unknown), # noqa: E501 - purge_unknown=schema[field].get('purge_unknown', self.purge_unknown)) # noqa: E501 + document_crumb=field, + schema_crumb=(field, 'schema'), + schema=rules.get('schema', {}), + allow_unknown=rules.get('allow_unknown', self.allow_unknown), # noqa: E501 + purge_unknown=rules.get('purge_unknown', self.purge_unknown), + require_all=rules.get('require_all', self.require_all), + ) # noqa: E501 value_type = type(mapping[field]) - result_value = validator.normalized(mapping[field], - always_return_document=True) + result_value = validator.normalized(mapping[field], always_return_document=True) mapping[field] = value_type(result_value) if validator._errors: self._error(validator._errors) - def __normalize_sequence(self, field, mapping, schema): - schema = dict(((k, schema[field]['schema']) - for k in range(len(mapping[field])))) + def __normalize_sequence_per_schema(self, field, mapping, schema): + schema = dict( + ((k, schema[field]['schema']) for k in range(len(mapping[field]))) + ) document = dict((k, v) for k, v in enumerate(mapping[field])) validator = self._get_child_validator( - document_crumb=field, schema_crumb=(field, 'schema'), - schema=schema) + document_crumb=field, schema_crumb=(field, 'schema'), schema=schema + ) + value_type = type(mapping[field]) + result = validator.normalized(document, always_return_document=True) + mapping[field] = value_type(result.values()) + if validator._errors: + self._drop_nodes_from_errorpaths(validator._errors, [], [2]) + self._error(validator._errors) + + def __normalize_sequence_per_items(self, field, mapping, schema): + rules, values = schema[field]['items'], mapping[field] + if len(rules) != len(values): + return + schema = dict(((k, v) for k, v in enumerate(rules))) + document = dict((k, v) for k, v in enumerate(values)) + validator = self._get_child_validator( + document_crumb=field, schema_crumb=(field, 'items'), schema=schema + ) value_type = type(mapping[field]) result = validator.normalized(document, always_return_document=True) mapping[field] = value_type(result.values()) @@ -747,12 +845,17 @@ class BareValidator(object): self._drop_nodes_from_errorpaths(validator._errors, [], [2]) self._error(validator._errors) + @staticmethod + def __normalize_purge_readonly(mapping, schema): + for field in [x for x in mapping if schema.get(x, {}).get('readonly', False)]: + mapping.pop(field) + return mapping + @staticmethod def _normalize_purge_unknown(mapping, schema): """ {'type': 'boolean'} """ - for field in tuple(mapping): - if field not in schema: - del mapping[field] + for field in [x for x in mapping if x not in schema]: + mapping.pop(field) return mapping def __normalize_rename_fields(self, mapping, schema): @@ -760,10 +863,13 @@ class BareValidator(object): if field in schema: self._normalize_rename(mapping, schema, field) self._normalize_rename_handler(mapping, schema, field) - elif isinstance(self.allow_unknown, Mapping) and \ - 'rename_handler' in self.allow_unknown: + elif ( + isinstance(self.allow_unknown, Mapping) + and 'rename_handler' in self.allow_unknown + ): self._normalize_rename_handler( - mapping, {field: self.allow_unknown}, field) + mapping, {field: self.allow_unknown}, field + ) return mapping def _normalize_rename(self, mapping, schema, field): @@ -783,47 +889,62 @@ class BareValidator(object): if 'rename_handler' not in schema[field]: return new_name = self.__normalize_coerce( - schema[field]['rename_handler'], field, field, - False, errors.RENAMING_FAILED) + schema[field]['rename_handler'], field, field, False, errors.RENAMING_FAILED + ) if new_name != field: mapping[new_name] = mapping[field] del mapping[field] def __validate_readonly_fields(self, mapping, schema): - for field in (x for x in schema if x in mapping and - self._resolve_rules_set(schema[x]).get('readonly')): - self._validate_readonly(schema[field]['readonly'], field, - mapping[field]) + for field in ( + x + for x in schema + if x in mapping and self._resolve_rules_set(schema[x]).get('readonly') + ): + self._validate_readonly(schema[field]['readonly'], field, mapping[field]) def __normalize_default_fields(self, mapping, schema): - fields = [x for x in schema if x not in mapping or - mapping[x] is None and not schema[x].get('nullable', False)] + empty_fields = [ + x + for x in schema + if x not in mapping + or ( + mapping[x] is None # noqa: W503 + and not schema[x].get('nullable', False) + ) # noqa: W503 + ] + try: - fields_with_default = [x for x in fields if 'default' in schema[x]] + fields_with_default = [x for x in empty_fields if 'default' in schema[x]] except TypeError: raise _SchemaRuleTypeError for field in fields_with_default: self._normalize_default(mapping, schema, field) known_fields_states = set() - fields = [x for x in fields if 'default_setter' in schema[x]] - while fields: - field = fields.pop(0) + fields_with_default_setter = [ + x for x in empty_fields if 'default_setter' in schema[x] + ] + while fields_with_default_setter: + field = fields_with_default_setter.pop(0) try: self._normalize_default_setter(mapping, schema, field) except KeyError: - fields.append(field) + fields_with_default_setter.append(field) except Exception as e: self._error(field, errors.SETTING_DEFAULT_FAILED, str(e)) - fields_state = tuple(fields) - if fields_state in known_fields_states: - for field in fields: - self._error(field, errors.SETTING_DEFAULT_FAILED, - 'Circular dependencies of default setters.') + fields_processing_state = hash(tuple(fields_with_default_setter)) + if fields_processing_state in known_fields_states: + for field in fields_with_default_setter: + self._error( + field, + errors.SETTING_DEFAULT_FAILED, + 'Circular dependencies of default setters.', + ) break else: - known_fields_states.add(fields_state) + known_fields_states.add(fields_processing_state) def _normalize_default(self, mapping, schema, field): """ {'nullable': True} """ @@ -837,8 +958,7 @@ class BareValidator(object): if 'default_setter' in schema[field]: setter = schema[field]['default_setter'] if isinstance(setter, _str_type): - setter = self.__get_rule_handler('normalize_default_setter', - setter) + setter = self.__get_rule_handler('normalize_default_setter', setter) mapping[field] = setter(mapping) # # Validating @@ -904,11 +1024,10 @@ class BareValidator(object): if isinstance(self.allow_unknown, (Mapping, _str_type)): # validate that unknown fields matches the schema # for unknown_fields - schema_crumb = 'allow_unknown' if self.is_child \ - else '__allow_unknown__' + schema_crumb = 'allow_unknown' if self.is_child else '__allow_unknown__' validator = self._get_child_validator( - schema_crumb=schema_crumb, - schema={field: self.allow_unknown}) + schema_crumb=schema_crumb, schema={field: self.allow_unknown} + ) if not validator({field: value}, normalize=False): self._error(validator._errors) else: @@ -924,14 +1043,21 @@ class BareValidator(object): definitions = self._resolve_rules_set(definitions) value = self.document[field] - rules_queue = [x for x in self.priority_validations - if x in definitions or x in self.mandatory_validations] - rules_queue.extend(x for x in self.mandatory_validations - if x not in rules_queue) - rules_queue.extend(x for x in definitions - if x not in rules_queue and - x not in self.normalization_rules and - x not in ('allow_unknown', 'required')) + rules_queue = [ + x + for x in self.priority_validations + if x in definitions or x in self.mandatory_validations + ] + rules_queue.extend( + x for x in self.mandatory_validations if x not in rules_queue + ) + rules_queue.extend( + x + for x in definitions + if x not in rules_queue + and x not in self.normalization_rules + and x not in ('allow_unknown', 'require_all', 'meta', 'required') + ) self._remaining_rules = rules_queue while self._remaining_rules: @@ -952,10 +1078,11 @@ class BareValidator(object): _validate_allow_unknown = dummy_for_rule_validation( """ {'oneof': [{'type': 'boolean'}, {'type': ['dict', 'string'], - 'validator': 'bulk_schema'}]} """) + 'check_with': 'bulk_schema'}]} """ + ) def _validate_allowed(self, allowed_values, field, value): - """ {'type': 'list'} """ + """ {'type': 'container'} """ if isinstance(value, Iterable) and not isinstance(value, _str_type): unallowed = set(value) - set(allowed_values) if unallowed: @@ -964,10 +1091,54 @@ class BareValidator(object): if value not in allowed_values: self._error(field, errors.UNALLOWED_VALUE, value) + def _validate_check_with(self, checks, field, value): + """ {'oneof': [ + {'type': 'callable'}, + {'type': 'list', + 'schema': {'oneof': [{'type': 'callable'}, + {'type': 'string'}]}}, + {'type': 'string'} + ]} """ + if isinstance(checks, _str_type): + try: + value_checker = self.__get_rule_handler('check_with', checks) + # TODO remove on next major release + except RuntimeError: + value_checker = self.__get_rule_handler('validator', checks) + warn( + "The 'validator' rule was renamed to 'check_with'. Please update " + "your schema and method names accordingly.", + DeprecationWarning, + ) + value_checker(field, value) + elif isinstance(checks, Iterable): + for v in checks: + self._validate_check_with(v, field, value) + else: + checks(field, value, self._error) + + def _validate_contains(self, expected_values, field, value): + """ {'empty': False } """ + if not isinstance(value, Iterable): + return + + if not isinstance(expected_values, Iterable) or isinstance( + expected_values, _str_type + ): + expected_values = set((expected_values,)) + else: + expected_values = set(expected_values) + + missing_values = expected_values - set(value) + if missing_values: + self._error(field, errors.MISSING_MEMBERS, missing_values) + def _validate_dependencies(self, dependencies, field, value): """ {'type': ('dict', 'hashable', 'list'), - 'validator': 'dependencies'} """ - if isinstance(dependencies, _str_type): + 'check_with': 'dependencies'} """ + if isinstance(dependencies, _str_type) or not isinstance( + dependencies, (Iterable, Mapping) + ): dependencies = (dependencies,) if isinstance(dependencies, Sequence): @@ -975,20 +1146,24 @@ class BareValidator(object): elif isinstance(dependencies, Mapping): self.__validate_dependencies_mapping(dependencies, field) - if self.document_error_tree.fetch_node_from( - self.schema_path + (field, 'dependencies')) is not None: + if ( + self.document_error_tree.fetch_node_from( + self.schema_path + (field, 'dependencies') + ) + is not None + ): return True def __validate_dependencies_mapping(self, dependencies, field): validated_dependencies_counter = 0 error_info = {} for dependency_name, dependency_values in dependencies.items(): - if (not isinstance(dependency_values, Sequence) or - isinstance(dependency_values, _str_type)): + if not isinstance(dependency_values, Sequence) or isinstance( + dependency_values, _str_type + ): dependency_values = [dependency_values] - wanted_field, wanted_field_value = \ - self._lookup_field(dependency_name) + wanted_field, wanted_field_value = self._lookup_field(dependency_name) if wanted_field_value in dependency_values: validated_dependencies_counter += 1 else: @@ -1004,59 +1179,71 @@ class BareValidator(object): def _validate_empty(self, empty, field, value): """ {'type': 'boolean'} """ - if isinstance(value, Iterable) and len(value) == 0: + if isinstance(value, Sized) and len(value) == 0: self._drop_remaining_rules( - 'allowed', 'forbidden', 'items', 'minlength', 'maxlength', - 'regex', 'validator') + 'allowed', + 'forbidden', + 'items', + 'minlength', + 'maxlength', + 'regex', + 'check_with', + ) if not empty: self._error(field, errors.EMPTY_NOT_ALLOWED) - def _validate_excludes(self, excludes, field, value): + def _validate_excludes(self, excluded_fields, field, value): """ {'type': ('hashable', 'list'), 'schema': {'type': 'hashable'}} """ - if isinstance(excludes, Hashable): - excludes = [excludes] + if isinstance(excluded_fields, Hashable): + excluded_fields = [excluded_fields] - # Save required field to be checked latter - if 'required' in self.schema[field] and self.schema[field]['required']: + # Mark the currently evaluated field as not required for now if it actually is. + # One of the so marked will be needed to pass when required fields are checked. + if self.schema[field].get('required', self.require_all): self._unrequired_by_excludes.add(field) - for exclude in excludes: - if (exclude in self.schema and - 'required' in self.schema[exclude] and - self.schema[exclude]['required']): - self._unrequired_by_excludes.add(exclude) + for excluded_field in excluded_fields: + if excluded_field in self.schema and self.schema[field].get( + 'required', self.require_all + ): - if [True for key in excludes if key in self.document]: - # Wrap each field in `excludes` list between quotes - exclusion_str = ', '.join("'{0}'" - .format(word) for word in excludes) + self._unrequired_by_excludes.add(excluded_field) + + if any(excluded_field in self.document for excluded_field in excluded_fields): + exclusion_str = ', '.join( + "'{0}'".format(field) for field in excluded_fields + ) self._error(field, errors.EXCLUDES_FIELD, exclusion_str) def _validate_forbidden(self, forbidden_values, field, value): """ {'type': 'list'} """ - if isinstance(value, _str_type): - if value in forbidden_values: - self._error(field, errors.FORBIDDEN_VALUE, value) - elif isinstance(value, Sequence): + if isinstance(value, Sequence) and not isinstance(value, _str_type): forbidden = set(value) & set(forbidden_values) if forbidden: self._error(field, errors.FORBIDDEN_VALUES, list(forbidden)) - elif isinstance(value, int): + else: if value in forbidden_values: self._error(field, errors.FORBIDDEN_VALUE, value) def _validate_items(self, items, field, values): - """ {'type': 'list', 'validator': 'items'} """ + """ {'type': 'list', 'check_with': 'items'} """ if len(items) != len(values): self._error(field, errors.ITEMS_LENGTH, len(items), len(values)) else: - schema = dict((i, definition) for i, definition in enumerate(items)) # noqa: E501 - validator = self._get_child_validator(document_crumb=field, - schema_crumb=(field, 'items'), # noqa: E501 - schema=schema) - if not validator(dict((i, value) for i, value in enumerate(values)), - update=self.update, normalize=False): + schema = dict( + (i, definition) for i, definition in enumerate(items) + ) # noqa: E501 + validator = self._get_child_validator( + document_crumb=field, + schema_crumb=(field, 'items'), # noqa: E501 + schema=schema, + ) + if not validator( + dict((i, value) for i, value in enumerate(values)), + update=self.update, + normalize=False, + ): self._error(field, errors.BAD_ITEMS, validator._errors) def __validate_logical(self, operator, definitions, field, value): @@ -1074,8 +1261,8 @@ class BareValidator(object): schema[field]['allow_unknown'] = self.allow_unknown validator = self._get_child_validator( - schema_crumb=(field, operator, i), - schema=schema, allow_unknown=True) + schema_crumb=(field, operator, i), schema=schema, allow_unknown=True + ) if validator(self.document, update=self.update, normalize=False): valid_counter += 1 else: @@ -1086,35 +1273,27 @@ class BareValidator(object): def _validate_anyof(self, definitions, field, value): """ {'type': 'list', 'logical': 'anyof'} """ - valids, _errors = \ - self.__validate_logical('anyof', definitions, field, value) + valids, _errors = self.__validate_logical('anyof', definitions, field, value) if valids < 1: - self._error(field, errors.ANYOF, _errors, - valids, len(definitions)) + self._error(field, errors.ANYOF, _errors, valids, len(definitions)) def _validate_allof(self, definitions, field, value): """ {'type': 'list', 'logical': 'allof'} """ - valids, _errors = \ - self.__validate_logical('allof', definitions, field, value) + valids, _errors = self.__validate_logical('allof', definitions, field, value) if valids < len(definitions): - self._error(field, errors.ALLOF, _errors, - valids, len(definitions)) + self._error(field, errors.ALLOF, _errors, valids, len(definitions)) def _validate_noneof(self, definitions, field, value): """ {'type': 'list', 'logical': 'noneof'} """ - valids, _errors = \ - self.__validate_logical('noneof', definitions, field, value) + valids, _errors = self.__validate_logical('noneof', definitions, field, value) if valids > 0: - self._error(field, errors.NONEOF, _errors, - valids, len(definitions)) + self._error(field, errors.NONEOF, _errors, valids, len(definitions)) def _validate_oneof(self, definitions, field, value): """ {'type': 'list', 'logical': 'oneof'} """ - valids, _errors = \ - self.__validate_logical('oneof', definitions, field, value) + valids, _errors = self.__validate_logical('oneof', definitions, field, value) if valids != 1: - self._error(field, errors.ONEOF, _errors, - valids, len(definitions)) + self._error(field, errors.ONEOF, _errors, valids, len(definitions)) def _validate_max(self, max_value, field, value): """ {'nullable': False } """ @@ -1137,6 +1316,8 @@ class BareValidator(object): if isinstance(value, Iterable) and len(value) > max_length: self._error(field, errors.MAX_LENGTH, len(value)) + _validate_meta = dummy_for_rule_validation('') + def _validate_minlength(self, min_length, field, value): """ {'type': 'integer'} """ if isinstance(value, Iterable) and len(value) < min_length: @@ -1148,23 +1329,33 @@ class BareValidator(object): if not nullable: self._error(field, errors.NOT_NULLABLE) self._drop_remaining_rules( - 'empty', 'forbidden', 'items', 'keyschema', 'min', 'max', - 'minlength', 'maxlength', 'regex', 'schema', 'type', - 'valueschema') + 'allowed', + 'empty', + 'forbidden', + 'items', + 'keysrules', + 'min', + 'max', + 'minlength', + 'maxlength', + 'regex', + 'schema', + 'type', + 'valuesrules', + ) - def _validate_keyschema(self, schema, field, value): - """ {'type': ['dict', 'string'], 'validator': 'bulk_schema', + def _validate_keysrules(self, schema, field, value): + """ {'type': ['dict', 'string'], 'check_with': 'bulk_schema', 'forbidden': ['rename', 'rename_handler']} """ if isinstance(value, Mapping): validator = self._get_child_validator( document_crumb=field, - schema_crumb=(field, 'keyschema'), - schema=dict(((k, schema) for k in value.keys()))) - if not validator(dict(((k, k) for k in value.keys())), - normalize=False): - self._drop_nodes_from_errorpaths(validator._errors, - [], [2, 4]) - self._error(field, errors.KEYSCHEMA, validator._errors) + schema_crumb=(field, 'keysrules'), + schema=dict(((k, schema) for k in value.keys())), + ) + if not validator(dict(((k, k) for k in value.keys())), normalize=False): + self._drop_nodes_from_errorpaths(validator._errors, [], [2, 4]) + self._error(field, errors.KEYSRULES, validator._errors) def _validate_readonly(self, readonly, field, value): """ {'type': 'boolean'} """ @@ -1174,9 +1365,12 @@ class BareValidator(object): # If the document was normalized (and therefore already been # checked for readonly fields), we still have to return True # if an error was filed. - has_error = errors.READONLY_FIELD in \ - self.document_error_tree.fetch_errors_from( - self.document_path + (field,)) + has_error = ( + errors.READONLY_FIELD + in self.document_error_tree.fetch_errors_from( + self.document_path + (field,) + ) + ) if self._is_normalized and has_error: self._drop_remaining_rules() @@ -1192,41 +1386,47 @@ class BareValidator(object): _validate_required = dummy_for_rule_validation(""" {'type': 'boolean'} """) + _validate_require_all = dummy_for_rule_validation(""" {'type': 'boolean'} """) + def __validate_required_fields(self, document): """ Validates that required fields are not missing. :param document: The document being validated. """ try: - required = set(field for field, definition in self.schema.items() - if self._resolve_rules_set(definition). - get('required') is True) + required = set( + field + for field, definition in self.schema.items() + if self._resolve_rules_set(definition).get('required', self.require_all) + is True + ) except AttributeError: if self.is_child and self.schema_path[-1] == 'schema': raise _SchemaRuleTypeError else: raise required -= self._unrequired_by_excludes - missing = required - set(field for field in document - if document.get(field) is not None or - not self.ignore_none_values) + missing = required - set( + field + for field in document + if document.get(field) is not None or not self.ignore_none_values + ) for field in missing: self._error(field, errors.REQUIRED_FIELD) - # At least on field from self._unrequired_by_excludes should be - # present in document + # At least one field from self._unrequired_by_excludes should be present in + # document. if self._unrequired_by_excludes: - fields = set(field for field in document - if document.get(field) is not None) + fields = set(field for field in document if document.get(field) is not None) if self._unrequired_by_excludes.isdisjoint(fields): for field in self._unrequired_by_excludes - fields: self._error(field, errors.REQUIRED_FIELD) def _validate_schema(self, schema, field, value): """ {'type': ['dict', 'string'], - 'anyof': [{'validator': 'schema'}, - {'validator': 'bulk_schema'}]} """ + 'anyof': [{'check_with': 'schema'}, + {'check_with': 'bulk_schema'}]} """ if schema is None: return @@ -1237,12 +1437,15 @@ class BareValidator(object): def __validate_schema_mapping(self, field, schema, value): schema = self._resolve_schema(schema) - allow_unknown = self.schema[field].get('allow_unknown', - self.allow_unknown) - validator = self._get_child_validator(document_crumb=field, - schema_crumb=(field, 'schema'), - schema=schema, - allow_unknown=allow_unknown) + allow_unknown = self.schema[field].get('allow_unknown', self.allow_unknown) + require_all = self.schema[field].get('require_all', self.require_all) + validator = self._get_child_validator( + document_crumb=field, + schema_crumb=(field, 'schema'), + schema=schema, + allow_unknown=allow_unknown, + require_all=require_all, + ) try: if not validator(value, update=self.update, normalize=False): self._error(field, errors.MAPPING_SCHEMA, validator._errors) @@ -1253,10 +1456,16 @@ class BareValidator(object): def __validate_schema_sequence(self, field, schema, value): schema = dict(((i, schema) for i in range(len(value)))) validator = self._get_child_validator( - document_crumb=field, schema_crumb=(field, 'schema'), - schema=schema, allow_unknown=self.allow_unknown) - validator(dict(((i, v) for i, v in enumerate(value))), - update=self.update, normalize=False) + document_crumb=field, + schema_crumb=(field, 'schema'), + schema=schema, + allow_unknown=self.allow_unknown, + ) + validator( + dict(((i, v) for i, v in enumerate(value))), + update=self.update, + normalize=False, + ) if validator._errors: self._drop_nodes_from_errorpaths(validator._errors, [], [2]) @@ -1264,7 +1473,7 @@ class BareValidator(object): def _validate_type(self, data_type, field, value): """ {'type': ['string', 'list'], - 'validator': 'type'} """ + 'check_with': 'type'} """ if not data_type: return @@ -1275,8 +1484,9 @@ class BareValidator(object): # this implementation still supports custom type validation methods type_definition = self.types_mapping.get(_type) if type_definition is not None: - matched = isinstance(value, type_definition.included_types) \ - and not isinstance(value, type_definition.excluded_types) + matched = isinstance( + value, type_definition.included_types + ) and not isinstance(value, type_definition.excluded_types) else: type_handler = self.__get_rule_handler('validate_type', _type) matched = type_handler(value) @@ -1293,43 +1503,28 @@ class BareValidator(object): self._error(field, errors.BAD_TYPE) self._drop_remaining_rules() - def _validate_validator(self, validator, field, value): - """ {'oneof': [ - {'type': 'callable'}, - {'type': 'list', - 'schema': {'oneof': [{'type': 'callable'}, - {'type': 'string'}]}}, - {'type': 'string'} - ]} """ - if isinstance(validator, _str_type): - validator = self.__get_rule_handler('validator', validator) - validator(field, value) - elif isinstance(validator, Iterable): - for v in validator: - self._validate_validator(v, field, value) - else: - validator(field, value, self._error) - - def _validate_valueschema(self, schema, field, value): - """ {'type': ['dict', 'string'], 'validator': 'bulk_schema', + def _validate_valuesrules(self, schema, field, value): + """ {'type': ['dict', 'string'], 'check_with': 'bulk_schema', 'forbidden': ['rename', 'rename_handler']} """ - schema_crumb = (field, 'valueschema') + schema_crumb = (field, 'valuesrules') if isinstance(value, Mapping): validator = self._get_child_validator( - document_crumb=field, schema_crumb=schema_crumb, - schema=dict((k, schema) for k in value)) + document_crumb=field, + schema_crumb=schema_crumb, + schema=dict((k, schema) for k in value), + ) validator(value, update=self.update, normalize=False) if validator._errors: self._drop_nodes_from_errorpaths(validator._errors, [], [2]) - self._error(field, errors.VALUESCHEMA, validator._errors) + self._error(field, errors.VALUESRULES, validator._errors) -RULE_SCHEMA_SEPARATOR = \ - "The rule's arguments are validated against this schema:" +RULE_SCHEMA_SEPARATOR = "The rule's arguments are validated against this schema:" class InspectedValidator(type): """ Metaclass for all validators """ + def __new__(cls, *args): if '__doc__' not in args[2]: args[2].update({'__doc__': args[1][0].__doc__}) @@ -1337,8 +1532,11 @@ class InspectedValidator(type): def __init__(cls, *args): def attributes_with_prefix(prefix): - return tuple(x.split('_', 2)[-1] for x in dir(cls) - if x.startswith('_' + prefix)) + return tuple( + x[len(prefix) + 2 :] + for x in dir(cls) + if x.startswith('_' + prefix + '_') + ) super(InspectedValidator, cls).__init__(*args) @@ -1346,20 +1544,27 @@ class InspectedValidator(type): for attribute in attributes_with_prefix('validate'): # TODO remove inspection of type test methods in next major release if attribute.startswith('type_'): - cls._types_from_methods += (attribute[len('type_'):],) + cls._types_from_methods += (attribute[len('type_') :],) else: - cls.validation_rules[attribute] = \ - cls.__get_rule_schema('_validate_' + attribute) + cls.validation_rules[attribute] = cls.__get_rule_schema( + '_validate_' + attribute + ) # TODO remove on next major release if cls._types_from_methods: - warn("Methods for type testing are deprecated, use TypeDefinition " - "and the 'types_mapping'-property of a Validator-instance " - "instead.", DeprecationWarning) + warn( + "Methods for type testing are deprecated, use TypeDefinition " + "and the 'types_mapping'-property of a Validator-instance " + "instead.", + DeprecationWarning, + ) - cls.validators = tuple(x for x in attributes_with_prefix('validator')) - x = cls.validation_rules['validator']['oneof'] - x[1]['schema']['oneof'][1]['allowed'] = x[2]['allowed'] = cls.validators + # TODO remove second summand on next major release + cls.checkers = tuple(x for x in attributes_with_prefix('check_with')) + tuple( + x for x in attributes_with_prefix('validator') + ) + x = cls.validation_rules['check_with']['oneof'] + x[1]['schema']['oneof'][1]['allowed'] = x[2]['allowed'] = cls.checkers for rule in (x for x in cls.mandatory_validations if x != 'nullable'): cls.validation_rules[rule]['required'] = True @@ -1367,19 +1572,20 @@ class InspectedValidator(type): cls.coercers, cls.default_setters, cls.normalization_rules = (), (), {} for attribute in attributes_with_prefix('normalize'): if attribute.startswith('coerce_'): - cls.coercers += (attribute[len('coerce_'):],) + cls.coercers += (attribute[len('coerce_') :],) elif attribute.startswith('default_setter_'): - cls.default_setters += (attribute[len('default_setter_'):],) + cls.default_setters += (attribute[len('default_setter_') :],) else: - cls.normalization_rules[attribute] = \ - cls.__get_rule_schema('_normalize_' + attribute) + cls.normalization_rules[attribute] = cls.__get_rule_schema( + '_normalize_' + attribute + ) for rule in ('coerce', 'rename_handler'): x = cls.normalization_rules[rule]['oneof'] - x[1]['schema']['oneof'][1]['allowed'] = \ - x[2]['allowed'] = cls.coercers - cls.normalization_rules['default_setter']['oneof'][1]['allowed'] = \ - cls.default_setters + x[1]['schema']['oneof'][1]['allowed'] = x[2]['allowed'] = cls.coercers + cls.normalization_rules['default_setter']['oneof'][1][ + 'allowed' + ] = cls.default_setters cls.rules = {} cls.rules.update(cls.validation_rules) @@ -1397,9 +1603,11 @@ class InspectedValidator(type): except Exception: result = {} - if not result: - warn("No validation schema is defined for the arguments of rule " - "'%s'" % method_name.split('_', 2)[-1]) + if not result and method_name != '_validate_meta': + warn( + "No validation schema is defined for the arguments of rule " + "'%s'" % method_name.split('_', 2)[-1] + ) return result diff --git a/pipenv/vendor/certifi/__init__.py b/pipenv/vendor/certifi/__init__.py index 50f2e130..632db8e1 100644 --- a/pipenv/vendor/certifi/__init__.py +++ b/pipenv/vendor/certifi/__init__.py @@ -1,3 +1,3 @@ -from .core import where, old_where +from .core import where -__version__ = "2018.10.15" +__version__ = "2019.03.09" diff --git a/pipenv/vendor/certifi/cacert.pem b/pipenv/vendor/certifi/cacert.pem index e75d85b3..84636dde 100644 --- a/pipenv/vendor/certifi/cacert.pem +++ b/pipenv/vendor/certifi/cacert.pem @@ -4268,3 +4268,391 @@ rYy0UGYwEAYJKwYBBAGCNxUBBAMCAQAwCgYIKoZIzj0EAwMDaAAwZQIwJsdpW9zV 57LnyAyMjMPdeYwbY9XJUpROTYJKcx6ygISpJcBMWm1JKWB4E+J+SOtkAjEA2zQg Mgj/mkkCtojeFK9dbJlxjRo/i9fgojaGHAeCOnZT/cKi7e97sIBPWA9LUzm9 -----END CERTIFICATE----- + +# Issuer: CN=GTS Root R1 O=Google Trust Services LLC +# Subject: CN=GTS Root R1 O=Google Trust Services LLC +# Label: "GTS Root R1" +# Serial: 146587175971765017618439757810265552097 +# MD5 Fingerprint: 82:1a:ef:d4:d2:4a:f2:9f:e2:3d:97:06:14:70:72:85 +# SHA1 Fingerprint: e1:c9:50:e6:ef:22:f8:4c:56:45:72:8b:92:20:60:d7:d5:a7:a3:e8 +# SHA256 Fingerprint: 2a:57:54:71:e3:13:40:bc:21:58:1c:bd:2c:f1:3e:15:84:63:20:3e:ce:94:bc:f9:d3:cc:19:6b:f0:9a:54:72 +-----BEGIN CERTIFICATE----- +MIIFWjCCA0KgAwIBAgIQbkepxUtHDA3sM9CJuRz04TANBgkqhkiG9w0BAQwFADBH +MQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExM +QzEUMBIGA1UEAxMLR1RTIFJvb3QgUjEwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIy +MDAwMDAwWjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNl +cnZpY2VzIExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjEwggIiMA0GCSqGSIb3DQEB +AQUAA4ICDwAwggIKAoICAQC2EQKLHuOhd5s73L+UPreVp0A8of2C+X0yBoJx9vaM +f/vo27xqLpeXo4xL+Sv2sfnOhB2x+cWX3u+58qPpvBKJXqeqUqv4IyfLpLGcY9vX +mX7wCl7raKb0xlpHDU0QM+NOsROjyBhsS+z8CZDfnWQpJSMHobTSPS5g4M/SCYe7 +zUjwTcLCeoiKu7rPWRnWr4+wB7CeMfGCwcDfLqZtbBkOtdh+JhpFAz2weaSUKK0P +fyblqAj+lug8aJRT7oM6iCsVlgmy4HqMLnXWnOunVmSPlk9orj2XwoSPwLxAwAtc +vfaHszVsrBhQf4TgTM2S0yDpM7xSma8ytSmzJSq0SPly4cpk9+aCEI3oncKKiPo4 +Zor8Y/kB+Xj9e1x3+naH+uzfsQ55lVe0vSbv1gHR6xYKu44LtcXFilWr06zqkUsp +zBmkMiVOKvFlRNACzqrOSbTqn3yDsEB750Orp2yjj32JgfpMpf/VjsPOS+C12LOO +Rc92wO1AK/1TD7Cn1TsNsYqiA94xrcx36m97PtbfkSIS5r762DL8EGMUUXLeXdYW +k70paDPvOmbsB4om3xPXV2V4J95eSRQAogB/mqghtqmxlbCluQ0WEdrHbEg8QOB+ +DVrNVjzRlwW5y0vtOUucxD/SVRNuJLDWcfr0wbrM7Rv1/oFB2ACYPTrIrnqYNxgF +lQIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNV +HQ4EFgQU5K8rJnEaK0gnhS9SZizv8IkTcT4wDQYJKoZIhvcNAQEMBQADggIBADiW +Cu49tJYeX++dnAsznyvgyv3SjgofQXSlfKqE1OXyHuY3UjKcC9FhHb8owbZEKTV1 +d5iyfNm9dKyKaOOpMQkpAWBz40d8U6iQSifvS9efk+eCNs6aaAyC58/UEBZvXw6Z +XPYfcX3v73svfuo21pdwCxXu11xWajOl40k4DLh9+42FpLFZXvRq4d2h9mREruZR +gyFmxhE+885H7pwoHyXa/6xmld01D1zvICxi/ZG6qcz8WpyTgYMpl0p8WnK0OdC3 +d8t5/Wk6kjftbjhlRn7pYL15iJdfOBL07q9bgsiG1eGZbYwE8na6SfZu6W0eX6Dv +J4J2QPim01hcDyxC2kLGe4g0x8HYRZvBPsVhHdljUEn2NIVq4BjFbkerQUIpm/Zg +DdIx02OYI5NaAIFItO/Nis3Jz5nu2Z6qNuFoS3FJFDYoOj0dzpqPJeaAcWErtXvM ++SUWgeExX6GjfhaknBZqlxi9dnKlC54dNuYvoS++cJEPqOba+MSSQGwlfnuzCdyy +F62ARPBopY+Udf90WuioAnwMCeKpSwughQtiue+hMZL77/ZRBIls6Kl0obsXs7X9 +SQ98POyDGCBDTtWTurQ0sR8WNh8M5mQ5Fkzc4P4dyKliPUDqysU0ArSuiYgzNdws +E3PYJ/HQcu51OyLemGhmW/HGY0dVHLqlCFF1pkgl +-----END CERTIFICATE----- + +# Issuer: CN=GTS Root R2 O=Google Trust Services LLC +# Subject: CN=GTS Root R2 O=Google Trust Services LLC +# Label: "GTS Root R2" +# Serial: 146587176055767053814479386953112547951 +# MD5 Fingerprint: 44:ed:9a:0e:a4:09:3b:00:f2:ae:4c:a3:c6:61:b0:8b +# SHA1 Fingerprint: d2:73:96:2a:2a:5e:39:9f:73:3f:e1:c7:1e:64:3f:03:38:34:fc:4d +# SHA256 Fingerprint: c4:5d:7b:b0:8e:6d:67:e6:2e:42:35:11:0b:56:4e:5f:78:fd:92:ef:05:8c:84:0a:ea:4e:64:55:d7:58:5c:60 +-----BEGIN CERTIFICATE----- +MIIFWjCCA0KgAwIBAgIQbkepxlqz5yDFMJo/aFLybzANBgkqhkiG9w0BAQwFADBH +MQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExM +QzEUMBIGA1UEAxMLR1RTIFJvb3QgUjIwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIy +MDAwMDAwWjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNl +cnZpY2VzIExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjIwggIiMA0GCSqGSIb3DQEB +AQUAA4ICDwAwggIKAoICAQDO3v2m++zsFDQ8BwZabFn3GTXd98GdVarTzTukk3Lv +CvptnfbwhYBboUhSnznFt+4orO/LdmgUud+tAWyZH8QiHZ/+cnfgLFuv5AS/T3Kg +GjSY6Dlo7JUle3ah5mm5hRm9iYz+re026nO8/4Piy33B0s5Ks40FnotJk9/BW9Bu +XvAuMC6C/Pq8tBcKSOWIm8Wba96wyrQD8Nr0kLhlZPdcTK3ofmZemde4wj7I0BOd +re7kRXuJVfeKH2JShBKzwkCX44ofR5GmdFrS+LFjKBC4swm4VndAoiaYecb+3yXu +PuWgf9RhD1FLPD+M2uFwdNjCaKH5wQzpoeJ/u1U8dgbuak7MkogwTZq9TwtImoS1 +mKPV+3PBV2HdKFZ1E66HjucMUQkQdYhMvI35ezzUIkgfKtzra7tEscszcTJGr61K +8YzodDqs5xoic4DSMPclQsciOzsSrZYuxsN2B6ogtzVJV+mSSeh2FnIxZyuWfoqj +x5RWIr9qS34BIbIjMt/kmkRtWVtd9QCgHJvGeJeNkP+byKq0rxFROV7Z+2et1VsR +nTKaG73VululycslaVNVJ1zgyjbLiGH7HrfQy+4W+9OmTN6SpdTi3/UGVN4unUu0 +kzCqgc7dGtxRcw1PcOnlthYhGXmy5okLdWTK1au8CcEYof/UVKGFPP0UJAOyh9Ok +twIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNV +HQ4EFgQUu//KjiOfT5nK2+JopqUVJxce2Q4wDQYJKoZIhvcNAQEMBQADggIBALZp +8KZ3/p7uC4Gt4cCpx/k1HUCCq+YEtN/L9x0Pg/B+E02NjO7jMyLDOfxA325BS0JT +vhaI8dI4XsRomRyYUpOM52jtG2pzegVATX9lO9ZY8c6DR2Dj/5epnGB3GFW1fgiT +z9D2PGcDFWEJ+YF59exTpJ/JjwGLc8R3dtyDovUMSRqodt6Sm2T4syzFJ9MHwAiA +pJiS4wGWAqoC7o87xdFtCjMwc3i5T1QWvwsHoaRc5svJXISPD+AVdyx+Jn7axEvb +pxZ3B7DNdehyQtaVhJ2Gg/LkkM0JR9SLA3DaWsYDQvTtN6LwG1BUSw7YhN4ZKJmB +R64JGz9I0cNv4rBgF/XuIwKl2gBbbZCr7qLpGzvpx0QnRY5rn/WkhLx3+WuXrD5R +RaIRpsyF7gpo8j5QOHokYh4XIDdtak23CZvJ/KRY9bb7nE4Yu5UC56GtmwfuNmsk +0jmGwZODUNKBRqhfYlcsu2xkiAhu7xNUX90txGdj08+JN7+dIPT7eoOboB6BAFDC +5AwiWVIQ7UNWhwD4FFKnHYuTjKJNRn8nxnGbJN7k2oaLDX5rIMHAnuFl2GqjpuiF +izoHCBy69Y9Vmhh1fuXsgWbRIXOhNUQLgD1bnF5vKheW0YMjiGZt5obicDIvUiLn +yOd/xCxgXS/Dr55FBcOEArf9LAhST4Ldo/DUhgkC +-----END CERTIFICATE----- + +# Issuer: CN=GTS Root R3 O=Google Trust Services LLC +# Subject: CN=GTS Root R3 O=Google Trust Services LLC +# Label: "GTS Root R3" +# Serial: 146587176140553309517047991083707763997 +# MD5 Fingerprint: 1a:79:5b:6b:04:52:9c:5d:c7:74:33:1b:25:9a:f9:25 +# SHA1 Fingerprint: 30:d4:24:6f:07:ff:db:91:89:8a:0b:e9:49:66:11:eb:8c:5e:46:e5 +# SHA256 Fingerprint: 15:d5:b8:77:46:19:ea:7d:54:ce:1c:a6:d0:b0:c4:03:e0:37:a9:17:f1:31:e8:a0:4e:1e:6b:7a:71:ba:bc:e5 +-----BEGIN CERTIFICATE----- +MIICDDCCAZGgAwIBAgIQbkepx2ypcyRAiQ8DVd2NHTAKBggqhkjOPQQDAzBHMQsw +CQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzEU +MBIGA1UEAxMLR1RTIFJvb3QgUjMwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIyMDAw +MDAwWjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZp +Y2VzIExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjMwdjAQBgcqhkjOPQIBBgUrgQQA +IgNiAAQfTzOHMymKoYTey8chWEGJ6ladK0uFxh1MJ7x/JlFyb+Kf1qPKzEUURout +736GjOyxfi//qXGdGIRFBEFVbivqJn+7kAHjSxm65FSWRQmx1WyRRK2EE46ajA2A +DDL24CejQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1Ud +DgQWBBTB8Sa6oC2uhYHP0/EqEr24Cmf9vDAKBggqhkjOPQQDAwNpADBmAjEAgFuk +fCPAlaUs3L6JbyO5o91lAFJekazInXJ0glMLfalAvWhgxeG4VDvBNhcl2MG9AjEA +njWSdIUlUfUk7GRSJFClH9voy8l27OyCbvWFGFPouOOaKaqW04MjyaR7YbPMAuhd +-----END CERTIFICATE----- + +# Issuer: CN=GTS Root R4 O=Google Trust Services LLC +# Subject: CN=GTS Root R4 O=Google Trust Services LLC +# Label: "GTS Root R4" +# Serial: 146587176229350439916519468929765261721 +# MD5 Fingerprint: 5d:b6:6a:c4:60:17:24:6a:1a:99:a8:4b:ee:5e:b4:26 +# SHA1 Fingerprint: 2a:1d:60:27:d9:4a:b1:0a:1c:4d:91:5c:cd:33:a0:cb:3e:2d:54:cb +# SHA256 Fingerprint: 71:cc:a5:39:1f:9e:79:4b:04:80:25:30:b3:63:e1:21:da:8a:30:43:bb:26:66:2f:ea:4d:ca:7f:c9:51:a4:bd +-----BEGIN CERTIFICATE----- +MIICCjCCAZGgAwIBAgIQbkepyIuUtui7OyrYorLBmTAKBggqhkjOPQQDAzBHMQsw +CQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzEU +MBIGA1UEAxMLR1RTIFJvb3QgUjQwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIyMDAw +MDAwWjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZp +Y2VzIExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjQwdjAQBgcqhkjOPQIBBgUrgQQA +IgNiAATzdHOnaItgrkO4NcWBMHtLSZ37wWHO5t5GvWvVYRg1rkDdc/eJkTBa6zzu +hXyiQHY7qca4R9gq55KRanPpsXI5nymfopjTX15YhmUPoYRlBtHci8nHc8iMai/l +xKvRHYqjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1Ud +DgQWBBSATNbrdP9JNqPV2Py1PsVq8JQdjDAKBggqhkjOPQQDAwNnADBkAjBqUFJ0 +CMRw3J5QdCHojXohw0+WbhXRIjVhLfoIN+4Zba3bssx9BzT1YBkstTTZbyACMANx +sbqjYAuG7ZoIapVon+Kz4ZNkfF6Tpt95LY2F45TPI11xzPKwTdb+mciUqXWi4w== +-----END CERTIFICATE----- + +# Issuer: CN=UCA Global G2 Root O=UniTrust +# Subject: CN=UCA Global G2 Root O=UniTrust +# Label: "UCA Global G2 Root" +# Serial: 124779693093741543919145257850076631279 +# MD5 Fingerprint: 80:fe:f0:c4:4a:f0:5c:62:32:9f:1c:ba:78:a9:50:f8 +# SHA1 Fingerprint: 28:f9:78:16:19:7a:ff:18:25:18:aa:44:fe:c1:a0:ce:5c:b6:4c:8a +# SHA256 Fingerprint: 9b:ea:11:c9:76:fe:01:47:64:c1:be:56:a6:f9:14:b5:a5:60:31:7a:bd:99:88:39:33:82:e5:16:1a:a0:49:3c +-----BEGIN CERTIFICATE----- +MIIFRjCCAy6gAwIBAgIQXd+x2lqj7V2+WmUgZQOQ7zANBgkqhkiG9w0BAQsFADA9 +MQswCQYDVQQGEwJDTjERMA8GA1UECgwIVW5pVHJ1c3QxGzAZBgNVBAMMElVDQSBH +bG9iYWwgRzIgUm9vdDAeFw0xNjAzMTEwMDAwMDBaFw00MDEyMzEwMDAwMDBaMD0x +CzAJBgNVBAYTAkNOMREwDwYDVQQKDAhVbmlUcnVzdDEbMBkGA1UEAwwSVUNBIEds +b2JhbCBHMiBSb290MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAxeYr +b3zvJgUno4Ek2m/LAfmZmqkywiKHYUGRO8vDaBsGxUypK8FnFyIdK+35KYmToni9 +kmugow2ifsqTs6bRjDXVdfkX9s9FxeV67HeToI8jrg4aA3++1NDtLnurRiNb/yzm +VHqUwCoV8MmNsHo7JOHXaOIxPAYzRrZUEaalLyJUKlgNAQLx+hVRZ2zA+te2G3/R +VogvGjqNO7uCEeBHANBSh6v7hn4PJGtAnTRnvI3HLYZveT6OqTwXS3+wmeOwcWDc +C/Vkw85DvG1xudLeJ1uK6NjGruFZfc8oLTW4lVYa8bJYS7cSN8h8s+1LgOGN+jIj +tm+3SJUIsUROhYw6AlQgL9+/V087OpAh18EmNVQg7Mc/R+zvWr9LesGtOxdQXGLY +D0tK3Cv6brxzks3sx1DoQZbXqX5t2Okdj4q1uViSukqSKwxW/YDrCPBeKW4bHAyv +j5OJrdu9o54hyokZ7N+1wxrrFv54NkzWbtA+FxyQF2smuvt6L78RHBgOLXMDj6Dl +NaBa4kx1HXHhOThTeEDMg5PXCp6dW4+K5OXgSORIskfNTip1KnvyIvbJvgmRlld6 +iIis7nCs+dwp4wwcOxJORNanTrAmyPPZGpeRaOrvjUYG0lZFWJo8DA+DuAUlwznP +O6Q0ibd5Ei9Hxeepl2n8pndntd978XplFeRhVmUCAwEAAaNCMEAwDgYDVR0PAQH/ +BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFIHEjMz15DD/pQwIX4wV +ZyF0Ad/fMA0GCSqGSIb3DQEBCwUAA4ICAQATZSL1jiutROTL/7lo5sOASD0Ee/oj +L3rtNtqyzm325p7lX1iPyzcyochltq44PTUbPrw7tgTQvPlJ9Zv3hcU2tsu8+Mg5 +1eRfB70VVJd0ysrtT7q6ZHafgbiERUlMjW+i67HM0cOU2kTC5uLqGOiiHycFutfl +1qnN3e92mI0ADs0b+gO3joBYDic/UvuUospeZcnWhNq5NXHzJsBPd+aBJ9J3O5oU +b3n09tDh05S60FdRvScFDcH9yBIw7m+NESsIndTUv4BFFJqIRNow6rSn4+7vW4LV +PtateJLbXDzz2K36uGt/xDYotgIVilQsnLAXc47QN6MUPJiVAAwpBVueSUmxX8fj +y88nZY41F7dXyDDZQVu5FLbowg+UMaeUmMxq67XhJ/UQqAHojhJi6IjMtX9Gl8Cb +EGY4GjZGXyJoPd/JxhMnq1MGrKI8hgZlb7F+sSlEmqO6SWkoaY/X5V+tBIZkbxqg +DMUIYs6Ao9Dz7GjevjPHF1t/gMRMTLGmhIrDO7gJzRSBuhjjVFc2/tsvfEehOjPI ++Vg7RE+xygKJBJYoaMVLuCaJu9YzL1DV/pqJuhgyklTGW+Cd+V7lDSKb9triyCGy +YiGqhkCyLmTTX8jjfhFnRR8F/uOi77Oos/N9j/gMHyIfLXC0uAE0djAA5SN4p1bX +UB+K+wb1whnw0A== +-----END CERTIFICATE----- + +# Issuer: CN=UCA Extended Validation Root O=UniTrust +# Subject: CN=UCA Extended Validation Root O=UniTrust +# Label: "UCA Extended Validation Root" +# Serial: 106100277556486529736699587978573607008 +# MD5 Fingerprint: a1:f3:5f:43:c6:34:9b:da:bf:8c:7e:05:53:ad:96:e2 +# SHA1 Fingerprint: a3:a1:b0:6f:24:61:23:4a:e3:36:a5:c2:37:fc:a6:ff:dd:f0:d7:3a +# SHA256 Fingerprint: d4:3a:f9:b3:54:73:75:5c:96:84:fc:06:d7:d8:cb:70:ee:5c:28:e7:73:fb:29:4e:b4:1e:e7:17:22:92:4d:24 +-----BEGIN CERTIFICATE----- +MIIFWjCCA0KgAwIBAgIQT9Irj/VkyDOeTzRYZiNwYDANBgkqhkiG9w0BAQsFADBH +MQswCQYDVQQGEwJDTjERMA8GA1UECgwIVW5pVHJ1c3QxJTAjBgNVBAMMHFVDQSBF +eHRlbmRlZCBWYWxpZGF0aW9uIFJvb3QwHhcNMTUwMzEzMDAwMDAwWhcNMzgxMjMx +MDAwMDAwWjBHMQswCQYDVQQGEwJDTjERMA8GA1UECgwIVW5pVHJ1c3QxJTAjBgNV +BAMMHFVDQSBFeHRlbmRlZCBWYWxpZGF0aW9uIFJvb3QwggIiMA0GCSqGSIb3DQEB +AQUAA4ICDwAwggIKAoICAQCpCQcoEwKwmeBkqh5DFnpzsZGgdT6o+uM4AHrsiWog +D4vFsJszA1qGxliG1cGFu0/GnEBNyr7uaZa4rYEwmnySBesFK5pI0Lh2PpbIILvS +sPGP2KxFRv+qZ2C0d35qHzwaUnoEPQc8hQ2E0B92CvdqFN9y4zR8V05WAT558aop +O2z6+I9tTcg1367r3CTueUWnhbYFiN6IXSV8l2RnCdm/WhUFhvMJHuxYMjMR83dk +sHYf5BA1FxvyDrFspCqjc/wJHx4yGVMR59mzLC52LqGj3n5qiAno8geK+LLNEOfi +c0CTuwjRP+H8C5SzJe98ptfRr5//lpr1kXuYC3fUfugH0mK1lTnj8/FtDw5lhIpj +VMWAtuCeS31HJqcBCF3RiJ7XwzJE+oJKCmhUfzhTA8ykADNkUVkLo4KRel7sFsLz +KuZi2irbWWIQJUoqgQtHB0MGcIfS+pMRKXpITeuUx3BNr2fVUbGAIAEBtHoIppB/ +TuDvB0GHr2qlXov7z1CymlSvw4m6WC31MJixNnI5fkkE/SmnTHnkBVfblLkWU41G +sx2VYVdWf6/wFlthWG82UBEL2KwrlRYaDh8IzTY0ZRBiZtWAXxQgXy0MoHgKaNYs +1+lvK9JKBZP8nm9rZ/+I8U6laUpSNwXqxhaN0sSZ0YIrO7o1dfdRUVjzyAfd5LQD +fwIDAQABo0IwQDAdBgNVHQ4EFgQU2XQ65DA9DfcS3H5aBZ8eNJr34RQwDwYDVR0T +AQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAYYwDQYJKoZIhvcNAQELBQADggIBADaN +l8xCFWQpN5smLNb7rhVpLGsaGvdftvkHTFnq88nIua7Mui563MD1sC3AO6+fcAUR +ap8lTwEpcOPlDOHqWnzcSbvBHiqB9RZLcpHIojG5qtr8nR/zXUACE/xOHAbKsxSQ +VBcZEhrxH9cMaVr2cXj0lH2RC47skFSOvG+hTKv8dGT9cZr4QQehzZHkPJrgmzI5 +c6sq1WnIeJEmMX3ixzDx/BR4dxIOE/TdFpS/S2d7cFOFyrC78zhNLJA5wA3CXWvp +4uXViI3WLL+rG761KIcSF3Ru/H38j9CHJrAb+7lsq+KePRXBOy5nAliRn+/4Qh8s +t2j1da3Ptfb/EX3C8CSlrdP6oDyp+l3cpaDvRKS+1ujl5BOWF3sGPjLtx7dCvHaj +2GU4Kzg1USEODm8uNBNA4StnDG1KQTAYI1oyVZnJF+A83vbsea0rWBmirSwiGpWO +vpaQXUJXxPkUAzUrHC1RVwinOt4/5Mi0A3PCwSaAuwtCH60NryZy2sy+s6ODWA2C +xR9GUeOcGMyNm43sSet1UNWMKFnKdDTajAshqx7qG+XH/RU+wBeq+yNuJkbL+vmx +cmtpzyKEC2IPrNkZAJSidjzULZrtBJ4tBmIQN1IchXIbJ+XMxjHsN+xjWZsLHXbM +fjKaiJUINlK73nZfdklJrX+9ZSCyycErdhh2n1ax +-----END CERTIFICATE----- + +# Issuer: CN=Certigna Root CA O=Dhimyotis OU=0002 48146308100036 +# Subject: CN=Certigna Root CA O=Dhimyotis OU=0002 48146308100036 +# Label: "Certigna Root CA" +# Serial: 269714418870597844693661054334862075617 +# MD5 Fingerprint: 0e:5c:30:62:27:eb:5b:bc:d7:ae:62:ba:e9:d5:df:77 +# SHA1 Fingerprint: 2d:0d:52:14:ff:9e:ad:99:24:01:74:20:47:6e:6c:85:27:27:f5:43 +# SHA256 Fingerprint: d4:8d:3d:23:ee:db:50:a4:59:e5:51:97:60:1c:27:77:4b:9d:7b:18:c9:4d:5a:05:95:11:a1:02:50:b9:31:68 +-----BEGIN CERTIFICATE----- +MIIGWzCCBEOgAwIBAgIRAMrpG4nxVQMNo+ZBbcTjpuEwDQYJKoZIhvcNAQELBQAw +WjELMAkGA1UEBhMCRlIxEjAQBgNVBAoMCURoaW15b3RpczEcMBoGA1UECwwTMDAw +MiA0ODE0NjMwODEwMDAzNjEZMBcGA1UEAwwQQ2VydGlnbmEgUm9vdCBDQTAeFw0x +MzEwMDEwODMyMjdaFw0zMzEwMDEwODMyMjdaMFoxCzAJBgNVBAYTAkZSMRIwEAYD +VQQKDAlEaGlteW90aXMxHDAaBgNVBAsMEzAwMDIgNDgxNDYzMDgxMDAwMzYxGTAX +BgNVBAMMEENlcnRpZ25hIFJvb3QgQ0EwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAw +ggIKAoICAQDNGDllGlmx6mQWDoyUJJV8g9PFOSbcDO8WV43X2KyjQn+Cyu3NW9sO +ty3tRQgXstmzy9YXUnIo245Onoq2C/mehJpNdt4iKVzSs9IGPjA5qXSjklYcoW9M +CiBtnyN6tMbaLOQdLNyzKNAT8kxOAkmhVECe5uUFoC2EyP+YbNDrihqECB63aCPu +I9Vwzm1RaRDuoXrC0SIxwoKF0vJVdlB8JXrJhFwLrN1CTivngqIkicuQstDuI7pm +TLtipPlTWmR7fJj6o0ieD5Wupxj0auwuA0Wv8HT4Ks16XdG+RCYyKfHx9WzMfgIh +C59vpD++nVPiz32pLHxYGpfhPTc3GGYo0kDFUYqMwy3OU4gkWGQwFsWq4NYKpkDf +ePb1BHxpE4S80dGnBs8B92jAqFe7OmGtBIyT46388NtEbVncSVmurJqZNjBBe3Yz +IoejwpKGbvlw7q6Hh5UbxHq9MfPU0uWZ/75I7HX1eBYdpnDBfzwboZL7z8g81sWT +Co/1VTp2lc5ZmIoJlXcymoO6LAQ6l73UL77XbJuiyn1tJslV1c/DeVIICZkHJC1k +JWumIWmbat10TWuXekG9qxf5kBdIjzb5LdXF2+6qhUVB+s06RbFo5jZMm5BX7CO5 +hwjCxAnxl4YqKE3idMDaxIzb3+KhF1nOJFl0Mdp//TBt2dzhauH8XwIDAQABo4IB +GjCCARYwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYE +FBiHVuBud+4kNTxOc5of1uHieX4rMB8GA1UdIwQYMBaAFBiHVuBud+4kNTxOc5of +1uHieX4rMEQGA1UdIAQ9MDswOQYEVR0gADAxMC8GCCsGAQUFBwIBFiNodHRwczov +L3d3d3cuY2VydGlnbmEuZnIvYXV0b3JpdGVzLzBtBgNVHR8EZjBkMC+gLaArhilo +dHRwOi8vY3JsLmNlcnRpZ25hLmZyL2NlcnRpZ25hcm9vdGNhLmNybDAxoC+gLYYr +aHR0cDovL2NybC5kaGlteW90aXMuY29tL2NlcnRpZ25hcm9vdGNhLmNybDANBgkq +hkiG9w0BAQsFAAOCAgEAlLieT/DjlQgi581oQfccVdV8AOItOoldaDgvUSILSo3L +6btdPrtcPbEo/uRTVRPPoZAbAh1fZkYJMyjhDSSXcNMQH+pkV5a7XdrnxIxPTGRG +HVyH41neQtGbqH6mid2PHMkwgu07nM3A6RngatgCdTer9zQoKJHyBApPNeNgJgH6 +0BGM+RFq7q89w1DTj18zeTyGqHNFkIwgtnJzFyO+B2XleJINugHA64wcZr+shncB +lA2c5uk5jR+mUYyZDDl34bSb+hxnV29qao6pK0xXeXpXIs/NX2NGjVxZOob4Mkdi +o2cNGJHc+6Zr9UhhcyNZjgKnvETq9Emd8VRY+WCv2hikLyhF3HqgiIZd8zvn/yk1 +gPxkQ5Tm4xxvvq0OKmOZK8l+hfZx6AYDlf7ej0gcWtSS6Cvu5zHbugRqh5jnxV/v +faci9wHYTfmJ0A6aBVmknpjZbyvKcL5kwlWj9Omvw5Ip3IgWJJk8jSaYtlu3zM63 +Nwf9JtmYhST/WSMDmu2dnajkXjjO11INb9I/bbEFa0nOipFGc/T2L/Coc3cOZayh +jWZSaX5LaAzHHjcng6WMxwLkFM1JAbBzs/3GkDpv0mztO+7skb6iQ12LAEpmJURw +3kAP+HwV96LOPNdeE4yBFxgX0b3xdxA61GU5wSesVywlVP+i2k+KYTlerj1KjL0= +-----END CERTIFICATE----- + +# Issuer: CN=emSign Root CA - G1 O=eMudhra Technologies Limited OU=emSign PKI +# Subject: CN=emSign Root CA - G1 O=eMudhra Technologies Limited OU=emSign PKI +# Label: "emSign Root CA - G1" +# Serial: 235931866688319308814040 +# MD5 Fingerprint: 9c:42:84:57:dd:cb:0b:a7:2e:95:ad:b6:f3:da:bc:ac +# SHA1 Fingerprint: 8a:c7:ad:8f:73:ac:4e:c1:b5:75:4d:a5:40:f4:fc:cf:7c:b5:8e:8c +# SHA256 Fingerprint: 40:f6:af:03:46:a9:9a:a1:cd:1d:55:5a:4e:9c:ce:62:c7:f9:63:46:03:ee:40:66:15:83:3d:c8:c8:d0:03:67 +-----BEGIN CERTIFICATE----- +MIIDlDCCAnygAwIBAgIKMfXkYgxsWO3W2DANBgkqhkiG9w0BAQsFADBnMQswCQYD +VQQGEwJJTjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBU +ZWNobm9sb2dpZXMgTGltaXRlZDEcMBoGA1UEAxMTZW1TaWduIFJvb3QgQ0EgLSBH +MTAeFw0xODAyMTgxODMwMDBaFw00MzAyMTgxODMwMDBaMGcxCzAJBgNVBAYTAklO +MRMwEQYDVQQLEwplbVNpZ24gUEtJMSUwIwYDVQQKExxlTXVkaHJhIFRlY2hub2xv +Z2llcyBMaW1pdGVkMRwwGgYDVQQDExNlbVNpZ24gUm9vdCBDQSAtIEcxMIIBIjAN +BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAk0u76WaK7p1b1TST0Bsew+eeuGQz +f2N4aLTNLnF115sgxk0pvLZoYIr3IZpWNVrzdr3YzZr/k1ZLpVkGoZM0Kd0WNHVO +8oG0x5ZOrRkVUkr+PHB1cM2vK6sVmjM8qrOLqs1D/fXqcP/tzxE7lM5OMhbTI0Aq +d7OvPAEsbO2ZLIvZTmmYsvePQbAyeGHWDV/D+qJAkh1cF+ZwPjXnorfCYuKrpDhM +tTk1b+oDafo6VGiFbdbyL0NVHpENDtjVaqSW0RM8LHhQ6DqS0hdW5TUaQBw+jSzt +Od9C4INBdN+jzcKGYEho42kLVACL5HZpIQ15TjQIXhTCzLG3rdd8cIrHhQIDAQAB +o0IwQDAdBgNVHQ4EFgQU++8Nhp6w492pufEhF38+/PB3KxowDgYDVR0PAQH/BAQD +AgEGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAFn/8oz1h31x +PaOfG1vR2vjTnGs2vZupYeveFix0PZ7mddrXuqe8QhfnPZHr5X3dPpzxz5KsbEjM +wiI/aTvFthUvozXGaCocV685743QNcMYDHsAVhzNixl03r4PEuDQqqE/AjSxcM6d +GNYIAwlG7mDgfrbESQRRfXBgvKqy/3lyeqYdPV8q+Mri/Tm3R7nrft8EI6/6nAYH +6ftjk4BAtcZsCjEozgyfz7MjNYBBjWzEN3uBL4ChQEKF6dk4jeihU80Bv2noWgby +RQuQ+q7hv53yrlc8pa6yVvSLZUDp/TGBLPQ5Cdjua6e0ph0VpZj3AYHYhX3zUVxx +iN66zB+Afko= +-----END CERTIFICATE----- + +# Issuer: CN=emSign ECC Root CA - G3 O=eMudhra Technologies Limited OU=emSign PKI +# Subject: CN=emSign ECC Root CA - G3 O=eMudhra Technologies Limited OU=emSign PKI +# Label: "emSign ECC Root CA - G3" +# Serial: 287880440101571086945156 +# MD5 Fingerprint: ce:0b:72:d1:9f:88:8e:d0:50:03:e8:e3:b8:8b:67:40 +# SHA1 Fingerprint: 30:43:fa:4f:f2:57:dc:a0:c3:80:ee:2e:58:ea:78:b2:3f:e6:bb:c1 +# SHA256 Fingerprint: 86:a1:ec:ba:08:9c:4a:8d:3b:be:27:34:c6:12:ba:34:1d:81:3e:04:3c:f9:e8:a8:62:cd:5c:57:a3:6b:be:6b +-----BEGIN CERTIFICATE----- +MIICTjCCAdOgAwIBAgIKPPYHqWhwDtqLhDAKBggqhkjOPQQDAzBrMQswCQYDVQQG +EwJJTjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBUZWNo +bm9sb2dpZXMgTGltaXRlZDEgMB4GA1UEAxMXZW1TaWduIEVDQyBSb290IENBIC0g +RzMwHhcNMTgwMjE4MTgzMDAwWhcNNDMwMjE4MTgzMDAwWjBrMQswCQYDVQQGEwJJ +TjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBUZWNobm9s +b2dpZXMgTGltaXRlZDEgMB4GA1UEAxMXZW1TaWduIEVDQyBSb290IENBIC0gRzMw +djAQBgcqhkjOPQIBBgUrgQQAIgNiAAQjpQy4LRL1KPOxst3iAhKAnjlfSU2fySU0 +WXTsuwYc58Byr+iuL+FBVIcUqEqy6HyC5ltqtdyzdc6LBtCGI79G1Y4PPwT01xyS +fvalY8L1X44uT6EYGQIrMgqCZH0Wk9GjQjBAMB0GA1UdDgQWBBR8XQKEE9TMipuB +zhccLikenEhjQjAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAKBggq +hkjOPQQDAwNpADBmAjEAvvNhzwIQHWSVB7gYboiFBS+DCBeQyh+KTOgNG3qxrdWB +CUfvO6wIBHxcmbHtRwfSAjEAnbpV/KlK6O3t5nYBQnvI+GDZjVGLVTv7jHvrZQnD ++JbNR6iC8hZVdyR+EhCVBCyj +-----END CERTIFICATE----- + +# Issuer: CN=emSign Root CA - C1 O=eMudhra Inc OU=emSign PKI +# Subject: CN=emSign Root CA - C1 O=eMudhra Inc OU=emSign PKI +# Label: "emSign Root CA - C1" +# Serial: 825510296613316004955058 +# MD5 Fingerprint: d8:e3:5d:01:21:fa:78:5a:b0:df:ba:d2:ee:2a:5f:68 +# SHA1 Fingerprint: e7:2e:f1:df:fc:b2:09:28:cf:5d:d4:d5:67:37:b1:51:cb:86:4f:01 +# SHA256 Fingerprint: 12:56:09:aa:30:1d:a0:a2:49:b9:7a:82:39:cb:6a:34:21:6f:44:dc:ac:9f:39:54:b1:42:92:f2:e8:c8:60:8f +-----BEGIN CERTIFICATE----- +MIIDczCCAlugAwIBAgILAK7PALrEzzL4Q7IwDQYJKoZIhvcNAQELBQAwVjELMAkG +A1UEBhMCVVMxEzARBgNVBAsTCmVtU2lnbiBQS0kxFDASBgNVBAoTC2VNdWRocmEg +SW5jMRwwGgYDVQQDExNlbVNpZ24gUm9vdCBDQSAtIEMxMB4XDTE4MDIxODE4MzAw +MFoXDTQzMDIxODE4MzAwMFowVjELMAkGA1UEBhMCVVMxEzARBgNVBAsTCmVtU2ln +biBQS0kxFDASBgNVBAoTC2VNdWRocmEgSW5jMRwwGgYDVQQDExNlbVNpZ24gUm9v +dCBDQSAtIEMxMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAz+upufGZ +BczYKCFK83M0UYRWEPWgTywS4/oTmifQz/l5GnRfHXk5/Fv4cI7gklL35CX5VIPZ +HdPIWoU/Xse2B+4+wM6ar6xWQio5JXDWv7V7Nq2s9nPczdcdioOl+yuQFTdrHCZH +3DspVpNqs8FqOp099cGXOFgFixwR4+S0uF2FHYP+eF8LRWgYSKVGczQ7/g/IdrvH +GPMF0Ybzhe3nudkyrVWIzqa2kbBPrH4VI5b2P/AgNBbeCsbEBEV5f6f9vtKppa+c +xSMq9zwhbL2vj07FOrLzNBL834AaSaTUqZX3noleoomslMuoaJuvimUnzYnu3Yy1 +aylwQ6BpC+S5DwIDAQABo0IwQDAdBgNVHQ4EFgQU/qHgcB4qAzlSWkK+XJGFehiq +TbUwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEL +BQADggEBAMJKVvoVIXsoounlHfv4LcQ5lkFMOycsxGwYFYDGrK9HWS8mC+M2sO87 +/kOXSTKZEhVb3xEp/6tT+LvBeA+snFOvV71ojD1pM/CjoCNjO2RnIkSt1XHLVip4 +kqNPEjE2NuLe/gDEo2APJ62gsIq1NnpSob0n9CAnYuhNlCQT5AoE6TyrLshDCUrG +YQTlSTR+08TI9Q/Aqum6VF7zYytPT1DU/rl7mYw9wC68AivTxEDkigcxHpvOJpkT ++xHqmiIMERnHXhuBUDDIlhJu58tBf5E7oke3VIAb3ADMmpDqw8NQBmIMMMAVSKeo +WXzhriKi4gp6D/piq1JM4fHfyr6DDUI= +-----END CERTIFICATE----- + +# Issuer: CN=emSign ECC Root CA - C3 O=eMudhra Inc OU=emSign PKI +# Subject: CN=emSign ECC Root CA - C3 O=eMudhra Inc OU=emSign PKI +# Label: "emSign ECC Root CA - C3" +# Serial: 582948710642506000014504 +# MD5 Fingerprint: 3e:53:b3:a3:81:ee:d7:10:f8:d3:b0:1d:17:92:f5:d5 +# SHA1 Fingerprint: b6:af:43:c2:9b:81:53:7d:f6:ef:6b:c3:1f:1f:60:15:0c:ee:48:66 +# SHA256 Fingerprint: bc:4d:80:9b:15:18:9d:78:db:3e:1d:8c:f4:f9:72:6a:79:5d:a1:64:3c:a5:f1:35:8e:1d:db:0e:dc:0d:7e:b3 +-----BEGIN CERTIFICATE----- +MIICKzCCAbGgAwIBAgIKe3G2gla4EnycqDAKBggqhkjOPQQDAzBaMQswCQYDVQQG +EwJVUzETMBEGA1UECxMKZW1TaWduIFBLSTEUMBIGA1UEChMLZU11ZGhyYSBJbmMx +IDAeBgNVBAMTF2VtU2lnbiBFQ0MgUm9vdCBDQSAtIEMzMB4XDTE4MDIxODE4MzAw +MFoXDTQzMDIxODE4MzAwMFowWjELMAkGA1UEBhMCVVMxEzARBgNVBAsTCmVtU2ln +biBQS0kxFDASBgNVBAoTC2VNdWRocmEgSW5jMSAwHgYDVQQDExdlbVNpZ24gRUND +IFJvb3QgQ0EgLSBDMzB2MBAGByqGSM49AgEGBSuBBAAiA2IABP2lYa57JhAd6bci +MK4G9IGzsUJxlTm801Ljr6/58pc1kjZGDoeVjbk5Wum739D+yAdBPLtVb4Ojavti +sIGJAnB9SMVK4+kiVCJNk7tCDK93nCOmfddhEc5lx/h//vXyqaNCMEAwHQYDVR0O +BBYEFPtaSNCAIEDyqOkAB2kZd6fmw/TPMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMB +Af8EBTADAQH/MAoGCCqGSM49BAMDA2gAMGUCMQC02C8Cif22TGK6Q04ThHK1rt0c +3ta13FaPWEBaLd4gTCKDypOofu4SQMfWh0/434UCMBwUZOR8loMRnLDRWmFLpg9J +0wD8ofzkpf9/rdcw0Md3f76BB1UwUCAU9Vc4CqgxUQ== +-----END CERTIFICATE----- + +# Issuer: CN=Hongkong Post Root CA 3 O=Hongkong Post +# Subject: CN=Hongkong Post Root CA 3 O=Hongkong Post +# Label: "Hongkong Post Root CA 3" +# Serial: 46170865288971385588281144162979347873371282084 +# MD5 Fingerprint: 11:fc:9f:bd:73:30:02:8a:fd:3f:f3:58:b9:cb:20:f0 +# SHA1 Fingerprint: 58:a2:d0:ec:20:52:81:5b:c1:f3:f8:64:02:24:4e:c2:8e:02:4b:02 +# SHA256 Fingerprint: 5a:2f:c0:3f:0c:83:b0:90:bb:fa:40:60:4b:09:88:44:6c:76:36:18:3d:f9:84:6e:17:10:1a:44:7f:b8:ef:d6 +-----BEGIN CERTIFICATE----- +MIIFzzCCA7egAwIBAgIUCBZfikyl7ADJk0DfxMauI7gcWqQwDQYJKoZIhvcNAQEL +BQAwbzELMAkGA1UEBhMCSEsxEjAQBgNVBAgTCUhvbmcgS29uZzESMBAGA1UEBxMJ +SG9uZyBLb25nMRYwFAYDVQQKEw1Ib25na29uZyBQb3N0MSAwHgYDVQQDExdIb25n +a29uZyBQb3N0IFJvb3QgQ0EgMzAeFw0xNzA2MDMwMjI5NDZaFw00MjA2MDMwMjI5 +NDZaMG8xCzAJBgNVBAYTAkhLMRIwEAYDVQQIEwlIb25nIEtvbmcxEjAQBgNVBAcT +CUhvbmcgS29uZzEWMBQGA1UEChMNSG9uZ2tvbmcgUG9zdDEgMB4GA1UEAxMXSG9u +Z2tvbmcgUG9zdCBSb290IENBIDMwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK +AoICAQCziNfqzg8gTr7m1gNt7ln8wlffKWihgw4+aMdoWJwcYEuJQwy51BWy7sFO +dem1p+/l6TWZ5Mwc50tfjTMwIDNT2aa71T4Tjukfh0mtUC1Qyhi+AViiE3CWu4mI +VoBc+L0sPOFMV4i707mV78vH9toxdCim5lSJ9UExyuUmGs2C4HDaOym71QP1mbpV +9WTRYA6ziUm4ii8F0oRFKHyPaFASePwLtVPLwpgchKOesL4jpNrcyCse2m5FHomY +2vkALgbpDDtw1VAliJnLzXNg99X/NWfFobxeq81KuEXryGgeDQ0URhLj0mRiikKY +vLTGCAj4/ahMZJx2Ab0vqWwzD9g/KLg8aQFChn5pwckGyuV6RmXpwtZQQS4/t+Tt +bNe/JgERohYpSms0BpDsE9K2+2p20jzt8NYt3eEV7KObLyzJPivkaTv/ciWxNoZb +x39ri1UbSsUgYT2uy1DhCDq+sI9jQVMwCFk8mB13umOResoQUGC/8Ne8lYePl8X+ +l2oBlKN8W4UdKjk60FSh0Tlxnf0h+bV78OLgAo9uliQlLKAeLKjEiafv7ZkGL7YK +TE/bosw3Gq9HhS2KX8Q0NEwA/RiTZxPRN+ZItIsGxVd7GYYKecsAyVKvQv83j+Gj +Hno9UKtjBucVtT+2RTeUN7F+8kjDf8V1/peNRY8apxpyKBpADwIDAQABo2MwYTAP +BgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBBjAfBgNVHSMEGDAWgBQXnc0e +i9Y5K3DTXNSguB+wAPzFYTAdBgNVHQ4EFgQUF53NHovWOStw01zUoLgfsAD8xWEw +DQYJKoZIhvcNAQELBQADggIBAFbVe27mIgHSQpsY1Q7XZiNc4/6gx5LS6ZStS6LG +7BJ8dNVI0lkUmcDrudHr9EgwW62nV3OZqdPlt9EuWSRY3GguLmLYauRwCy0gUCCk +MpXRAJi70/33MvJJrsZ64Ee+bs7Lo3I6LWldy8joRTnU+kLBEUx3XZL7av9YROXr +gZ6voJmtvqkBZss4HTzfQx/0TW60uhdG/H39h4F5ag0zD/ov+BS5gLNdTaqX4fnk +GMX41TiMJjz98iji7lpJiCzfeT2OnpA8vUFKOt1b9pq0zj8lMH8yfaIDlNDceqFS +3m6TjRgm/VWsvY+b0s+v54Ysyx8Jb6NvqYTUc79NoXQbTiNg8swOqn+knEwlqLJm +Ozj/2ZQw9nKEvmhVEA/GcywWaZMH/rFF7buiVWqw2rVKAiUnhde3t4ZEFolsgCs+ +l6mc1X5VTMbeRRAc6uk7nwNT7u56AQIWeNTowr5GdogTPyK7SBIdUgC0An4hGh6c +JfTzPV4e0hz5sy229zdcxsshTrD3mUcYhcErulWuBurQB7Lcq9CClnXO0lD+mefP +L5/ndtFhKvshuzHQqp9HpLIiyhY6UFfEW0NnxWViA0kB60PZ2Pierc+xYw5F9KBa +LJstxabArahH9CdMOA0uG0k7UvToiIMrVCjU8jVStDKDYmlkDJGcn5fqdBb9HxEG +mpv0 +-----END CERTIFICATE----- diff --git a/pipenv/vendor/certifi/core.py b/pipenv/vendor/certifi/core.py index eab9d1d1..7271acf4 100644 --- a/pipenv/vendor/certifi/core.py +++ b/pipenv/vendor/certifi/core.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- """ @@ -8,30 +7,9 @@ certifi.py This module returns the installation location of cacert.pem. """ import os -import warnings - - -class DeprecatedBundleWarning(DeprecationWarning): - """ - The weak security bundle is being deprecated. Please bother your service - provider to get them to stop using cross-signed roots. - """ def where(): f = os.path.dirname(__file__) return os.path.join(f, 'cacert.pem') - - -def old_where(): - warnings.warn( - "The weak security bundle has been removed. certifi.old_where() is now an alias " - "of certifi.where(). Please update your code to use certifi.where() instead. " - "certifi.old_where() will be removed in 2018.", - DeprecatedBundleWarning - ) - return where() - -if __name__ == '__main__': - print(where()) diff --git a/pipenv/vendor/click_completion/__init__.py b/pipenv/vendor/click_completion/__init__.py index e30cc0e3..620d7926 100644 --- a/pipenv/vendor/click_completion/__init__.py +++ b/pipenv/vendor/click_completion/__init__.py @@ -19,7 +19,7 @@ from click_completion.core import completion_configuration, get_code, install, s from click_completion.lib import get_auto_shell from click_completion.patch import patch as _patch -__version__ = '0.5.0' +__version__ = '0.5.1' _initialized = False diff --git a/pipenv/vendor/click_completion/core.py b/pipenv/vendor/click_completion/core.py index dc47d471..36150d14 100644 --- a/pipenv/vendor/click_completion/core.py +++ b/pipenv/vendor/click_completion/core.py @@ -131,8 +131,9 @@ def get_choices(cli, prog_name, args, incomplete): choices.append((opt, None)) if isinstance(ctx.command, MultiCommand): for name in ctx.command.list_commands(ctx): - if match(name, incomplete): - choices.append((name, ctx.command.get_command_short_help(ctx, name))) + command = ctx.command.get_command(ctx, name) + if match(name, incomplete) and not command.hidden: + choices.append((name, command.get_short_help_str())) for item, help in choices: yield (item, help) @@ -201,7 +202,7 @@ def do_fish_complete(cli, prog_name): for item, help in get_choices(cli, prog_name, args, incomplete): if help: - echo("%s\t%s" % (item, re.sub('\s', ' ', help))) + echo("%s\t%s" % (item, re.sub(r'\s', ' ', help))) else: echo(item) @@ -232,11 +233,11 @@ def do_zsh_complete(cli, prog_name): incomplete = '' def escape(s): - return s.replace('"', '""').replace("'", "''").replace('$', '\\$') + return s.replace('"', '""').replace("'", "''").replace('$', '\\$').replace('`', '\\`') res = [] for item, help in get_choices(cli, prog_name, args, incomplete): if help: - res.append('"%s"\:"%s"' % (escape(item), escape(help))) + res.append(r'"%s"\:"%s"' % (escape(item), escape(help))) else: res.append('"%s"' % escape(item)) if res: @@ -349,13 +350,8 @@ def install(shell=None, prog_name=None, env_name=None, path=None, append=None, e path = path or os.path.expanduser('~') + '/.bash_completion' mode = mode or 'a' elif shell == 'zsh': - ohmyzsh = os.path.expanduser('~') + '/.oh-my-zsh' - if os.path.exists(ohmyzsh): - path = path or ohmyzsh + '/completions/_%s' % prog_name - mode = mode or 'w' - else: - path = path or os.path.expanduser('~') + '/.zshrc' - mode = mode or 'a' + path = path or os.path.expanduser('~') + '/.zshrc' + mode = mode or 'a' elif shell == 'powershell': subprocess.check_call(['powershell', 'Set-ExecutionPolicy Unrestricted -Scope CurrentUser']) path = path or subprocess.check_output(['powershell', '-NoProfile', 'echo $profile']).strip() if install else '' diff --git a/pipenv/vendor/click_completion/zsh.j2 b/pipenv/vendor/click_completion/zsh.j2 index 9e1024a8..ac796615 100644 --- a/pipenv/vendor/click_completion/zsh.j2 +++ b/pipenv/vendor/click_completion/zsh.j2 @@ -3,6 +3,5 @@ _{{prog_name}}() { eval $(env COMMANDLINE="${words[1,$CURRENT]}" {{complete_var}}=complete-zsh {% for k, v in extra_env.items() %} {{k}}={{v}}{% endfor %} {{prog_name}}) } if [[ "$(basename -- ${(%):-%x})" != "_{{prog_name}}" ]]; then - autoload -U compinit && compinit compdef _{{prog_name}} {{prog_name}} fi diff --git a/pipenv/vendor/colorama/LICENSE.txt b/pipenv/vendor/colorama/LICENSE.txt index 5f567799..3105888e 100644 --- a/pipenv/vendor/colorama/LICENSE.txt +++ b/pipenv/vendor/colorama/LICENSE.txt @@ -25,4 +25,3 @@ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - diff --git a/pipenv/vendor/colorama/__init__.py b/pipenv/vendor/colorama/__init__.py index f4d9ce21..2a3bf471 100644 --- a/pipenv/vendor/colorama/__init__.py +++ b/pipenv/vendor/colorama/__init__.py @@ -3,5 +3,4 @@ from .initialise import init, deinit, reinit, colorama_text from .ansi import Fore, Back, Style, Cursor from .ansitowin32 import AnsiToWin32 -__version__ = '0.3.9' - +__version__ = '0.4.1' diff --git a/pipenv/vendor/colorama/ansitowin32.py b/pipenv/vendor/colorama/ansitowin32.py index 1d6e6059..359c92be 100644 --- a/pipenv/vendor/colorama/ansitowin32.py +++ b/pipenv/vendor/colorama/ansitowin32.py @@ -13,14 +13,6 @@ if windll is not None: winterm = WinTerm() -def is_stream_closed(stream): - return not hasattr(stream, 'closed') or stream.closed - - -def is_a_tty(stream): - return hasattr(stream, 'isatty') and stream.isatty() - - class StreamWrapper(object): ''' Wraps a stream (such as stdout), acting as a transparent proxy for all @@ -36,9 +28,38 @@ class StreamWrapper(object): def __getattr__(self, name): return getattr(self.__wrapped, name) + def __enter__(self, *args, **kwargs): + # special method lookup bypasses __getattr__/__getattribute__, see + # https://stackoverflow.com/questions/12632894/why-doesnt-getattr-work-with-exit + # thus, contextlib magic methods are not proxied via __getattr__ + return self.__wrapped.__enter__(*args, **kwargs) + + def __exit__(self, *args, **kwargs): + return self.__wrapped.__exit__(*args, **kwargs) + def write(self, text): self.__convertor.write(text) + def isatty(self): + stream = self.__wrapped + if 'PYCHARM_HOSTED' in os.environ: + if stream is not None and (stream is sys.__stdout__ or stream is sys.__stderr__): + return True + try: + stream_isatty = stream.isatty + except AttributeError: + return False + else: + return stream_isatty() + + @property + def closed(self): + stream = self.__wrapped + try: + return stream.closed + except AttributeError: + return True + class AnsiToWin32(object): ''' @@ -68,12 +89,12 @@ class AnsiToWin32(object): # should we strip ANSI sequences from our output? if strip is None: - strip = conversion_supported or (not is_stream_closed(wrapped) and not is_a_tty(wrapped)) + strip = conversion_supported or (not self.stream.closed and not self.stream.isatty()) self.strip = strip # should we should convert ANSI sequences into win32 calls? if convert is None: - convert = conversion_supported and not is_stream_closed(wrapped) and is_a_tty(wrapped) + convert = conversion_supported and not self.stream.closed and self.stream.isatty() self.convert = convert # dict of ansi codes to win32 functions and parameters @@ -149,7 +170,7 @@ class AnsiToWin32(object): def reset_all(self): if self.convert: self.call_win32('m', (0,)) - elif not self.strip and not is_stream_closed(self.wrapped): + elif not self.strip and not self.stream.closed: self.wrapped.write(Style.RESET_ALL) diff --git a/pipenv/vendor/colorama/initialise.py b/pipenv/vendor/colorama/initialise.py index 834962a3..430d0668 100644 --- a/pipenv/vendor/colorama/initialise.py +++ b/pipenv/vendor/colorama/initialise.py @@ -78,5 +78,3 @@ def wrap_stream(stream, convert, strip, autoreset, wrap): if wrapper.should_wrap(): stream = wrapper.stream return stream - - diff --git a/pipenv/vendor/colorama/win32.py b/pipenv/vendor/colorama/win32.py index 8262e350..c2d83603 100644 --- a/pipenv/vendor/colorama/win32.py +++ b/pipenv/vendor/colorama/win32.py @@ -89,11 +89,6 @@ else: ] _SetConsoleTitleW.restype = wintypes.BOOL - handles = { - STDOUT: _GetStdHandle(STDOUT), - STDERR: _GetStdHandle(STDERR), - } - def _winapi_test(handle): csbi = CONSOLE_SCREEN_BUFFER_INFO() success = _GetConsoleScreenBufferInfo( @@ -101,17 +96,18 @@ else: return bool(success) def winapi_test(): - return any(_winapi_test(h) for h in handles.values()) + return any(_winapi_test(h) for h in + (_GetStdHandle(STDOUT), _GetStdHandle(STDERR))) def GetConsoleScreenBufferInfo(stream_id=STDOUT): - handle = handles[stream_id] + handle = _GetStdHandle(stream_id) csbi = CONSOLE_SCREEN_BUFFER_INFO() success = _GetConsoleScreenBufferInfo( handle, byref(csbi)) return csbi def SetConsoleTextAttribute(stream_id, attrs): - handle = handles[stream_id] + handle = _GetStdHandle(stream_id) return _SetConsoleTextAttribute(handle, attrs) def SetConsoleCursorPosition(stream_id, position, adjust=True): @@ -129,11 +125,11 @@ else: adjusted_position.Y += sr.Top adjusted_position.X += sr.Left # Resume normal processing - handle = handles[stream_id] + handle = _GetStdHandle(stream_id) return _SetConsoleCursorPosition(handle, adjusted_position) def FillConsoleOutputCharacter(stream_id, char, length, start): - handle = handles[stream_id] + handle = _GetStdHandle(stream_id) char = c_char(char.encode()) length = wintypes.DWORD(length) num_written = wintypes.DWORD(0) @@ -144,7 +140,7 @@ else: def FillConsoleOutputAttribute(stream_id, attr, length, start): ''' FillConsoleOutputAttribute( hConsole, csbi.wAttributes, dwConSize, coordScreen, &cCharsWritten )''' - handle = handles[stream_id] + handle = _GetStdHandle(stream_id) attribute = wintypes.WORD(attr) length = wintypes.DWORD(length) num_written = wintypes.DWORD(0) diff --git a/pipenv/vendor/colorama/winterm.py b/pipenv/vendor/colorama/winterm.py index 60309d3c..0fdb4ec4 100644 --- a/pipenv/vendor/colorama/winterm.py +++ b/pipenv/vendor/colorama/winterm.py @@ -44,6 +44,7 @@ class WinTerm(object): def reset_all(self, on_stderr=None): self.set_attrs(self._default) self.set_console(attrs=self._default) + self._light = 0 def fore(self, fore=None, light=False, on_stderr=False): if fore is None: @@ -122,12 +123,15 @@ class WinTerm(object): if mode == 0: from_coord = csbi.dwCursorPosition cells_to_erase = cells_in_screen - cells_before_cursor - if mode == 1: + elif mode == 1: from_coord = win32.COORD(0, 0) cells_to_erase = cells_before_cursor elif mode == 2: from_coord = win32.COORD(0, 0) cells_to_erase = cells_in_screen + else: + # invalid mode + return # fill the entire screen with blanks win32.FillConsoleOutputCharacter(handle, ' ', cells_to_erase, from_coord) # now set the buffer's attributes accordingly @@ -147,12 +151,15 @@ class WinTerm(object): if mode == 0: from_coord = csbi.dwCursorPosition cells_to_erase = csbi.dwSize.X - csbi.dwCursorPosition.X - if mode == 1: + elif mode == 1: from_coord = win32.COORD(0, csbi.dwCursorPosition.Y) cells_to_erase = csbi.dwCursorPosition.X elif mode == 2: from_coord = win32.COORD(0, csbi.dwCursorPosition.Y) cells_to_erase = csbi.dwSize.X + else: + # invalid mode + return # fill the entire screen with blanks win32.FillConsoleOutputCharacter(handle, ' ', cells_to_erase, from_coord) # now set the buffer's attributes accordingly diff --git a/pipenv/vendor/cursor/LICENSE b/pipenv/vendor/cursor/LICENSE deleted file mode 100644 index 00023c80..00000000 --- a/pipenv/vendor/cursor/LICENSE +++ /dev/null @@ -1,5 +0,0 @@ -This work is licensed under the Creative Commons -Attribution-ShareAlike 2.5 International License. To view a copy of -this license, visit http://creativecommons.org/licenses/by-sa/2.5/ or -send a letter to Creative Commons, PO Box 1866, Mountain View, -CA 94042, USA. diff --git a/pipenv/vendor/cursor/__init__.py b/pipenv/vendor/cursor/__init__.py deleted file mode 100644 index 76a4f671..00000000 --- a/pipenv/vendor/cursor/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from .cursor import hide, show, HiddenCursor - -__all__ = ["hide", "show", "HiddenCursor"] - diff --git a/pipenv/vendor/cursor/cursor.py b/pipenv/vendor/cursor/cursor.py deleted file mode 100644 index e4407c02..00000000 --- a/pipenv/vendor/cursor/cursor.py +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env python2 -# -*- coding: utf-8 -*- - -## Author: James Spencer: http://stackoverflow.com/users/1375885/james-spencer -## Packager: Gijs TImmers: https://github.com/GijsTimmers - -## Based on James Spencer's answer on StackOverflow: -## http://stackoverflow.com/questions/5174810/how-to-turn-off-blinking-cursor-in-command-window - -## Licence: CC-BY-SA-2.5 -## http://creativecommons.org/licenses/by-sa/2.5/ - -## This work is licensed under the Creative Commons -## Attribution-ShareAlike 2.5 International License. To view a copy of -## this license, visit http://creativecommons.org/licenses/by-sa/2.5/ or -## send a letter to Creative Commons, PO Box 1866, Mountain View, -## CA 94042, USA. - -import sys -import os - -if os.name == 'nt': - import ctypes - - class _CursorInfo(ctypes.Structure): - _fields_ = [("size", ctypes.c_int), - ("visible", ctypes.c_byte)] - -def hide(stream=sys.stdout): - if os.name == 'nt': - ci = _CursorInfo() - handle = ctypes.windll.kernel32.GetStdHandle(-11) - ctypes.windll.kernel32.GetConsoleCursorInfo(handle, ctypes.byref(ci)) - ci.visible = False - ctypes.windll.kernel32.SetConsoleCursorInfo(handle, ctypes.byref(ci)) - elif os.name == 'posix': - stream.write("\033[?25l") - stream.flush() - -def show(stream=sys.stdout): - if os.name == 'nt': - ci = _CursorInfo() - handle = ctypes.windll.kernel32.GetStdHandle(-11) - ctypes.windll.kernel32.GetConsoleCursorInfo(handle, ctypes.byref(ci)) - ci.visible = True - ctypes.windll.kernel32.SetConsoleCursorInfo(handle, ctypes.byref(ci)) - elif os.name == 'posix': - stream.write("\033[?25h") - stream.flush() - -class HiddenCursor(object): - def __init__(self, stream=sys.stdout): - self._stream = stream - def __enter__(self): - hide(stream=self._stream) - def __exit__(self, type, value, traceback): - show(stream=self._stream) \ No newline at end of file diff --git a/pipenv/vendor/distlib/__init__.py b/pipenv/vendor/distlib/__init__.py index a786b4d3..08fe1fc4 100644 --- a/pipenv/vendor/distlib/__init__.py +++ b/pipenv/vendor/distlib/__init__.py @@ -6,7 +6,7 @@ # import logging -__version__ = '0.2.8' +__version__ = '0.2.9' class DistlibException(Exception): pass diff --git a/pipenv/vendor/distlib/index.py b/pipenv/vendor/distlib/index.py index 2406be21..7a87cdcf 100644 --- a/pipenv/vendor/distlib/index.py +++ b/pipenv/vendor/distlib/index.py @@ -22,7 +22,7 @@ from .util import cached_property, zip_dir, ServerProxy logger = logging.getLogger(__name__) -DEFAULT_INDEX = 'https://pypi.python.org/pypi' +DEFAULT_INDEX = 'https://pypi.org/pypi' DEFAULT_REALM = 'pypi' class PackageIndex(object): diff --git a/pipenv/vendor/distlib/locators.py b/pipenv/vendor/distlib/locators.py index 5c655c3e..a7ed9469 100644 --- a/pipenv/vendor/distlib/locators.py +++ b/pipenv/vendor/distlib/locators.py @@ -36,7 +36,7 @@ logger = logging.getLogger(__name__) HASHER_HASH = re.compile(r'^(\w+)=([a-f0-9]+)') CHARSET = re.compile(r';\s*charset\s*=\s*(.*)\s*$', re.I) HTML_CONTENT_TYPE = re.compile('text/html|application/x(ht)?ml') -DEFAULT_INDEX = 'https://pypi.python.org/pypi' +DEFAULT_INDEX = 'https://pypi.org/pypi' def get_all_distribution_names(url=None): """ @@ -197,7 +197,7 @@ class Locator(object): is_downloadable = basename.endswith(self.downloadable_extensions) if is_wheel: compatible = is_compatible(Wheel(basename), self.wheel_tags) - return (t.scheme == 'https', 'pypi.python.org' in t.netloc, + return (t.scheme == 'https', 'pypi.org' in t.netloc, is_downloadable, is_wheel, compatible, basename) def prefer_url(self, url1, url2): @@ -1049,7 +1049,7 @@ class AggregatingLocator(Locator): # versions which don't conform to PEP 426 / PEP 440. default_locator = AggregatingLocator( JSONLocator(), - SimpleScrapingLocator('https://pypi.python.org/simple/', + SimpleScrapingLocator('https://pypi.org/simple/', timeout=3.0), scheme='legacy') diff --git a/pipenv/vendor/distlib/metadata.py b/pipenv/vendor/distlib/metadata.py index 77eed7f9..2d61378e 100644 --- a/pipenv/vendor/distlib/metadata.py +++ b/pipenv/vendor/distlib/metadata.py @@ -91,9 +91,11 @@ _426_FIELDS = ('Metadata-Version', 'Name', 'Version', 'Platform', _426_MARKERS = ('Private-Version', 'Provides-Extra', 'Obsoleted-By', 'Setup-Requires-Dist', 'Extension') -# See issue #106: Sometimes 'Requires' occurs wrongly in the metadata. Include -# it in the tuple literal below to allow it (for now) -_566_FIELDS = _426_FIELDS + ('Description-Content-Type', 'Requires') +# See issue #106: Sometimes 'Requires' and 'Provides' occur wrongly in +# the metadata. Include them in the tuple literal below to allow them +# (for now). +_566_FIELDS = _426_FIELDS + ('Description-Content-Type', + 'Requires', 'Provides') _566_MARKERS = ('Description-Content-Type',) diff --git a/pipenv/vendor/distlib/scripts.py b/pipenv/vendor/distlib/scripts.py index 8e22cb91..5965e241 100644 --- a/pipenv/vendor/distlib/scripts.py +++ b/pipenv/vendor/distlib/scripts.py @@ -39,27 +39,12 @@ _DEFAULT_MANIFEST = ''' # check if Python is called on the first line with this expression FIRST_LINE_RE = re.compile(b'^#!.*pythonw?[0-9.]*([ \t].*)?$') SCRIPT_TEMPLATE = r'''# -*- coding: utf-8 -*- +import re +import sys +from %(module)s import %(import_name)s if __name__ == '__main__': - import sys, re - - def _resolve(module, func): - __import__(module) - mod = sys.modules[module] - parts = func.split('.') - result = getattr(mod, parts.pop(0)) - for p in parts: - result = getattr(result, p) - return result - - try: - sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) - - func = _resolve('%(module)s', '%(func)s') - rc = func() # None interpreted as 0 - except Exception as e: # only supporting Python >= 2.6 - sys.stderr.write('%%s\n' %% e) - rc = 1 - sys.exit(rc) + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(%(func)s()) ''' @@ -225,6 +210,7 @@ class ScriptMaker(object): def _get_script_text(self, entry): return self.script_template % dict(module=entry.prefix, + import_name=entry.suffix.split('.')[0], func=entry.suffix) manifest = _DEFAULT_MANIFEST diff --git a/pipenv/vendor/distlib/util.py b/pipenv/vendor/distlib/util.py index 9d4bfd3b..e851146c 100644 --- a/pipenv/vendor/distlib/util.py +++ b/pipenv/vendor/distlib/util.py @@ -804,11 +804,15 @@ def ensure_slash(s): def parse_credentials(netloc): username = password = None if '@' in netloc: - prefix, netloc = netloc.split('@', 1) + prefix, netloc = netloc.rsplit('@', 1) if ':' not in prefix: username = prefix else: username, password = prefix.split(':', 1) + if username: + username = unquote(username) + if password: + password = unquote(password) return username, password, netloc diff --git a/pipenv/vendor/distlib/wheel.py b/pipenv/vendor/distlib/wheel.py index b04bfaef..0c8efad9 100644 --- a/pipenv/vendor/distlib/wheel.py +++ b/pipenv/vendor/distlib/wheel.py @@ -433,6 +433,22 @@ class Wheel(object): self.build_zip(pathname, archive_paths) return pathname + def skip_entry(self, arcname): + """ + Determine whether an archive entry should be skipped when verifying + or installing. + """ + # The signature file won't be in RECORD, + # and we don't currently don't do anything with it + # We also skip directories, as they won't be in RECORD + # either. See: + # + # https://github.com/pypa/wheel/issues/294 + # https://github.com/pypa/wheel/issues/287 + # https://github.com/pypa/wheel/pull/289 + # + return arcname.endswith(('/', '/RECORD.jws')) + def install(self, paths, maker, **kwargs): """ Install a wheel to the specified paths. If kwarg ``warner`` is @@ -514,9 +530,7 @@ class Wheel(object): u_arcname = arcname else: u_arcname = arcname.decode('utf-8') - # The signature file won't be in RECORD, - # and we don't currently don't do anything with it - if u_arcname.endswith('/RECORD.jws'): + if self.skip_entry(u_arcname): continue row = records[u_arcname] if row[2] and str(zinfo.file_size) != row[2]: @@ -786,13 +800,15 @@ class Wheel(object): u_arcname = arcname else: u_arcname = arcname.decode('utf-8') - if '..' in u_arcname: + # See issue #115: some wheels have .. in their entries, but + # in the filename ... e.g. __main__..py ! So the check is + # updated to look for .. in the directory portions + p = u_arcname.split('/') + if '..' in p: raise DistlibException('invalid entry in ' 'wheel: %r' % u_arcname) - # The signature file won't be in RECORD, - # and we don't currently don't do anything with it - if u_arcname.endswith('/RECORD.jws'): + if self.skip_entry(u_arcname): continue row = records[u_arcname] if row[2] and str(zinfo.file_size) != row[2]: diff --git a/pipenv/vendor/dotenv/__init__.py b/pipenv/vendor/dotenv/__init__.py index 50f27cd4..b88d9bc2 100644 --- a/pipenv/vendor/dotenv/__init__.py +++ b/pipenv/vendor/dotenv/__init__.py @@ -1,12 +1,18 @@ +from .compat import IS_TYPE_CHECKING from .main import load_dotenv, get_key, set_key, unset_key, find_dotenv, dotenv_values +if IS_TYPE_CHECKING: + from typing import Any, Optional + def load_ipython_extension(ipython): + # type: (Any) -> None from .ipython import load_ipython_extension load_ipython_extension(ipython) def get_cli_string(path=None, action=None, key=None, value=None, quote=None): + # type: (Optional[str], Optional[str], Optional[str], Optional[str], Optional[str]) -> str """Returns a string suitable for running as a shell script. Useful for converting a arguments passed to a fabric task diff --git a/pipenv/vendor/dotenv/cli.py b/pipenv/vendor/dotenv/cli.py index 4e03c12a..829b14ad 100644 --- a/pipenv/vendor/dotenv/cli.py +++ b/pipenv/vendor/dotenv/cli.py @@ -8,9 +8,13 @@ except ImportError: 'Run pip install "python-dotenv[cli]" to fix this.') sys.exit(1) +from .compat import IS_TYPE_CHECKING from .main import dotenv_values, get_key, set_key, unset_key, run_command from .version import __version__ +if IS_TYPE_CHECKING: + from typing import Any, List + @click.group() @click.option('-f', '--file', default=os.path.join(os.getcwd(), '.env'), @@ -22,6 +26,7 @@ from .version import __version__ @click.version_option(version=__version__) @click.pass_context def cli(ctx, file, quote): + # type: (click.Context, Any, Any) -> None '''This script is used to set, get or unset values from a .env file.''' ctx.obj = {} ctx.obj['FILE'] = file @@ -31,6 +36,7 @@ def cli(ctx, file, quote): @cli.command() @click.pass_context def list(ctx): + # type: (click.Context) -> None '''Display all the stored key/value.''' file = ctx.obj['FILE'] dotenv_as_dict = dotenv_values(file) @@ -43,6 +49,7 @@ def list(ctx): @click.argument('key', required=True) @click.argument('value', required=True) def set(ctx, key, value): + # type: (click.Context, Any, Any) -> None '''Store the given key/value.''' file = ctx.obj['FILE'] quote = ctx.obj['QUOTE'] @@ -57,6 +64,7 @@ def set(ctx, key, value): @click.pass_context @click.argument('key', required=True) def get(ctx, key): + # type: (click.Context, Any) -> None '''Retrieve the value for the given key.''' file = ctx.obj['FILE'] stored_value = get_key(file, key) @@ -70,6 +78,7 @@ def get(ctx, key): @click.pass_context @click.argument('key', required=True) def unset(ctx, key): + # type: (click.Context, Any) -> None '''Removes the given key.''' file = ctx.obj['FILE'] quote = ctx.obj['QUOTE'] @@ -84,13 +93,14 @@ def unset(ctx, key): @click.pass_context @click.argument('commandline', nargs=-1, type=click.UNPROCESSED) def run(ctx, commandline): + # type: (click.Context, List[str]) -> None """Run command with environment variables present.""" file = ctx.obj['FILE'] dotenv_as_dict = dotenv_values(file) if not commandline: click.echo('No command given.') exit(1) - ret = run_command(commandline, dotenv_as_dict) + ret = run_command(commandline, dotenv_as_dict) # type: ignore exit(ret) diff --git a/pipenv/vendor/dotenv/compat.py b/pipenv/vendor/dotenv/compat.py index c4a481e6..7a8694fc 100644 --- a/pipenv/vendor/dotenv/compat.py +++ b/pipenv/vendor/dotenv/compat.py @@ -1,4 +1,21 @@ -try: - from StringIO import StringIO # noqa -except ImportError: +import os +import sys + +if sys.version_info >= (3, 0): from io import StringIO # noqa +else: + from StringIO import StringIO # noqa + +PY2 = sys.version_info[0] == 2 # type: bool + + +def is_type_checking(): + # type: () -> bool + try: + from typing import TYPE_CHECKING + except ImportError: + return False + return TYPE_CHECKING + + +IS_TYPE_CHECKING = os.environ.get("MYPY_RUNNING", is_type_checking()) diff --git a/pipenv/vendor/dotenv/ipython.py b/pipenv/vendor/dotenv/ipython.py index 06252f1e..7f1b13d6 100644 --- a/pipenv/vendor/dotenv/ipython.py +++ b/pipenv/vendor/dotenv/ipython.py @@ -1,8 +1,8 @@ from __future__ import print_function -from IPython.core.magic import Magics, line_magic, magics_class -from IPython.core.magic_arguments import (argument, magic_arguments, - parse_argstring) +from IPython.core.magic import Magics, line_magic, magics_class # type: ignore +from IPython.core.magic_arguments import (argument, magic_arguments, # type: ignore + parse_argstring) # type: ignore from .main import find_dotenv, load_dotenv diff --git a/pipenv/vendor/dotenv/main.py b/pipenv/vendor/dotenv/main.py index 6ba28bbb..64d42696 100644 --- a/pipenv/vendor/dotenv/main.py +++ b/pipenv/vendor/dotenv/main.py @@ -2,71 +2,154 @@ from __future__ import absolute_import, print_function, unicode_literals import codecs -import fileinput import io import os import re +import shutil import sys -from subprocess import Popen, PIPE, STDOUT +from subprocess import Popen +import tempfile import warnings from collections import OrderedDict +from contextlib import contextmanager -from .compat import StringIO +from .compat import StringIO, PY2, IS_TYPE_CHECKING -__escape_decoder = codecs.getdecoder('unicode_escape') -__posix_variable = re.compile('\$\{[^\}]*\}') +if IS_TYPE_CHECKING: # pragma: no cover + from typing import ( + Dict, Iterator, List, Match, Optional, Pattern, Union, + Text, IO, Tuple + ) + if sys.version_info >= (3, 6): + _PathLike = os.PathLike + else: + _PathLike = Text + + if sys.version_info >= (3, 0): + _StringIO = StringIO + else: + _StringIO = StringIO[Text] + +__posix_variable = re.compile(r'\$\{[^\}]*\}') # type: Pattern[Text] + +_binding = re.compile( + r""" + ( + \s* # leading whitespace + (?:export{0}+)? # export + + ( '[^']+' # single-quoted key + | [^=\#\s]+ # or unquoted key + )? + + (?: + (?:{0}*={0}*) # equal sign + + ( '(?:\\'|[^'])*' # single-quoted value + | "(?:\\"|[^"])*" # or double-quoted value + | [^\#\r\n]* # or unquoted value + ) + )? + + \s* # trailing whitespace + (?:\#[^\r\n]*)? # comment + (?:\r|\n|\r\n)? # newline + ) + """.format(r'[^\S\r\n]'), + re.MULTILINE | re.VERBOSE, +) # type: Pattern[Text] + +_escape_sequence = re.compile(r"\\[\\'\"abfnrtv]") # type: Pattern[Text] + +try: + from typing import NamedTuple, Optional, Text + Binding = NamedTuple("Binding", [("key", Optional[Text]), + ("value", Optional[Text]), + ("original", Text)]) +except ImportError: + from collections import namedtuple + Binding = namedtuple("Binding", ["key", "value", "original"]) -def decode_escaped(escaped): - return __escape_decoder(escaped)[0] +def decode_escapes(string): + # type: (Text) -> Text + def decode_match(match): + # type: (Match[Text]) -> Text + return codecs.decode(match.group(0), 'unicode-escape') # type: ignore + + return _escape_sequence.sub(decode_match, string) -def parse_line(line): - line = line.strip() +def is_surrounded_by(string, char): + # type: (Text, Text) -> bool + return ( + len(string) > 1 + and string[0] == string[-1] == char + ) - # Ignore lines with `#` or which doesn't have `=` in it. - if not line or line.startswith('#') or '=' not in line: - return None, None - k, v = line.split('=', 1) +def parse_binding(string, position): + # type: (Text, int) -> Tuple[Binding, int] + match = _binding.match(string, position) + assert match is not None + (matched, key, value) = match.groups() + if key is None or value is None: + key = None + value = None + else: + value_quoted = is_surrounded_by(value, "'") or is_surrounded_by(value, '"') + if value_quoted: + value = decode_escapes(value[1:-1]) + else: + value = value.strip() + return (Binding(key=key, value=value, original=matched), match.end()) - if k.startswith('export '): - (_, _, k) = k.partition('export ') - # Remove any leading and trailing spaces in key, value - k, v = k.strip(), v.strip() +def parse_stream(stream): + # type:(IO[Text]) -> Iterator[Binding] + string = stream.read() + position = 0 + length = len(string) + while position < length: + (binding, position) = parse_binding(string, position) + yield binding - if v: - v = v.encode('unicode-escape').decode('ascii') - quoted = v[0] == v[-1] in ['"', "'"] - if quoted: - v = decode_escaped(v[1:-1]) - return k, v +def to_env(text): + # type: (Text) -> str + """ + Encode a string the same way whether it comes from the environment or a `.env` file. + """ + if PY2: + return text.encode(sys.getfilesystemencoding() or "utf-8") + else: + return text class DotEnv(): - def __init__(self, dotenv_path, verbose=False): - self.dotenv_path = dotenv_path - self._dict = None - self.verbose = verbose + def __init__(self, dotenv_path, verbose=False, encoding=None): + # type: (Union[Text, _PathLike, _StringIO], bool, Union[None, Text]) -> None + self.dotenv_path = dotenv_path # type: Union[Text,_PathLike, _StringIO] + self._dict = None # type: Optional[Dict[Text, Text]] + self.verbose = verbose # type: bool + self.encoding = encoding # type: Union[None, Text] + @contextmanager def _get_stream(self): - self._is_file = False + # type: () -> Iterator[IO[Text]] if isinstance(self.dotenv_path, StringIO): - return self.dotenv_path - - if os.path.exists(self.dotenv_path): - self._is_file = True - return io.open(self.dotenv_path) - - if self.verbose: - warnings.warn("File doesn't exist {}".format(self.dotenv_path)) - - return StringIO('') + yield self.dotenv_path + elif os.path.isfile(self.dotenv_path): + with io.open(self.dotenv_path, encoding=self.encoding) as stream: + yield stream + else: + if self.verbose: + warnings.warn("File doesn't exist {}".format(self.dotenv_path)) # type: ignore + yield StringIO('') def dict(self): + # type: () -> Dict[Text, Text] """Return dotenv as dict""" if self._dict: return self._dict @@ -76,37 +159,26 @@ class DotEnv(): return self._dict def parse(self): - f = self._get_stream() - - for line in f: - key, value = parse_line(line) - if not key: - continue - - yield key, value - - if self._is_file: - f.close() + # type: () -> Iterator[Tuple[Text, Text]] + with self._get_stream() as stream: + for mapping in parse_stream(stream): + if mapping.key is not None and mapping.value is not None: + yield mapping.key, mapping.value def set_as_environment_variables(self, override=False): + # type: (bool) -> bool """ Load the current dotenv as system environemt variable. """ for k, v in self.dict().items(): if k in os.environ and not override: continue - # With Python 2 on Windows, ensuree environment variables are - # system strings to avoid "TypeError: environment can only contain - # strings" in Python's subprocess module. - if sys.version_info.major < 3 and sys.platform == 'win32': - from pipenv.utils import fs_str - k = fs_str(k) - v = fs_str(v) - os.environ[k] = v + os.environ[to_env(k)] = to_env(v) return True def get(self, key): + # type: (Text) -> Optional[Text] """ """ data = self.dict() @@ -115,10 +187,13 @@ class DotEnv(): return data[key] if self.verbose: - warnings.warn("key %s not found in %s." % (key, self.dotenv_path)) + warnings.warn("key %s not found in %s." % (key, self.dotenv_path)) # type: ignore + + return None def get_key(dotenv_path, key_to_get): + # type: (Union[Text, _PathLike], Text) -> Optional[Text] """ Gets the value of a given key from the given .env @@ -127,7 +202,23 @@ def get_key(dotenv_path, key_to_get): return DotEnv(dotenv_path, verbose=True).get(key_to_get) +@contextmanager +def rewrite(path): + # type: (_PathLike) -> Iterator[Tuple[IO[Text], IO[Text]]] + try: + with tempfile.NamedTemporaryFile(mode="w+", delete=False) as dest: + with io.open(path) as source: + yield (source, dest) # type: ignore + except BaseException: + if os.path.isfile(dest.name): + os.unlink(dest.name) + raise + else: + shutil.move(dest.name, path) + + def set_key(dotenv_path, key_to_set, value_to_set, quote_mode="always"): + # type: (_PathLike, Text, Text, Text) -> Tuple[Optional[bool], Text, Text] """ Adds or Updates a key/value to the given .env @@ -136,81 +227,86 @@ def set_key(dotenv_path, key_to_set, value_to_set, quote_mode="always"): """ value_to_set = value_to_set.strip("'").strip('"') if not os.path.exists(dotenv_path): - warnings.warn("can't write to %s - it doesn't exist." % dotenv_path) + warnings.warn("can't write to %s - it doesn't exist." % dotenv_path) # type: ignore return None, key_to_set, value_to_set if " " in value_to_set: quote_mode = "always" - line_template = '{}="{}"' if quote_mode == "always" else '{}={}' + line_template = '{}="{}"\n' if quote_mode == "always" else '{}={}\n' line_out = line_template.format(key_to_set, value_to_set) - replaced = False - for line in fileinput.input(dotenv_path, inplace=True): - k, v = parse_line(line) - if k == key_to_set: - replaced = True - line = line_out - print(line, end='') - - if not replaced: - with io.open(dotenv_path, "a") as f: - f.write("{}\n".format(line_out)) + with rewrite(dotenv_path) as (source, dest): + replaced = False + for mapping in parse_stream(source): + if mapping.key == key_to_set: + dest.write(line_out) + replaced = True + else: + dest.write(mapping.original) + if not replaced: + dest.write(line_out) return True, key_to_set, value_to_set def unset_key(dotenv_path, key_to_unset, quote_mode="always"): + # type: (_PathLike, Text, Text) -> Tuple[Optional[bool], Text] """ Removes a given key from the given .env If the .env path given doesn't exist, fails If the given key doesn't exist in the .env, fails """ - removed = False - if not os.path.exists(dotenv_path): - warnings.warn("can't delete from %s - it doesn't exist." % dotenv_path) + warnings.warn("can't delete from %s - it doesn't exist." % dotenv_path) # type: ignore return None, key_to_unset - for line in fileinput.input(dotenv_path, inplace=True): - k, v = parse_line(line) - if k == key_to_unset: - removed = True - line = '' - print(line, end='') + removed = False + with rewrite(dotenv_path) as (source, dest): + for mapping in parse_stream(source): + if mapping.key == key_to_unset: + removed = True + else: + dest.write(mapping.original) if not removed: - warnings.warn("key %s not removed from %s - key doesn't exist." % (key_to_unset, dotenv_path)) + warnings.warn("key %s not removed from %s - key doesn't exist." % (key_to_unset, dotenv_path)) # type: ignore return None, key_to_unset return removed, key_to_unset def resolve_nested_variables(values): + # type: (Dict[Text, Text]) -> Dict[Text, Text] def _replacement(name): + # type: (Text) -> Text """ get appropriate value for a variable name. first search in environ, if not found, then look into the dotenv variables """ - ret = os.getenv(name, values.get(name, "")) + ret = os.getenv(name, new_values.get(name, "")) return ret def _re_sub_callback(match_object): + # type: (Match[Text]) -> Text """ From a match object gets the variable name and returns the correct replacement """ return _replacement(match_object.group()[2:-1]) - for k, v in values.items(): - values[k] = __posix_variable.sub(_re_sub_callback, v) + new_values = {} - return values + for k, v in values.items(): + new_values[k] = __posix_variable.sub(_re_sub_callback, v) + + return new_values def _walk_to_root(path): + # type: (Text) -> Iterator[Text] """ Yield directories starting from the given directory up to the root """ @@ -229,6 +325,7 @@ def _walk_to_root(path): def find_dotenv(filename='.env', raise_error_if_not_found=False, usecwd=False): + # type: (Text, bool, bool) -> Text """ Search in increasingly higher folders for the given file @@ -241,14 +338,21 @@ def find_dotenv(filename='.env', raise_error_if_not_found=False, usecwd=False): # will work for .py files frame = sys._getframe() # find first frame that is outside of this file - while frame.f_code.co_filename == __file__: + if PY2 and not __file__.endswith('.py'): + # in Python2 __file__ extension could be .pyc or .pyo (this doesn't account + # for edge case of Python compiled for non-standard extension) + current_file = __file__.rsplit('.', 1)[0] + '.py' + else: + current_file = __file__ + + while frame.f_code.co_filename == current_file: frame = frame.f_back frame_filename = frame.f_code.co_filename path = os.path.dirname(os.path.abspath(frame_filename)) for dirname in _walk_to_root(path): check_path = os.path.join(dirname, filename) - if os.path.exists(check_path): + if os.path.isfile(check_path): return check_path if raise_error_if_not_found: @@ -257,17 +361,20 @@ def find_dotenv(filename='.env', raise_error_if_not_found=False, usecwd=False): return '' -def load_dotenv(dotenv_path=None, stream=None, verbose=False, override=False): +def load_dotenv(dotenv_path=None, stream=None, verbose=False, override=False, **kwargs): + # type: (Union[Text, _PathLike, None], Optional[_StringIO], bool, bool, Union[None, Text]) -> bool f = dotenv_path or stream or find_dotenv() - return DotEnv(f, verbose=verbose).set_as_environment_variables(override=override) + return DotEnv(f, verbose=verbose, **kwargs).set_as_environment_variables(override=override) -def dotenv_values(dotenv_path=None, stream=None, verbose=False): +def dotenv_values(dotenv_path=None, stream=None, verbose=False, **kwargs): + # type: (Union[Text, _PathLike, None], Optional[_StringIO], bool, Union[None, Text]) -> Dict[Text, Text] f = dotenv_path or stream or find_dotenv() - return DotEnv(f, verbose=verbose).dict() + return DotEnv(f, verbose=verbose, **kwargs).dict() def run_command(command, env): + # type: (List[str], Dict[str, str]) -> int """Run command in sub process. Runs the command in a sub process with the variables from `env` @@ -292,19 +399,10 @@ def run_command(command, env): cmd_env.update(env) p = Popen(command, - stdin=PIPE, - stdout=PIPE, - stderr=STDOUT, universal_newlines=True, bufsize=0, shell=False, env=cmd_env) - try: - out, _ = p.communicate() - print(out) - except Exception: - warnings.warn('An error occured, running the command:') - out, _ = p.communicate() - warnings.warn(out) + _, _ = p.communicate() return p.returncode diff --git a/pipenv/vendor/dotenv/py.typed b/pipenv/vendor/dotenv/py.typed new file mode 100644 index 00000000..7632ecf7 --- /dev/null +++ b/pipenv/vendor/dotenv/py.typed @@ -0,0 +1 @@ +# Marker file for PEP 561 diff --git a/pipenv/vendor/dotenv/version.py b/pipenv/vendor/dotenv/version.py index d69d16e9..17c1a626 100644 --- a/pipenv/vendor/dotenv/version.py +++ b/pipenv/vendor/dotenv/version.py @@ -1 +1 @@ -__version__ = "0.9.1" +__version__ = "0.10.2" diff --git a/pipenv/vendor/idna/__init__.py b/pipenv/vendor/idna/__init__.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/idna/codec.py b/pipenv/vendor/idna/codec.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/idna/compat.py b/pipenv/vendor/idna/compat.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/idna/core.py b/pipenv/vendor/idna/core.py old mode 100755 new mode 100644 index 090c2c18..104624ad --- a/pipenv/vendor/idna/core.py +++ b/pipenv/vendor/idna/core.py @@ -267,10 +267,7 @@ def alabel(label): try: label = label.encode('ascii') - try: - ulabel(label) - except IDNAError: - raise IDNAError('The label {0} is not a valid A-label'.format(label)) + ulabel(label) if not valid_label_length(label): raise IDNAError('Label too long') return label diff --git a/pipenv/vendor/idna/idnadata.py b/pipenv/vendor/idna/idnadata.py old mode 100755 new mode 100644 index 17974e23..a80c959d --- a/pipenv/vendor/idna/idnadata.py +++ b/pipenv/vendor/idna/idnadata.py @@ -1,6 +1,6 @@ # This file is automatically generated by tools/idna-data -__version__ = "10.0.0" +__version__ = "11.0.0" scripts = { 'Greek': ( 0x37000000374, @@ -49,7 +49,7 @@ scripts = { 0x30210000302a, 0x30380000303c, 0x340000004db6, - 0x4e0000009feb, + 0x4e0000009ff0, 0xf9000000fa6e, 0xfa700000fada, 0x200000002a6d7, @@ -62,7 +62,7 @@ scripts = { 'Hebrew': ( 0x591000005c8, 0x5d0000005eb, - 0x5f0000005f5, + 0x5ef000005f5, 0xfb1d0000fb37, 0xfb380000fb3d, 0xfb3e0000fb3f, @@ -248,6 +248,7 @@ joining_types = { 0x6fb: 68, 0x6fc: 68, 0x6ff: 68, + 0x70f: 84, 0x710: 82, 0x712: 68, 0x713: 68, @@ -522,6 +523,7 @@ joining_types = { 0x1875: 68, 0x1876: 68, 0x1877: 68, + 0x1878: 68, 0x1880: 85, 0x1881: 85, 0x1882: 85, @@ -690,6 +692,70 @@ joining_types = { 0x10bad: 68, 0x10bae: 68, 0x10baf: 85, + 0x10d00: 76, + 0x10d01: 68, + 0x10d02: 68, + 0x10d03: 68, + 0x10d04: 68, + 0x10d05: 68, + 0x10d06: 68, + 0x10d07: 68, + 0x10d08: 68, + 0x10d09: 68, + 0x10d0a: 68, + 0x10d0b: 68, + 0x10d0c: 68, + 0x10d0d: 68, + 0x10d0e: 68, + 0x10d0f: 68, + 0x10d10: 68, + 0x10d11: 68, + 0x10d12: 68, + 0x10d13: 68, + 0x10d14: 68, + 0x10d15: 68, + 0x10d16: 68, + 0x10d17: 68, + 0x10d18: 68, + 0x10d19: 68, + 0x10d1a: 68, + 0x10d1b: 68, + 0x10d1c: 68, + 0x10d1d: 68, + 0x10d1e: 68, + 0x10d1f: 68, + 0x10d20: 68, + 0x10d21: 68, + 0x10d22: 82, + 0x10d23: 68, + 0x10f30: 68, + 0x10f31: 68, + 0x10f32: 68, + 0x10f33: 82, + 0x10f34: 68, + 0x10f35: 68, + 0x10f36: 68, + 0x10f37: 68, + 0x10f38: 68, + 0x10f39: 68, + 0x10f3a: 68, + 0x10f3b: 68, + 0x10f3c: 68, + 0x10f3d: 68, + 0x10f3e: 68, + 0x10f3f: 68, + 0x10f40: 68, + 0x10f41: 68, + 0x10f42: 68, + 0x10f43: 68, + 0x10f44: 68, + 0x10f45: 85, + 0x10f51: 68, + 0x10f52: 68, + 0x10f53: 68, + 0x10f54: 82, + 0x110bd: 85, + 0x110cd: 85, 0x1e900: 68, 0x1e901: 68, 0x1e902: 68, @@ -1034,14 +1100,15 @@ codepoint_classes = { 0x52d0000052e, 0x52f00000530, 0x5590000055a, - 0x56100000587, + 0x56000000587, + 0x58800000589, 0x591000005be, 0x5bf000005c0, 0x5c1000005c3, 0x5c4000005c6, 0x5c7000005c8, 0x5d0000005eb, - 0x5f0000005f3, + 0x5ef000005f3, 0x6100000061b, 0x62000000640, 0x64100000660, @@ -1054,12 +1121,13 @@ codepoint_classes = { 0x7100000074b, 0x74d000007b2, 0x7c0000007f6, + 0x7fd000007fe, 0x8000000082e, 0x8400000085c, 0x8600000086b, 0x8a0000008b5, 0x8b6000008be, - 0x8d4000008e2, + 0x8d3000008e2, 0x8e300000958, 0x96000000964, 0x96600000970, @@ -1077,6 +1145,7 @@ codepoint_classes = { 0x9e0000009e4, 0x9e6000009f2, 0x9fc000009fd, + 0x9fe000009ff, 0xa0100000a04, 0xa0500000a0b, 0xa0f00000a11, @@ -1136,8 +1205,7 @@ codepoint_classes = { 0xbd000000bd1, 0xbd700000bd8, 0xbe600000bf0, - 0xc0000000c04, - 0xc0500000c0d, + 0xc0000000c0d, 0xc0e00000c11, 0xc1200000c29, 0xc2a00000c3a, @@ -1276,7 +1344,7 @@ codepoint_classes = { 0x17dc000017de, 0x17e0000017ea, 0x18100000181a, - 0x182000001878, + 0x182000001879, 0x1880000018ab, 0x18b0000018f6, 0x19000000191f, @@ -1544,11 +1612,11 @@ codepoint_classes = { 0x309d0000309f, 0x30a1000030fb, 0x30fc000030ff, - 0x31050000312f, + 0x310500003130, 0x31a0000031bb, 0x31f000003200, 0x340000004db6, - 0x4e0000009feb, + 0x4e0000009ff0, 0xa0000000a48d, 0xa4d00000a4fe, 0xa5000000a60d, @@ -1655,8 +1723,10 @@ codepoint_classes = { 0xa7a50000a7a6, 0xa7a70000a7a8, 0xa7a90000a7aa, + 0xa7af0000a7b0, 0xa7b50000a7b6, 0xa7b70000a7b8, + 0xa7b90000a7ba, 0xa7f70000a7f8, 0xa7fa0000a828, 0xa8400000a874, @@ -1664,8 +1734,7 @@ codepoint_classes = { 0xa8d00000a8da, 0xa8e00000a8f8, 0xa8fb0000a8fc, - 0xa8fd0000a8fe, - 0xa9000000a92e, + 0xa8fd0000a92e, 0xa9300000a954, 0xa9800000a9c1, 0xa9cf0000a9da, @@ -1743,7 +1812,7 @@ codepoint_classes = { 0x10a0500010a07, 0x10a0c00010a14, 0x10a1500010a18, - 0x10a1900010a34, + 0x10a1900010a36, 0x10a3800010a3b, 0x10a3f00010a40, 0x10a6000010a7d, @@ -1756,6 +1825,11 @@ codepoint_classes = { 0x10b8000010b92, 0x10c0000010c49, 0x10cc000010cf3, + 0x10d0000010d28, + 0x10d3000010d3a, + 0x10f0000010f1d, + 0x10f2700010f28, + 0x10f3000010f51, 0x1100000011047, 0x1106600011070, 0x1107f000110bb, @@ -1763,10 +1837,11 @@ codepoint_classes = { 0x110f0000110fa, 0x1110000011135, 0x1113600011140, + 0x1114400011147, 0x1115000011174, 0x1117600011177, 0x11180000111c5, - 0x111ca000111cd, + 0x111c9000111cd, 0x111d0000111db, 0x111dc000111dd, 0x1120000011212, @@ -1786,7 +1861,7 @@ codepoint_classes = { 0x1132a00011331, 0x1133200011334, 0x113350001133a, - 0x1133c00011345, + 0x1133b00011345, 0x1134700011349, 0x1134b0001134e, 0x1135000011351, @@ -1796,6 +1871,7 @@ codepoint_classes = { 0x1137000011375, 0x114000001144b, 0x114500001145a, + 0x1145e0001145f, 0x11480000114c6, 0x114c7000114c8, 0x114d0000114da, @@ -1807,15 +1883,17 @@ codepoint_classes = { 0x116500001165a, 0x11680000116b8, 0x116c0000116ca, - 0x117000001171a, + 0x117000001171b, 0x1171d0001172c, 0x117300001173a, + 0x118000001183b, 0x118c0000118ea, 0x118ff00011900, 0x11a0000011a3f, 0x11a4700011a48, 0x11a5000011a84, 0x11a8600011a9a, + 0x11a9d00011a9e, 0x11ac000011af9, 0x11c0000011c09, 0x11c0a00011c37, @@ -1831,6 +1909,13 @@ codepoint_classes = { 0x11d3c00011d3e, 0x11d3f00011d48, 0x11d5000011d5a, + 0x11d6000011d66, + 0x11d6700011d69, + 0x11d6a00011d8f, + 0x11d9000011d92, + 0x11d9300011d99, + 0x11da000011daa, + 0x11ee000011ef7, 0x120000001239a, 0x1248000012544, 0x130000001342f, @@ -1845,11 +1930,12 @@ codepoint_classes = { 0x16b5000016b5a, 0x16b6300016b78, 0x16b7d00016b90, + 0x16e6000016e80, 0x16f0000016f45, 0x16f5000016f7f, 0x16f8f00016fa0, 0x16fe000016fe2, - 0x17000000187ed, + 0x17000000187f2, 0x1880000018af3, 0x1b0000001b11f, 0x1b1700001b2fc, diff --git a/pipenv/vendor/idna/intranges.py b/pipenv/vendor/idna/intranges.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/idna/package_data.py b/pipenv/vendor/idna/package_data.py old mode 100755 new mode 100644 index 39c192ba..257e8989 --- a/pipenv/vendor/idna/package_data.py +++ b/pipenv/vendor/idna/package_data.py @@ -1,2 +1,2 @@ -__version__ = '2.7' +__version__ = '2.8' diff --git a/pipenv/vendor/idna/uts46data.py b/pipenv/vendor/idna/uts46data.py old mode 100755 new mode 100644 index 79731cb9..a68ed4c0 --- a/pipenv/vendor/idna/uts46data.py +++ b/pipenv/vendor/idna/uts46data.py @@ -4,7 +4,7 @@ """IDNA Mapping Table from UTS46.""" -__version__ = "10.0.0" +__version__ = "11.0.0" def _seg_0(): return [ (0x0, '3'), @@ -1029,11 +1029,8 @@ def _seg_9(): (0x556, 'M', u'ֆ'), (0x557, 'X'), (0x559, 'V'), - (0x560, 'X'), - (0x561, 'V'), (0x587, 'M', u'եւ'), - (0x588, 'X'), - (0x589, 'V'), + (0x588, 'V'), (0x58B, 'X'), (0x58D, 'V'), (0x590, 'X'), @@ -1041,15 +1038,15 @@ def _seg_9(): (0x5C8, 'X'), (0x5D0, 'V'), (0x5EB, 'X'), - (0x5F0, 'V'), + (0x5EF, 'V'), (0x5F5, 'X'), + (0x606, 'V'), + (0x61C, 'X'), + (0x61E, 'V'), ] def _seg_10(): return [ - (0x606, 'V'), - (0x61C, 'X'), - (0x61E, 'V'), (0x675, 'M', u'اٴ'), (0x676, 'M', u'وٴ'), (0x677, 'M', u'ۇٴ'), @@ -1064,7 +1061,7 @@ def _seg_10(): (0x7B2, 'X'), (0x7C0, 'V'), (0x7FB, 'X'), - (0x800, 'V'), + (0x7FD, 'V'), (0x82E, 'X'), (0x830, 'V'), (0x83F, 'X'), @@ -1078,7 +1075,7 @@ def _seg_10(): (0x8B5, 'X'), (0x8B6, 'V'), (0x8BE, 'X'), - (0x8D4, 'V'), + (0x8D3, 'V'), (0x8E2, 'X'), (0x8E3, 'V'), (0x958, 'M', u'क़'), @@ -1118,7 +1115,7 @@ def _seg_10(): (0x9E0, 'V'), (0x9E4, 'X'), (0x9E6, 'V'), - (0x9FE, 'X'), + (0x9FF, 'X'), (0xA01, 'V'), (0xA04, 'X'), (0xA05, 'V'), @@ -1147,19 +1144,19 @@ def _seg_10(): (0xA4E, 'X'), (0xA51, 'V'), (0xA52, 'X'), + (0xA59, 'M', u'ਖ਼'), + (0xA5A, 'M', u'ਗ਼'), + (0xA5B, 'M', u'ਜ਼'), ] def _seg_11(): return [ - (0xA59, 'M', u'ਖ਼'), - (0xA5A, 'M', u'ਗ਼'), - (0xA5B, 'M', u'ਜ਼'), (0xA5C, 'V'), (0xA5D, 'X'), (0xA5E, 'M', u'ਫ਼'), (0xA5F, 'X'), (0xA66, 'V'), - (0xA76, 'X'), + (0xA77, 'X'), (0xA81, 'V'), (0xA84, 'X'), (0xA85, 'V'), @@ -1250,16 +1247,14 @@ def _seg_11(): (0xBE6, 'V'), (0xBFB, 'X'), (0xC00, 'V'), - (0xC04, 'X'), - ] - -def _seg_12(): - return [ - (0xC05, 'V'), (0xC0D, 'X'), (0xC0E, 'V'), (0xC11, 'X'), (0xC12, 'V'), + ] + +def _seg_12(): + return [ (0xC29, 'X'), (0xC2A, 'V'), (0xC3A, 'X'), @@ -1278,8 +1273,6 @@ def _seg_12(): (0xC66, 'V'), (0xC70, 'X'), (0xC78, 'V'), - (0xC84, 'X'), - (0xC85, 'V'), (0xC8D, 'X'), (0xC8E, 'V'), (0xC91, 'X'), @@ -1355,10 +1348,6 @@ def _seg_12(): (0xE83, 'X'), (0xE84, 'V'), (0xE85, 'X'), - ] - -def _seg_13(): - return [ (0xE87, 'V'), (0xE89, 'X'), (0xE8A, 'V'), @@ -1366,6 +1355,10 @@ def _seg_13(): (0xE8D, 'V'), (0xE8E, 'X'), (0xE94, 'V'), + ] + +def _seg_13(): + return [ (0xE98, 'X'), (0xE99, 'V'), (0xEA0, 'X'), @@ -1459,10 +1452,6 @@ def _seg_13(): (0x124E, 'X'), (0x1250, 'V'), (0x1257, 'X'), - ] - -def _seg_14(): - return [ (0x1258, 'V'), (0x1259, 'X'), (0x125A, 'V'), @@ -1470,6 +1459,10 @@ def _seg_14(): (0x1260, 'V'), (0x1289, 'X'), (0x128A, 'V'), + ] + +def _seg_14(): + return [ (0x128E, 'X'), (0x1290, 'V'), (0x12B1, 'X'), @@ -1538,7 +1531,7 @@ def _seg_14(): (0x1810, 'V'), (0x181A, 'X'), (0x1820, 'V'), - (0x1878, 'X'), + (0x1879, 'X'), (0x1880, 'V'), (0x18AB, 'X'), (0x18B0, 'V'), @@ -1563,10 +1556,6 @@ def _seg_14(): (0x19DB, 'X'), (0x19DE, 'V'), (0x1A1C, 'X'), - ] - -def _seg_15(): - return [ (0x1A1E, 'V'), (0x1A5F, 'X'), (0x1A60, 'V'), @@ -1574,6 +1563,10 @@ def _seg_15(): (0x1A7F, 'V'), (0x1A8A, 'X'), (0x1A90, 'V'), + ] + +def _seg_15(): + return [ (0x1A9A, 'X'), (0x1AA0, 'V'), (0x1AAE, 'X'), @@ -1667,10 +1660,6 @@ def _seg_15(): (0x1D68, 'M', u'ρ'), (0x1D69, 'M', u'φ'), (0x1D6A, 'M', u'χ'), - ] - -def _seg_16(): - return [ (0x1D6B, 'V'), (0x1D78, 'M', u'н'), (0x1D79, 'V'), @@ -1678,6 +1667,10 @@ def _seg_16(): (0x1D9C, 'M', u'c'), (0x1D9D, 'M', u'ɕ'), (0x1D9E, 'M', u'ð'), + ] + +def _seg_16(): + return [ (0x1D9F, 'M', u'ɜ'), (0x1DA0, 'M', u'f'), (0x1DA1, 'M', u'ɟ'), @@ -1771,10 +1764,6 @@ def _seg_16(): (0x1E36, 'M', u'ḷ'), (0x1E37, 'V'), (0x1E38, 'M', u'ḹ'), - ] - -def _seg_17(): - return [ (0x1E39, 'V'), (0x1E3A, 'M', u'ḻ'), (0x1E3B, 'V'), @@ -1782,6 +1771,10 @@ def _seg_17(): (0x1E3D, 'V'), (0x1E3E, 'M', u'ḿ'), (0x1E3F, 'V'), + ] + +def _seg_17(): + return [ (0x1E40, 'M', u'ṁ'), (0x1E41, 'V'), (0x1E42, 'M', u'ṃ'), @@ -1875,10 +1868,6 @@ def _seg_17(): (0x1E9F, 'V'), (0x1EA0, 'M', u'ạ'), (0x1EA1, 'V'), - ] - -def _seg_18(): - return [ (0x1EA2, 'M', u'ả'), (0x1EA3, 'V'), (0x1EA4, 'M', u'ấ'), @@ -1886,6 +1875,10 @@ def _seg_18(): (0x1EA6, 'M', u'ầ'), (0x1EA7, 'V'), (0x1EA8, 'M', u'ẩ'), + ] + +def _seg_18(): + return [ (0x1EA9, 'V'), (0x1EAA, 'M', u'ẫ'), (0x1EAB, 'V'), @@ -1979,10 +1972,6 @@ def _seg_18(): (0x1F0B, 'M', u'ἃ'), (0x1F0C, 'M', u'ἄ'), (0x1F0D, 'M', u'ἅ'), - ] - -def _seg_19(): - return [ (0x1F0E, 'M', u'ἆ'), (0x1F0F, 'M', u'ἇ'), (0x1F10, 'V'), @@ -1990,6 +1979,10 @@ def _seg_19(): (0x1F18, 'M', u'ἐ'), (0x1F19, 'M', u'ἑ'), (0x1F1A, 'M', u'ἒ'), + ] + +def _seg_19(): + return [ (0x1F1B, 'M', u'ἓ'), (0x1F1C, 'M', u'ἔ'), (0x1F1D, 'M', u'ἕ'), @@ -2083,10 +2076,6 @@ def _seg_19(): (0x1F9A, 'M', u'ἢι'), (0x1F9B, 'M', u'ἣι'), (0x1F9C, 'M', u'ἤι'), - ] - -def _seg_20(): - return [ (0x1F9D, 'M', u'ἥι'), (0x1F9E, 'M', u'ἦι'), (0x1F9F, 'M', u'ἧι'), @@ -2094,6 +2083,10 @@ def _seg_20(): (0x1FA1, 'M', u'ὡι'), (0x1FA2, 'M', u'ὢι'), (0x1FA3, 'M', u'ὣι'), + ] + +def _seg_20(): + return [ (0x1FA4, 'M', u'ὤι'), (0x1FA5, 'M', u'ὥι'), (0x1FA6, 'M', u'ὦι'), @@ -2187,10 +2180,6 @@ def _seg_20(): (0x2024, 'X'), (0x2027, 'V'), (0x2028, 'X'), - ] - -def _seg_21(): - return [ (0x202F, '3', u' '), (0x2030, 'V'), (0x2033, 'M', u'′′'), @@ -2198,6 +2187,10 @@ def _seg_21(): (0x2035, 'V'), (0x2036, 'M', u'‵‵'), (0x2037, 'M', u'‵‵‵'), + ] + +def _seg_21(): + return [ (0x2038, 'V'), (0x203C, '3', u'!!'), (0x203D, 'V'), @@ -2291,10 +2284,6 @@ def _seg_21(): (0x2120, 'M', u'sm'), (0x2121, 'M', u'tel'), (0x2122, 'M', u'tm'), - ] - -def _seg_22(): - return [ (0x2123, 'V'), (0x2124, 'M', u'z'), (0x2125, 'V'), @@ -2302,6 +2291,10 @@ def _seg_22(): (0x2127, 'V'), (0x2128, 'M', u'z'), (0x2129, 'V'), + ] + +def _seg_22(): + return [ (0x212A, 'M', u'k'), (0x212B, 'M', u'å'), (0x212C, 'M', u'b'), @@ -2395,10 +2388,6 @@ def _seg_22(): (0x226E, '3'), (0x2270, 'V'), (0x2329, 'M', u'〈'), - ] - -def _seg_23(): - return [ (0x232A, 'M', u'〉'), (0x232B, 'V'), (0x2427, 'X'), @@ -2406,6 +2395,10 @@ def _seg_23(): (0x244B, 'X'), (0x2460, 'M', u'1'), (0x2461, 'M', u'2'), + ] + +def _seg_23(): + return [ (0x2462, 'M', u'3'), (0x2463, 'M', u'4'), (0x2464, 'M', u'5'), @@ -2499,10 +2492,6 @@ def _seg_23(): (0x24CF, 'M', u'z'), (0x24D0, 'M', u'a'), (0x24D1, 'M', u'b'), - ] - -def _seg_24(): - return [ (0x24D2, 'M', u'c'), (0x24D3, 'M', u'd'), (0x24D4, 'M', u'e'), @@ -2510,6 +2499,10 @@ def _seg_24(): (0x24D6, 'M', u'g'), (0x24D7, 'M', u'h'), (0x24D8, 'M', u'i'), + ] + +def _seg_24(): + return [ (0x24D9, 'M', u'j'), (0x24DA, 'M', u'k'), (0x24DB, 'M', u'l'), @@ -2541,13 +2534,9 @@ def _seg_24(): (0x2B76, 'V'), (0x2B96, 'X'), (0x2B98, 'V'), - (0x2BBA, 'X'), - (0x2BBD, 'V'), (0x2BC9, 'X'), (0x2BCA, 'V'), - (0x2BD3, 'X'), - (0x2BEC, 'V'), - (0x2BF0, 'X'), + (0x2BFF, 'X'), (0x2C00, 'M', u'ⰰ'), (0x2C01, 'M', u'ⰱ'), (0x2C02, 'M', u'ⰲ'), @@ -2603,10 +2592,6 @@ def _seg_24(): (0x2C62, 'M', u'ɫ'), (0x2C63, 'M', u'ᵽ'), (0x2C64, 'M', u'ɽ'), - ] - -def _seg_25(): - return [ (0x2C65, 'V'), (0x2C67, 'M', u'ⱨ'), (0x2C68, 'V'), @@ -2618,6 +2603,10 @@ def _seg_25(): (0x2C6E, 'M', u'ɱ'), (0x2C6F, 'M', u'ɐ'), (0x2C70, 'M', u'ɒ'), + ] + +def _seg_25(): + return [ (0x2C71, 'V'), (0x2C72, 'M', u'ⱳ'), (0x2C73, 'V'), @@ -2707,10 +2696,6 @@ def _seg_25(): (0x2CCD, 'V'), (0x2CCE, 'M', u'ⳏ'), (0x2CCF, 'V'), - ] - -def _seg_26(): - return [ (0x2CD0, 'M', u'ⳑ'), (0x2CD1, 'V'), (0x2CD2, 'M', u'ⳓ'), @@ -2722,6 +2707,10 @@ def _seg_26(): (0x2CD8, 'M', u'ⳙ'), (0x2CD9, 'V'), (0x2CDA, 'M', u'ⳛ'), + ] + +def _seg_26(): + return [ (0x2CDB, 'V'), (0x2CDC, 'M', u'ⳝ'), (0x2CDD, 'V'), @@ -2768,7 +2757,7 @@ def _seg_26(): (0x2DD8, 'V'), (0x2DDF, 'X'), (0x2DE0, 'V'), - (0x2E4A, 'X'), + (0x2E4F, 'X'), (0x2E80, 'V'), (0x2E9A, 'X'), (0x2E9B, 'V'), @@ -2811,10 +2800,6 @@ def _seg_26(): (0x2F20, 'M', u'士'), (0x2F21, 'M', u'夂'), (0x2F22, 'M', u'夊'), - ] - -def _seg_27(): - return [ (0x2F23, 'M', u'夕'), (0x2F24, 'M', u'大'), (0x2F25, 'M', u'女'), @@ -2826,6 +2811,10 @@ def _seg_27(): (0x2F2B, 'M', u'尸'), (0x2F2C, 'M', u'屮'), (0x2F2D, 'M', u'山'), + ] + +def _seg_27(): + return [ (0x2F2E, 'M', u'巛'), (0x2F2F, 'M', u'工'), (0x2F30, 'M', u'己'), @@ -2915,10 +2904,6 @@ def _seg_27(): (0x2F84, 'M', u'至'), (0x2F85, 'M', u'臼'), (0x2F86, 'M', u'舌'), - ] - -def _seg_28(): - return [ (0x2F87, 'M', u'舛'), (0x2F88, 'M', u'舟'), (0x2F89, 'M', u'艮'), @@ -2930,6 +2915,10 @@ def _seg_28(): (0x2F8F, 'M', u'行'), (0x2F90, 'M', u'衣'), (0x2F91, 'M', u'襾'), + ] + +def _seg_28(): + return [ (0x2F92, 'M', u'見'), (0x2F93, 'M', u'角'), (0x2F94, 'M', u'言'), @@ -3019,13 +3008,9 @@ def _seg_28(): (0x309F, 'M', u'より'), (0x30A0, 'V'), (0x30FF, 'M', u'コト'), - ] - -def _seg_29(): - return [ (0x3100, 'X'), (0x3105, 'V'), - (0x312F, 'X'), + (0x3130, 'X'), (0x3131, 'M', u'ᄀ'), (0x3132, 'M', u'ᄁ'), (0x3133, 'M', u'ᆪ'), @@ -3034,6 +3019,10 @@ def _seg_29(): (0x3136, 'M', u'ᆭ'), (0x3137, 'M', u'ᄃ'), (0x3138, 'M', u'ᄄ'), + ] + +def _seg_29(): + return [ (0x3139, 'M', u'ᄅ'), (0x313A, 'M', u'ᆰ'), (0x313B, 'M', u'ᆱ'), @@ -3123,10 +3112,6 @@ def _seg_29(): (0x318F, 'X'), (0x3190, 'V'), (0x3192, 'M', u'一'), - ] - -def _seg_30(): - return [ (0x3193, 'M', u'二'), (0x3194, 'M', u'三'), (0x3195, 'M', u'四'), @@ -3138,6 +3123,10 @@ def _seg_30(): (0x319B, 'M', u'丙'), (0x319C, 'M', u'丁'), (0x319D, 'M', u'天'), + ] + +def _seg_30(): + return [ (0x319E, 'M', u'地'), (0x319F, 'M', u'人'), (0x31A0, 'V'), @@ -3227,10 +3216,6 @@ def _seg_30(): (0x3256, 'M', u'26'), (0x3257, 'M', u'27'), (0x3258, 'M', u'28'), - ] - -def _seg_31(): - return [ (0x3259, 'M', u'29'), (0x325A, 'M', u'30'), (0x325B, 'M', u'31'), @@ -3242,6 +3227,10 @@ def _seg_31(): (0x3261, 'M', u'ᄂ'), (0x3262, 'M', u'ᄃ'), (0x3263, 'M', u'ᄅ'), + ] + +def _seg_31(): + return [ (0x3264, 'M', u'ᄆ'), (0x3265, 'M', u'ᄇ'), (0x3266, 'M', u'ᄉ'), @@ -3331,10 +3320,6 @@ def _seg_31(): (0x32BA, 'M', u'45'), (0x32BB, 'M', u'46'), (0x32BC, 'M', u'47'), - ] - -def _seg_32(): - return [ (0x32BD, 'M', u'48'), (0x32BE, 'M', u'49'), (0x32BF, 'M', u'50'), @@ -3346,6 +3331,10 @@ def _seg_32(): (0x32C5, 'M', u'6月'), (0x32C6, 'M', u'7月'), (0x32C7, 'M', u'8月'), + ] + +def _seg_32(): + return [ (0x32C8, 'M', u'9月'), (0x32C9, 'M', u'10月'), (0x32CA, 'M', u'11月'), @@ -3435,10 +3424,6 @@ def _seg_32(): (0x331E, 'M', u'コーポ'), (0x331F, 'M', u'サイクル'), (0x3320, 'M', u'サンチーム'), - ] - -def _seg_33(): - return [ (0x3321, 'M', u'シリング'), (0x3322, 'M', u'センチ'), (0x3323, 'M', u'セント'), @@ -3450,6 +3435,10 @@ def _seg_33(): (0x3329, 'M', u'ノット'), (0x332A, 'M', u'ハイツ'), (0x332B, 'M', u'パーセント'), + ] + +def _seg_33(): + return [ (0x332C, 'M', u'パーツ'), (0x332D, 'M', u'バーレル'), (0x332E, 'M', u'ピアストル'), @@ -3539,10 +3528,6 @@ def _seg_33(): (0x3382, 'M', u'μa'), (0x3383, 'M', u'ma'), (0x3384, 'M', u'ka'), - ] - -def _seg_34(): - return [ (0x3385, 'M', u'kb'), (0x3386, 'M', u'mb'), (0x3387, 'M', u'gb'), @@ -3554,6 +3539,10 @@ def _seg_34(): (0x338D, 'M', u'μg'), (0x338E, 'M', u'mg'), (0x338F, 'M', u'kg'), + ] + +def _seg_34(): + return [ (0x3390, 'M', u'hz'), (0x3391, 'M', u'khz'), (0x3392, 'M', u'mhz'), @@ -3643,10 +3632,6 @@ def _seg_34(): (0x33E6, 'M', u'7日'), (0x33E7, 'M', u'8日'), (0x33E8, 'M', u'9日'), - ] - -def _seg_35(): - return [ (0x33E9, 'M', u'10日'), (0x33EA, 'M', u'11日'), (0x33EB, 'M', u'12日'), @@ -3658,6 +3643,10 @@ def _seg_35(): (0x33F1, 'M', u'18日'), (0x33F2, 'M', u'19日'), (0x33F3, 'M', u'20日'), + ] + +def _seg_35(): + return [ (0x33F4, 'M', u'21日'), (0x33F5, 'M', u'22日'), (0x33F6, 'M', u'23日'), @@ -3673,7 +3662,7 @@ def _seg_35(): (0x3400, 'V'), (0x4DB6, 'X'), (0x4DC0, 'V'), - (0x9FEB, 'X'), + (0x9FF0, 'X'), (0xA000, 'V'), (0xA48D, 'X'), (0xA490, 'V'), @@ -3747,10 +3736,6 @@ def _seg_35(): (0xA692, 'M', u'ꚓ'), (0xA693, 'V'), (0xA694, 'M', u'ꚕ'), - ] - -def _seg_36(): - return [ (0xA695, 'V'), (0xA696, 'M', u'ꚗ'), (0xA697, 'V'), @@ -3762,6 +3747,10 @@ def _seg_36(): (0xA69D, 'M', u'ь'), (0xA69E, 'V'), (0xA6F8, 'X'), + ] + +def _seg_36(): + return [ (0xA700, 'V'), (0xA722, 'M', u'ꜣ'), (0xA723, 'V'), @@ -3851,10 +3840,6 @@ def _seg_36(): (0xA780, 'M', u'ꞁ'), (0xA781, 'V'), (0xA782, 'M', u'ꞃ'), - ] - -def _seg_37(): - return [ (0xA783, 'V'), (0xA784, 'M', u'ꞅ'), (0xA785, 'V'), @@ -3866,6 +3851,10 @@ def _seg_37(): (0xA78E, 'V'), (0xA790, 'M', u'ꞑ'), (0xA791, 'V'), + ] + +def _seg_37(): + return [ (0xA792, 'M', u'ꞓ'), (0xA793, 'V'), (0xA796, 'M', u'ꞗ'), @@ -3893,7 +3882,7 @@ def _seg_37(): (0xA7AC, 'M', u'ɡ'), (0xA7AD, 'M', u'ɬ'), (0xA7AE, 'M', u'ɪ'), - (0xA7AF, 'X'), + (0xA7AF, 'V'), (0xA7B0, 'M', u'ʞ'), (0xA7B1, 'M', u'ʇ'), (0xA7B2, 'M', u'ʝ'), @@ -3903,6 +3892,8 @@ def _seg_37(): (0xA7B6, 'M', u'ꞷ'), (0xA7B7, 'V'), (0xA7B8, 'X'), + (0xA7B9, 'V'), + (0xA7BA, 'X'), (0xA7F7, 'V'), (0xA7F8, 'M', u'ħ'), (0xA7F9, 'M', u'œ'), @@ -3917,8 +3908,6 @@ def _seg_37(): (0xA8CE, 'V'), (0xA8DA, 'X'), (0xA8E0, 'V'), - (0xA8FE, 'X'), - (0xA900, 'V'), (0xA954, 'X'), (0xA95F, 'V'), (0xA97D, 'X'), @@ -3955,10 +3944,6 @@ def _seg_37(): (0xAB5F, 'M', u'ꭒ'), (0xAB60, 'V'), (0xAB66, 'X'), - ] - -def _seg_38(): - return [ (0xAB70, 'M', u'Ꭰ'), (0xAB71, 'M', u'Ꭱ'), (0xAB72, 'M', u'Ꭲ'), @@ -3970,6 +3955,10 @@ def _seg_38(): (0xAB78, 'M', u'Ꭸ'), (0xAB79, 'M', u'Ꭹ'), (0xAB7A, 'M', u'Ꭺ'), + ] + +def _seg_38(): + return [ (0xAB7B, 'M', u'Ꭻ'), (0xAB7C, 'M', u'Ꭼ'), (0xAB7D, 'M', u'Ꭽ'), @@ -4059,10 +4048,6 @@ def _seg_38(): (0xF907, 'M', u'龜'), (0xF909, 'M', u'契'), (0xF90A, 'M', u'金'), - ] - -def _seg_39(): - return [ (0xF90B, 'M', u'喇'), (0xF90C, 'M', u'奈'), (0xF90D, 'M', u'懶'), @@ -4074,6 +4059,10 @@ def _seg_39(): (0xF913, 'M', u'邏'), (0xF914, 'M', u'樂'), (0xF915, 'M', u'洛'), + ] + +def _seg_39(): + return [ (0xF916, 'M', u'烙'), (0xF917, 'M', u'珞'), (0xF918, 'M', u'落'), @@ -4163,10 +4152,6 @@ def _seg_39(): (0xF96C, 'M', u'塞'), (0xF96D, 'M', u'省'), (0xF96E, 'M', u'葉'), - ] - -def _seg_40(): - return [ (0xF96F, 'M', u'說'), (0xF970, 'M', u'殺'), (0xF971, 'M', u'辰'), @@ -4178,6 +4163,10 @@ def _seg_40(): (0xF977, 'M', u'亮'), (0xF978, 'M', u'兩'), (0xF979, 'M', u'凉'), + ] + +def _seg_40(): + return [ (0xF97A, 'M', u'梁'), (0xF97B, 'M', u'糧'), (0xF97C, 'M', u'良'), @@ -4267,10 +4256,6 @@ def _seg_40(): (0xF9D0, 'M', u'類'), (0xF9D1, 'M', u'六'), (0xF9D2, 'M', u'戮'), - ] - -def _seg_41(): - return [ (0xF9D3, 'M', u'陸'), (0xF9D4, 'M', u'倫'), (0xF9D5, 'M', u'崙'), @@ -4282,6 +4267,10 @@ def _seg_41(): (0xF9DB, 'M', u'率'), (0xF9DC, 'M', u'隆'), (0xF9DD, 'M', u'利'), + ] + +def _seg_41(): + return [ (0xF9DE, 'M', u'吏'), (0xF9DF, 'M', u'履'), (0xF9E0, 'M', u'易'), @@ -4371,10 +4360,6 @@ def _seg_41(): (0xFA39, 'M', u'塀'), (0xFA3A, 'M', u'墨'), (0xFA3B, 'M', u'層'), - ] - -def _seg_42(): - return [ (0xFA3C, 'M', u'屮'), (0xFA3D, 'M', u'悔'), (0xFA3E, 'M', u'慨'), @@ -4386,6 +4371,10 @@ def _seg_42(): (0xFA44, 'M', u'梅'), (0xFA45, 'M', u'海'), (0xFA46, 'M', u'渚'), + ] + +def _seg_42(): + return [ (0xFA47, 'M', u'漢'), (0xFA48, 'M', u'煮'), (0xFA49, 'M', u'爫'), @@ -4475,10 +4464,6 @@ def _seg_42(): (0xFA9F, 'M', u'犯'), (0xFAA0, 'M', u'猪'), (0xFAA1, 'M', u'瑱'), - ] - -def _seg_43(): - return [ (0xFAA2, 'M', u'甆'), (0xFAA3, 'M', u'画'), (0xFAA4, 'M', u'瘝'), @@ -4490,6 +4475,10 @@ def _seg_43(): (0xFAAA, 'M', u'着'), (0xFAAB, 'M', u'磌'), (0xFAAC, 'M', u'窱'), + ] + +def _seg_43(): + return [ (0xFAAD, 'M', u'節'), (0xFAAE, 'M', u'类'), (0xFAAF, 'M', u'絛'), @@ -4579,10 +4568,6 @@ def _seg_43(): (0xFB38, 'M', u'טּ'), (0xFB39, 'M', u'יּ'), (0xFB3A, 'M', u'ךּ'), - ] - -def _seg_44(): - return [ (0xFB3B, 'M', u'כּ'), (0xFB3C, 'M', u'לּ'), (0xFB3D, 'X'), @@ -4594,6 +4579,10 @@ def _seg_44(): (0xFB43, 'M', u'ףּ'), (0xFB44, 'M', u'פּ'), (0xFB45, 'X'), + ] + +def _seg_44(): + return [ (0xFB46, 'M', u'צּ'), (0xFB47, 'M', u'קּ'), (0xFB48, 'M', u'רּ'), @@ -4683,10 +4672,6 @@ def _seg_44(): (0xFC19, 'M', u'خج'), (0xFC1A, 'M', u'خح'), (0xFC1B, 'M', u'خم'), - ] - -def _seg_45(): - return [ (0xFC1C, 'M', u'سج'), (0xFC1D, 'M', u'سح'), (0xFC1E, 'M', u'سخ'), @@ -4698,6 +4683,10 @@ def _seg_45(): (0xFC24, 'M', u'ضخ'), (0xFC25, 'M', u'ضم'), (0xFC26, 'M', u'طح'), + ] + +def _seg_45(): + return [ (0xFC27, 'M', u'طم'), (0xFC28, 'M', u'ظم'), (0xFC29, 'M', u'عج'), @@ -4787,10 +4776,6 @@ def _seg_45(): (0xFC7D, 'M', u'في'), (0xFC7E, 'M', u'قى'), (0xFC7F, 'M', u'قي'), - ] - -def _seg_46(): - return [ (0xFC80, 'M', u'كا'), (0xFC81, 'M', u'كل'), (0xFC82, 'M', u'كم'), @@ -4802,6 +4787,10 @@ def _seg_46(): (0xFC88, 'M', u'ما'), (0xFC89, 'M', u'مم'), (0xFC8A, 'M', u'نر'), + ] + +def _seg_46(): + return [ (0xFC8B, 'M', u'نز'), (0xFC8C, 'M', u'نم'), (0xFC8D, 'M', u'نن'), @@ -4891,10 +4880,6 @@ def _seg_46(): (0xFCE1, 'M', u'بم'), (0xFCE2, 'M', u'به'), (0xFCE3, 'M', u'تم'), - ] - -def _seg_47(): - return [ (0xFCE4, 'M', u'ته'), (0xFCE5, 'M', u'ثم'), (0xFCE6, 'M', u'ثه'), @@ -4906,6 +4891,10 @@ def _seg_47(): (0xFCEC, 'M', u'كم'), (0xFCED, 'M', u'لم'), (0xFCEE, 'M', u'نم'), + ] + +def _seg_47(): + return [ (0xFCEF, 'M', u'نه'), (0xFCF0, 'M', u'يم'), (0xFCF1, 'M', u'يه'), @@ -4995,10 +4984,6 @@ def _seg_47(): (0xFD57, 'M', u'تمخ'), (0xFD58, 'M', u'جمح'), (0xFD5A, 'M', u'حمي'), - ] - -def _seg_48(): - return [ (0xFD5B, 'M', u'حمى'), (0xFD5C, 'M', u'سحج'), (0xFD5D, 'M', u'سجح'), @@ -5010,6 +4995,10 @@ def _seg_48(): (0xFD66, 'M', u'صمم'), (0xFD67, 'M', u'شحم'), (0xFD69, 'M', u'شجي'), + ] + +def _seg_48(): + return [ (0xFD6A, 'M', u'شمخ'), (0xFD6C, 'M', u'شمم'), (0xFD6E, 'M', u'ضحى'), @@ -5099,10 +5088,6 @@ def _seg_48(): (0xFDF3, 'M', u'اكبر'), (0xFDF4, 'M', u'محمد'), (0xFDF5, 'M', u'صلعم'), - ] - -def _seg_49(): - return [ (0xFDF6, 'M', u'رسول'), (0xFDF7, 'M', u'عليه'), (0xFDF8, 'M', u'وسلم'), @@ -5114,6 +5099,10 @@ def _seg_49(): (0xFDFE, 'X'), (0xFE00, 'I'), (0xFE10, '3', u','), + ] + +def _seg_49(): + return [ (0xFE11, 'M', u'、'), (0xFE12, 'X'), (0xFE13, '3', u':'), @@ -5203,10 +5192,6 @@ def _seg_49(): (0xFE8F, 'M', u'ب'), (0xFE93, 'M', u'ة'), (0xFE95, 'M', u'ت'), - ] - -def _seg_50(): - return [ (0xFE99, 'M', u'ث'), (0xFE9D, 'M', u'ج'), (0xFEA1, 'M', u'ح'), @@ -5218,6 +5203,10 @@ def _seg_50(): (0xFEB1, 'M', u'س'), (0xFEB5, 'M', u'ش'), (0xFEB9, 'M', u'ص'), + ] + +def _seg_50(): + return [ (0xFEBD, 'M', u'ض'), (0xFEC1, 'M', u'ط'), (0xFEC5, 'M', u'ظ'), @@ -5307,10 +5296,6 @@ def _seg_50(): (0xFF41, 'M', u'a'), (0xFF42, 'M', u'b'), (0xFF43, 'M', u'c'), - ] - -def _seg_51(): - return [ (0xFF44, 'M', u'd'), (0xFF45, 'M', u'e'), (0xFF46, 'M', u'f'), @@ -5322,6 +5307,10 @@ def _seg_51(): (0xFF4C, 'M', u'l'), (0xFF4D, 'M', u'm'), (0xFF4E, 'M', u'n'), + ] + +def _seg_51(): + return [ (0xFF4F, 'M', u'o'), (0xFF50, 'M', u'p'), (0xFF51, 'M', u'q'), @@ -5411,10 +5400,6 @@ def _seg_51(): (0xFFA5, 'M', u'ᆬ'), (0xFFA6, 'M', u'ᆭ'), (0xFFA7, 'M', u'ᄃ'), - ] - -def _seg_52(): - return [ (0xFFA8, 'M', u'ᄄ'), (0xFFA9, 'M', u'ᄅ'), (0xFFAA, 'M', u'ᆰ'), @@ -5426,6 +5411,10 @@ def _seg_52(): (0xFFB0, 'M', u'ᄚ'), (0xFFB1, 'M', u'ᄆ'), (0xFFB2, 'M', u'ᄇ'), + ] + +def _seg_52(): + return [ (0xFFB3, 'M', u'ᄈ'), (0xFFB4, 'M', u'ᄡ'), (0xFFB5, 'M', u'ᄉ'), @@ -5515,10 +5504,6 @@ def _seg_52(): (0x10300, 'V'), (0x10324, 'X'), (0x1032D, 'V'), - ] - -def _seg_53(): - return [ (0x1034B, 'X'), (0x10350, 'V'), (0x1037B, 'X'), @@ -5530,6 +5515,10 @@ def _seg_53(): (0x103D6, 'X'), (0x10400, 'M', u'𐐨'), (0x10401, 'M', u'𐐩'), + ] + +def _seg_53(): + return [ (0x10402, 'M', u'𐐪'), (0x10403, 'M', u'𐐫'), (0x10404, 'M', u'𐐬'), @@ -5619,10 +5608,6 @@ def _seg_53(): (0x10570, 'X'), (0x10600, 'V'), (0x10737, 'X'), - ] - -def _seg_54(): - return [ (0x10740, 'V'), (0x10756, 'X'), (0x10760, 'V'), @@ -5634,6 +5619,10 @@ def _seg_54(): (0x1080A, 'V'), (0x10836, 'X'), (0x10837, 'V'), + ] + +def _seg_54(): + return [ (0x10839, 'X'), (0x1083C, 'V'), (0x1083D, 'X'), @@ -5666,11 +5655,11 @@ def _seg_54(): (0x10A15, 'V'), (0x10A18, 'X'), (0x10A19, 'V'), - (0x10A34, 'X'), + (0x10A36, 'X'), (0x10A38, 'V'), (0x10A3B, 'X'), (0x10A3F, 'V'), - (0x10A48, 'X'), + (0x10A49, 'X'), (0x10A50, 'V'), (0x10A59, 'X'), (0x10A60, 'V'), @@ -5723,10 +5712,6 @@ def _seg_54(): (0x10C9B, 'M', u'𐳛'), (0x10C9C, 'M', u'𐳜'), (0x10C9D, 'M', u'𐳝'), - ] - -def _seg_55(): - return [ (0x10C9E, 'M', u'𐳞'), (0x10C9F, 'M', u'𐳟'), (0x10CA0, 'M', u'𐳠'), @@ -5738,6 +5723,10 @@ def _seg_55(): (0x10CA6, 'M', u'𐳦'), (0x10CA7, 'M', u'𐳧'), (0x10CA8, 'M', u'𐳨'), + ] + +def _seg_55(): + return [ (0x10CA9, 'M', u'𐳩'), (0x10CAA, 'M', u'𐳪'), (0x10CAB, 'M', u'𐳫'), @@ -5752,9 +5741,15 @@ def _seg_55(): (0x10CC0, 'V'), (0x10CF3, 'X'), (0x10CFA, 'V'), - (0x10D00, 'X'), + (0x10D28, 'X'), + (0x10D30, 'V'), + (0x10D3A, 'X'), (0x10E60, 'V'), (0x10E7F, 'X'), + (0x10F00, 'V'), + (0x10F28, 'X'), + (0x10F30, 'V'), + (0x10F5A, 'X'), (0x11000, 'V'), (0x1104E, 'X'), (0x11052, 'V'), @@ -5770,7 +5765,7 @@ def _seg_55(): (0x11100, 'V'), (0x11135, 'X'), (0x11136, 'V'), - (0x11144, 'X'), + (0x11147, 'X'), (0x11150, 'V'), (0x11177, 'X'), (0x11180, 'V'), @@ -5811,7 +5806,7 @@ def _seg_55(): (0x11334, 'X'), (0x11335, 'V'), (0x1133A, 'X'), - (0x1133C, 'V'), + (0x1133B, 'V'), (0x11345, 'X'), (0x11347, 'V'), (0x11349, 'X'), @@ -5827,16 +5822,16 @@ def _seg_55(): (0x1136D, 'X'), (0x11370, 'V'), (0x11375, 'X'), - ] - -def _seg_56(): - return [ (0x11400, 'V'), (0x1145A, 'X'), (0x1145B, 'V'), (0x1145C, 'X'), (0x1145D, 'V'), - (0x1145E, 'X'), + ] + +def _seg_56(): + return [ + (0x1145F, 'X'), (0x11480, 'V'), (0x114C8, 'X'), (0x114D0, 'V'), @@ -5856,11 +5851,13 @@ def _seg_56(): (0x116C0, 'V'), (0x116CA, 'X'), (0x11700, 'V'), - (0x1171A, 'X'), + (0x1171B, 'X'), (0x1171D, 'V'), (0x1172C, 'X'), (0x11730, 'V'), (0x11740, 'X'), + (0x11800, 'V'), + (0x1183C, 'X'), (0x118A0, 'M', u'𑣀'), (0x118A1, 'M', u'𑣁'), (0x118A2, 'M', u'𑣂'), @@ -5902,8 +5899,6 @@ def _seg_56(): (0x11A50, 'V'), (0x11A84, 'X'), (0x11A86, 'V'), - (0x11A9D, 'X'), - (0x11A9E, 'V'), (0x11AA3, 'X'), (0x11AC0, 'V'), (0x11AF9, 'X'), @@ -5931,14 +5926,28 @@ def _seg_56(): (0x11D3B, 'X'), (0x11D3C, 'V'), (0x11D3E, 'X'), - ] - -def _seg_57(): - return [ (0x11D3F, 'V'), (0x11D48, 'X'), (0x11D50, 'V'), (0x11D5A, 'X'), + (0x11D60, 'V'), + ] + +def _seg_57(): + return [ + (0x11D66, 'X'), + (0x11D67, 'V'), + (0x11D69, 'X'), + (0x11D6A, 'V'), + (0x11D8F, 'X'), + (0x11D90, 'V'), + (0x11D92, 'X'), + (0x11D93, 'V'), + (0x11D99, 'X'), + (0x11DA0, 'V'), + (0x11DAA, 'X'), + (0x11EE0, 'V'), + (0x11EF9, 'X'), (0x12000, 'V'), (0x1239A, 'X'), (0x12400, 'V'), @@ -5973,6 +5982,8 @@ def _seg_57(): (0x16B78, 'X'), (0x16B7D, 'V'), (0x16B90, 'X'), + (0x16E60, 'V'), + (0x16E9B, 'X'), (0x16F00, 'V'), (0x16F45, 'X'), (0x16F50, 'V'), @@ -5982,7 +5993,7 @@ def _seg_57(): (0x16FE0, 'V'), (0x16FE2, 'X'), (0x17000, 'V'), - (0x187ED, 'X'), + (0x187F2, 'X'), (0x18800, 'V'), (0x18AF3, 'X'), (0x1B000, 'V'), @@ -6024,21 +6035,23 @@ def _seg_57(): (0x1D1C1, 'V'), (0x1D1E9, 'X'), (0x1D200, 'V'), + ] + +def _seg_58(): + return [ (0x1D246, 'X'), + (0x1D2E0, 'V'), + (0x1D2F4, 'X'), (0x1D300, 'V'), (0x1D357, 'X'), (0x1D360, 'V'), - (0x1D372, 'X'), + (0x1D379, 'X'), (0x1D400, 'M', u'a'), (0x1D401, 'M', u'b'), (0x1D402, 'M', u'c'), (0x1D403, 'M', u'd'), (0x1D404, 'M', u'e'), (0x1D405, 'M', u'f'), - ] - -def _seg_58(): - return [ (0x1D406, 'M', u'g'), (0x1D407, 'M', u'h'), (0x1D408, 'M', u'i'), @@ -6126,6 +6139,10 @@ def _seg_58(): (0x1D45A, 'M', u'm'), (0x1D45B, 'M', u'n'), (0x1D45C, 'M', u'o'), + ] + +def _seg_59(): + return [ (0x1D45D, 'M', u'p'), (0x1D45E, 'M', u'q'), (0x1D45F, 'M', u'r'), @@ -6139,10 +6156,6 @@ def _seg_58(): (0x1D467, 'M', u'z'), (0x1D468, 'M', u'a'), (0x1D469, 'M', u'b'), - ] - -def _seg_59(): - return [ (0x1D46A, 'M', u'c'), (0x1D46B, 'M', u'd'), (0x1D46C, 'M', u'e'), @@ -6230,6 +6243,10 @@ def _seg_59(): (0x1D4C1, 'M', u'l'), (0x1D4C2, 'M', u'm'), (0x1D4C3, 'M', u'n'), + ] + +def _seg_60(): + return [ (0x1D4C4, 'X'), (0x1D4C5, 'M', u'p'), (0x1D4C6, 'M', u'q'), @@ -6243,10 +6260,6 @@ def _seg_59(): (0x1D4CE, 'M', u'y'), (0x1D4CF, 'M', u'z'), (0x1D4D0, 'M', u'a'), - ] - -def _seg_60(): - return [ (0x1D4D1, 'M', u'b'), (0x1D4D2, 'M', u'c'), (0x1D4D3, 'M', u'd'), @@ -6334,6 +6347,10 @@ def _seg_60(): (0x1D526, 'M', u'i'), (0x1D527, 'M', u'j'), (0x1D528, 'M', u'k'), + ] + +def _seg_61(): + return [ (0x1D529, 'M', u'l'), (0x1D52A, 'M', u'm'), (0x1D52B, 'M', u'n'), @@ -6347,10 +6364,6 @@ def _seg_60(): (0x1D533, 'M', u'v'), (0x1D534, 'M', u'w'), (0x1D535, 'M', u'x'), - ] - -def _seg_61(): - return [ (0x1D536, 'M', u'y'), (0x1D537, 'M', u'z'), (0x1D538, 'M', u'a'), @@ -6438,6 +6451,10 @@ def _seg_61(): (0x1D58C, 'M', u'g'), (0x1D58D, 'M', u'h'), (0x1D58E, 'M', u'i'), + ] + +def _seg_62(): + return [ (0x1D58F, 'M', u'j'), (0x1D590, 'M', u'k'), (0x1D591, 'M', u'l'), @@ -6451,10 +6468,6 @@ def _seg_61(): (0x1D599, 'M', u't'), (0x1D59A, 'M', u'u'), (0x1D59B, 'M', u'v'), - ] - -def _seg_62(): - return [ (0x1D59C, 'M', u'w'), (0x1D59D, 'M', u'x'), (0x1D59E, 'M', u'y'), @@ -6542,6 +6555,10 @@ def _seg_62(): (0x1D5F0, 'M', u'c'), (0x1D5F1, 'M', u'd'), (0x1D5F2, 'M', u'e'), + ] + +def _seg_63(): + return [ (0x1D5F3, 'M', u'f'), (0x1D5F4, 'M', u'g'), (0x1D5F5, 'M', u'h'), @@ -6555,10 +6572,6 @@ def _seg_62(): (0x1D5FD, 'M', u'p'), (0x1D5FE, 'M', u'q'), (0x1D5FF, 'M', u'r'), - ] - -def _seg_63(): - return [ (0x1D600, 'M', u's'), (0x1D601, 'M', u't'), (0x1D602, 'M', u'u'), @@ -6646,6 +6659,10 @@ def _seg_63(): (0x1D654, 'M', u'y'), (0x1D655, 'M', u'z'), (0x1D656, 'M', u'a'), + ] + +def _seg_64(): + return [ (0x1D657, 'M', u'b'), (0x1D658, 'M', u'c'), (0x1D659, 'M', u'd'), @@ -6659,10 +6676,6 @@ def _seg_63(): (0x1D661, 'M', u'l'), (0x1D662, 'M', u'm'), (0x1D663, 'M', u'n'), - ] - -def _seg_64(): - return [ (0x1D664, 'M', u'o'), (0x1D665, 'M', u'p'), (0x1D666, 'M', u'q'), @@ -6750,6 +6763,10 @@ def _seg_64(): (0x1D6B9, 'M', u'θ'), (0x1D6BA, 'M', u'σ'), (0x1D6BB, 'M', u'τ'), + ] + +def _seg_65(): + return [ (0x1D6BC, 'M', u'υ'), (0x1D6BD, 'M', u'φ'), (0x1D6BE, 'M', u'χ'), @@ -6763,10 +6780,6 @@ def _seg_64(): (0x1D6C6, 'M', u'ε'), (0x1D6C7, 'M', u'ζ'), (0x1D6C8, 'M', u'η'), - ] - -def _seg_65(): - return [ (0x1D6C9, 'M', u'θ'), (0x1D6CA, 'M', u'ι'), (0x1D6CB, 'M', u'κ'), @@ -6854,6 +6867,10 @@ def _seg_65(): (0x1D71F, 'M', u'δ'), (0x1D720, 'M', u'ε'), (0x1D721, 'M', u'ζ'), + ] + +def _seg_66(): + return [ (0x1D722, 'M', u'η'), (0x1D723, 'M', u'θ'), (0x1D724, 'M', u'ι'), @@ -6867,10 +6884,6 @@ def _seg_65(): (0x1D72C, 'M', u'ρ'), (0x1D72D, 'M', u'θ'), (0x1D72E, 'M', u'σ'), - ] - -def _seg_66(): - return [ (0x1D72F, 'M', u'τ'), (0x1D730, 'M', u'υ'), (0x1D731, 'M', u'φ'), @@ -6958,6 +6971,10 @@ def _seg_66(): (0x1D785, 'M', u'φ'), (0x1D786, 'M', u'χ'), (0x1D787, 'M', u'ψ'), + ] + +def _seg_67(): + return [ (0x1D788, 'M', u'ω'), (0x1D789, 'M', u'∂'), (0x1D78A, 'M', u'ε'), @@ -6971,10 +6988,6 @@ def _seg_66(): (0x1D792, 'M', u'γ'), (0x1D793, 'M', u'δ'), (0x1D794, 'M', u'ε'), - ] - -def _seg_67(): - return [ (0x1D795, 'M', u'ζ'), (0x1D796, 'M', u'η'), (0x1D797, 'M', u'θ'), @@ -7062,6 +7075,10 @@ def _seg_67(): (0x1D7EC, 'M', u'0'), (0x1D7ED, 'M', u'1'), (0x1D7EE, 'M', u'2'), + ] + +def _seg_68(): + return [ (0x1D7EF, 'M', u'3'), (0x1D7F0, 'M', u'4'), (0x1D7F1, 'M', u'5'), @@ -7075,10 +7092,6 @@ def _seg_67(): (0x1D7F9, 'M', u'3'), (0x1D7FA, 'M', u'4'), (0x1D7FB, 'M', u'5'), - ] - -def _seg_68(): - return [ (0x1D7FC, 'M', u'6'), (0x1D7FD, 'M', u'7'), (0x1D7FE, 'M', u'8'), @@ -7143,6 +7156,8 @@ def _seg_68(): (0x1E95A, 'X'), (0x1E95E, 'V'), (0x1E960, 'X'), + (0x1EC71, 'V'), + (0x1ECB5, 'X'), (0x1EE00, 'M', u'ا'), (0x1EE01, 'M', u'ب'), (0x1EE02, 'M', u'ج'), @@ -7164,6 +7179,10 @@ def _seg_68(): (0x1EE12, 'M', u'ق'), (0x1EE13, 'M', u'ر'), (0x1EE14, 'M', u'ش'), + ] + +def _seg_69(): + return [ (0x1EE15, 'M', u'ت'), (0x1EE16, 'M', u'ث'), (0x1EE17, 'M', u'خ'), @@ -7179,10 +7198,6 @@ def _seg_68(): (0x1EE21, 'M', u'ب'), (0x1EE22, 'M', u'ج'), (0x1EE23, 'X'), - ] - -def _seg_69(): - return [ (0x1EE24, 'M', u'ه'), (0x1EE25, 'X'), (0x1EE27, 'M', u'ح'), @@ -7268,6 +7283,10 @@ def _seg_69(): (0x1EE81, 'M', u'ب'), (0x1EE82, 'M', u'ج'), (0x1EE83, 'M', u'د'), + ] + +def _seg_70(): + return [ (0x1EE84, 'M', u'ه'), (0x1EE85, 'M', u'و'), (0x1EE86, 'M', u'ز'), @@ -7283,10 +7302,6 @@ def _seg_69(): (0x1EE90, 'M', u'ف'), (0x1EE91, 'M', u'ص'), (0x1EE92, 'M', u'ق'), - ] - -def _seg_70(): - return [ (0x1EE93, 'M', u'ر'), (0x1EE94, 'M', u'ش'), (0x1EE95, 'M', u'ت'), @@ -7372,6 +7387,10 @@ def _seg_70(): (0x1F122, '3', u'(s)'), (0x1F123, '3', u'(t)'), (0x1F124, '3', u'(u)'), + ] + +def _seg_71(): + return [ (0x1F125, '3', u'(v)'), (0x1F126, '3', u'(w)'), (0x1F127, '3', u'(x)'), @@ -7382,15 +7401,11 @@ def _seg_70(): (0x1F12C, 'M', u'r'), (0x1F12D, 'M', u'cd'), (0x1F12E, 'M', u'wz'), - (0x1F12F, 'X'), + (0x1F12F, 'V'), (0x1F130, 'M', u'a'), (0x1F131, 'M', u'b'), (0x1F132, 'M', u'c'), (0x1F133, 'M', u'd'), - ] - -def _seg_71(): - return [ (0x1F134, 'M', u'e'), (0x1F135, 'M', u'f'), (0x1F136, 'M', u'g'), @@ -7476,6 +7491,10 @@ def _seg_71(): (0x1F239, 'M', u'割'), (0x1F23A, 'M', u'営'), (0x1F23B, 'M', u'配'), + ] + +def _seg_72(): + return [ (0x1F23C, 'X'), (0x1F240, 'M', u'〔本〕'), (0x1F241, 'M', u'〔三〕'), @@ -7491,21 +7510,17 @@ def _seg_71(): (0x1F251, 'M', u'可'), (0x1F252, 'X'), (0x1F260, 'V'), - ] - -def _seg_72(): - return [ (0x1F266, 'X'), (0x1F300, 'V'), (0x1F6D5, 'X'), (0x1F6E0, 'V'), (0x1F6ED, 'X'), (0x1F6F0, 'V'), - (0x1F6F9, 'X'), + (0x1F6FA, 'X'), (0x1F700, 'V'), (0x1F774, 'X'), (0x1F780, 'V'), - (0x1F7D5, 'X'), + (0x1F7D9, 'X'), (0x1F800, 'V'), (0x1F80C, 'X'), (0x1F810, 'V'), @@ -7521,15 +7536,21 @@ def _seg_72(): (0x1F910, 'V'), (0x1F93F, 'X'), (0x1F940, 'V'), - (0x1F94D, 'X'), - (0x1F950, 'V'), - (0x1F96C, 'X'), - (0x1F980, 'V'), - (0x1F998, 'X'), + (0x1F971, 'X'), + (0x1F973, 'V'), + (0x1F977, 'X'), + (0x1F97A, 'V'), + (0x1F97B, 'X'), + (0x1F97C, 'V'), + (0x1F9A3, 'X'), + (0x1F9B0, 'V'), + (0x1F9BA, 'X'), (0x1F9C0, 'V'), - (0x1F9C1, 'X'), + (0x1F9C3, 'X'), (0x1F9D0, 'V'), - (0x1F9E7, 'X'), + (0x1FA00, 'X'), + (0x1FA60, 'V'), + (0x1FA6E, 'X'), (0x20000, 'V'), (0x2A6D7, 'X'), (0x2A700, 'V'), @@ -7574,6 +7595,10 @@ def _seg_72(): (0x2F81F, 'M', u'㓟'), (0x2F820, 'M', u'刻'), (0x2F821, 'M', u'剆'), + ] + +def _seg_73(): + return [ (0x2F822, 'M', u'割'), (0x2F823, 'M', u'剷'), (0x2F824, 'M', u'㔕'), @@ -7595,10 +7620,6 @@ def _seg_72(): (0x2F836, 'M', u'及'), (0x2F837, 'M', u'叟'), (0x2F838, 'M', u'𠭣'), - ] - -def _seg_73(): - return [ (0x2F839, 'M', u'叫'), (0x2F83A, 'M', u'叱'), (0x2F83B, 'M', u'吆'), @@ -7678,6 +7699,10 @@ def _seg_73(): (0x2F887, 'M', u'幩'), (0x2F888, 'M', u'㡢'), (0x2F889, 'M', u'𢆃'), + ] + +def _seg_74(): + return [ (0x2F88A, 'M', u'㡼'), (0x2F88B, 'M', u'庰'), (0x2F88C, 'M', u'庳'), @@ -7699,10 +7724,6 @@ def _seg_73(): (0x2F89E, 'M', u'志'), (0x2F89F, 'M', u'忹'), (0x2F8A0, 'M', u'悁'), - ] - -def _seg_74(): - return [ (0x2F8A1, 'M', u'㤺'), (0x2F8A2, 'M', u'㤜'), (0x2F8A3, 'M', u'悔'), @@ -7782,6 +7803,10 @@ def _seg_74(): (0x2F8ED, 'M', u'櫛'), (0x2F8EE, 'M', u'㰘'), (0x2F8EF, 'M', u'次'), + ] + +def _seg_75(): + return [ (0x2F8F0, 'M', u'𣢧'), (0x2F8F1, 'M', u'歔'), (0x2F8F2, 'M', u'㱎'), @@ -7803,10 +7828,6 @@ def _seg_74(): (0x2F902, 'M', u'流'), (0x2F903, 'M', u'浩'), (0x2F904, 'M', u'浸'), - ] - -def _seg_75(): - return [ (0x2F905, 'M', u'涅'), (0x2F906, 'M', u'𣴞'), (0x2F907, 'M', u'洴'), @@ -7886,6 +7907,10 @@ def _seg_75(): (0x2F953, 'M', u'祖'), (0x2F954, 'M', u'𥚚'), (0x2F955, 'M', u'𥛅'), + ] + +def _seg_76(): + return [ (0x2F956, 'M', u'福'), (0x2F957, 'M', u'秫'), (0x2F958, 'M', u'䄯'), @@ -7907,10 +7932,6 @@ def _seg_75(): (0x2F969, 'M', u'糣'), (0x2F96A, 'M', u'紀'), (0x2F96B, 'M', u'𥾆'), - ] - -def _seg_76(): - return [ (0x2F96C, 'M', u'絣'), (0x2F96D, 'M', u'䌁'), (0x2F96E, 'M', u'緇'), @@ -7990,6 +8011,10 @@ def _seg_76(): (0x2F9B8, 'M', u'蚈'), (0x2F9B9, 'M', u'蜎'), (0x2F9BA, 'M', u'蛢'), + ] + +def _seg_77(): + return [ (0x2F9BB, 'M', u'蝹'), (0x2F9BC, 'M', u'蜨'), (0x2F9BD, 'M', u'蝫'), @@ -8011,10 +8036,6 @@ def _seg_76(): (0x2F9CD, 'M', u'䚾'), (0x2F9CE, 'M', u'䛇'), (0x2F9CF, 'M', u'誠'), - ] - -def _seg_77(): - return [ (0x2F9D0, 'M', u'諭'), (0x2F9D1, 'M', u'變'), (0x2F9D2, 'M', u'豕'), @@ -8094,6 +8115,10 @@ def _seg_77(): (0x2FA1D, 'M', u'𪘀'), (0x2FA1E, 'X'), (0xE0100, 'I'), + ] + +def _seg_78(): + return [ (0xE01F0, 'X'), ] @@ -8176,4 +8201,5 @@ uts46data = tuple( + _seg_75() + _seg_76() + _seg_77() + + _seg_78() ) diff --git a/pipenv/vendor/markupsafe/LICENSE b/pipenv/vendor/markupsafe/LICENSE deleted file mode 100644 index 5d269389..00000000 --- a/pipenv/vendor/markupsafe/LICENSE +++ /dev/null @@ -1,33 +0,0 @@ -Copyright (c) 2010 by Armin Ronacher and contributors. See AUTHORS -for more details. - -Some rights reserved. - -Redistribution and use in source and binary forms of the software as well -as documentation, with or without modification, are permitted provided -that the following conditions are met: - -* Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials provided - with the distribution. - -* The names of the contributors may not be used to endorse or - promote products derived from this software without specific - prior written permission. - -THIS SOFTWARE AND DOCUMENTATION IS PROVIDED BY THE COPYRIGHT HOLDERS AND -CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT -NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER -OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE AND DOCUMENTATION, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH -DAMAGE. diff --git a/pipenv/vendor/markupsafe/LICENSE.rst b/pipenv/vendor/markupsafe/LICENSE.rst new file mode 100644 index 00000000..9d227a0c --- /dev/null +++ b/pipenv/vendor/markupsafe/LICENSE.rst @@ -0,0 +1,28 @@ +Copyright 2010 Pallets + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/pipenv/vendor/markupsafe/__init__.py b/pipenv/vendor/markupsafe/__init__.py index 68dc85f6..da05ed32 100644 --- a/pipenv/vendor/markupsafe/__init__.py +++ b/pipenv/vendor/markupsafe/__init__.py @@ -1,75 +1,74 @@ # -*- coding: utf-8 -*- """ - markupsafe - ~~~~~~~~~~ +markupsafe +~~~~~~~~~~ - Implements a Markup string. +Implements an escape function and a Markup string to replace HTML +special characters with safe representations. - :copyright: (c) 2010 by Armin Ronacher. - :license: BSD, see LICENSE for more details. +:copyright: 2010 Pallets +:license: BSD-3-Clause """ import re import string -from collections import Mapping -from markupsafe._compat import text_type, string_types, int_types, \ - unichr, iteritems, PY2 -__version__ = "1.0" +from ._compat import int_types +from ._compat import iteritems +from ._compat import Mapping +from ._compat import PY2 +from ._compat import string_types +from ._compat import text_type +from ._compat import unichr -__all__ = ['Markup', 'soft_unicode', 'escape', 'escape_silent'] +__version__ = "1.1.1" +__all__ = ["Markup", "soft_unicode", "escape", "escape_silent"] -_striptags_re = re.compile(r'(|<[^>]*>)') -_entity_re = re.compile(r'&([^& ;]+);') +_striptags_re = re.compile(r"(|<[^>]*>)") +_entity_re = re.compile(r"&([^& ;]+);") class Markup(text_type): - r"""Marks a string as being safe for inclusion in HTML/XML output without - needing to be escaped. This implements the `__html__` interface a couple - of frameworks and web applications use. :class:`Markup` is a direct - subclass of `unicode` and provides all the methods of `unicode` just that - it escapes arguments passed and always returns `Markup`. + """A string that is ready to be safely inserted into an HTML or XML + document, either because it was escaped or because it was marked + safe. - The `escape` function returns markup objects so that double escaping can't - happen. + Passing an object to the constructor converts it to text and wraps + it to mark it safe without escaping. To escape the text, use the + :meth:`escape` class method instead. - The constructor of the :class:`Markup` class can be used for three - different things: When passed an unicode object it's assumed to be safe, - when passed an object with an HTML representation (has an `__html__` - method) that representation is used, otherwise the object passed is - converted into a unicode string and then assumed to be safe: + >>> Markup('Hello, World!') + Markup('Hello, World!') + >>> Markup(42) + Markup('42') + >>> Markup.escape('Hello, World!') + Markup('Hello <em>World</em>!') - >>> Markup("Hello World!") - Markup(u'Hello World!') - >>> class Foo(object): - ... def __html__(self): - ... return 'foo' + This implements the ``__html__()`` interface that some frameworks + use. Passing an object that implements ``__html__()`` will wrap the + output of that method, marking it safe. + + >>> class Foo: + ... def __html__(self): + ... return 'foo' ... >>> Markup(Foo()) - Markup(u'foo') + Markup('foo') - If you want object passed being always treated as unsafe you can use the - :meth:`escape` classmethod to create a :class:`Markup` object: + This is a subclass of the text type (``str`` in Python 3, + ``unicode`` in Python 2). It has the same methods as that type, but + all methods escape their arguments and return a ``Markup`` instance. - >>> Markup.escape("Hello World!") - Markup(u'Hello <em>World</em>!') - - Operations on a markup string are markup aware which means that all - arguments are passed through the :func:`escape` function: - - >>> em = Markup("%s") - >>> em % "foo & bar" - Markup(u'foo & bar') - >>> strong = Markup("%(text)s") - >>> strong % {'text': 'hacker here'} - Markup(u'<blink>hacker here</blink>') - >>> Markup("Hello ") + "" - Markup(u'Hello <foo>') + >>> Markup('%s') % 'foo & bar' + Markup('foo & bar') + >>> Markup('Hello ') + '' + Markup('Hello <foo>') """ + __slots__ = () - def __new__(cls, base=u'', encoding=None, errors='strict'): - if hasattr(base, '__html__'): + def __new__(cls, base=u"", encoding=None, errors="strict"): + if hasattr(base, "__html__"): base = base.__html__() if encoding is None: return text_type.__new__(cls, base) @@ -79,12 +78,12 @@ class Markup(text_type): return self def __add__(self, other): - if isinstance(other, string_types) or hasattr(other, '__html__'): + if isinstance(other, string_types) or hasattr(other, "__html__"): return self.__class__(super(Markup, self).__add__(self.escape(other))) return NotImplemented def __radd__(self, other): - if hasattr(other, '__html__') or isinstance(other, string_types): + if hasattr(other, "__html__") or isinstance(other, string_types): return self.escape(other).__add__(self) return NotImplemented @@ -92,6 +91,7 @@ class Markup(text_type): if isinstance(num, int_types): return self.__class__(text_type.__mul__(self, num)) return NotImplemented + __rmul__ = __mul__ def __mod__(self, arg): @@ -102,115 +102,124 @@ class Markup(text_type): return self.__class__(text_type.__mod__(self, arg)) def __repr__(self): - return '%s(%s)' % ( - self.__class__.__name__, - text_type.__repr__(self) - ) + return "%s(%s)" % (self.__class__.__name__, text_type.__repr__(self)) def join(self, seq): return self.__class__(text_type.join(self, map(self.escape, seq))) + join.__doc__ = text_type.join.__doc__ def split(self, *args, **kwargs): return list(map(self.__class__, text_type.split(self, *args, **kwargs))) + split.__doc__ = text_type.split.__doc__ def rsplit(self, *args, **kwargs): return list(map(self.__class__, text_type.rsplit(self, *args, **kwargs))) + rsplit.__doc__ = text_type.rsplit.__doc__ def splitlines(self, *args, **kwargs): - return list(map(self.__class__, text_type.splitlines( - self, *args, **kwargs))) + return list(map(self.__class__, text_type.splitlines(self, *args, **kwargs))) + splitlines.__doc__ = text_type.splitlines.__doc__ def unescape(self): - r"""Unescape markup again into an text_type string. This also resolves - known HTML4 and XHTML entities: + """Convert escaped markup back into a text string. This replaces + HTML entities with the characters they represent. - >>> Markup("Main » About").unescape() - u'Main \xbb About' + >>> Markup('Main » About').unescape() + 'Main » About' """ - from markupsafe._constants import HTML_ENTITIES + from ._constants import HTML_ENTITIES + def handle_match(m): name = m.group(1) if name in HTML_ENTITIES: return unichr(HTML_ENTITIES[name]) try: - if name[:2] in ('#x', '#X'): + if name[:2] in ("#x", "#X"): return unichr(int(name[2:], 16)) - elif name.startswith('#'): + elif name.startswith("#"): return unichr(int(name[1:])) except ValueError: pass # Don't modify unexpected input. return m.group() + return _entity_re.sub(handle_match, text_type(self)) def striptags(self): - r"""Unescape markup into an text_type string and strip all tags. This - also resolves known HTML4 and XHTML entities. Whitespace is - normalized to one: + """:meth:`unescape` the markup, remove tags, and normalize + whitespace to single spaces. - >>> Markup("Main » About").striptags() - u'Main \xbb About' + >>> Markup('Main »\tAbout').striptags() + 'Main » About' """ - stripped = u' '.join(_striptags_re.sub('', self).split()) + stripped = u" ".join(_striptags_re.sub("", self).split()) return Markup(stripped).unescape() @classmethod def escape(cls, s): - """Escape the string. Works like :func:`escape` with the difference - that for subclasses of :class:`Markup` this function would return the - correct subclass. + """Escape a string. Calls :func:`escape` and ensures that for + subclasses the correct type is returned. """ rv = escape(s) if rv.__class__ is not cls: return cls(rv) return rv - def make_simple_escaping_wrapper(name): + def make_simple_escaping_wrapper(name): # noqa: B902 orig = getattr(text_type, name) + def func(self, *args, **kwargs): args = _escape_argspec(list(args), enumerate(args), self.escape) _escape_argspec(kwargs, iteritems(kwargs), self.escape) return self.__class__(orig(self, *args, **kwargs)) + func.__name__ = orig.__name__ func.__doc__ = orig.__doc__ return func - for method in '__getitem__', 'capitalize', \ - 'title', 'lower', 'upper', 'replace', 'ljust', \ - 'rjust', 'lstrip', 'rstrip', 'center', 'strip', \ - 'translate', 'expandtabs', 'swapcase', 'zfill': + for method in ( + "__getitem__", + "capitalize", + "title", + "lower", + "upper", + "replace", + "ljust", + "rjust", + "lstrip", + "rstrip", + "center", + "strip", + "translate", + "expandtabs", + "swapcase", + "zfill", + ): locals()[method] = make_simple_escaping_wrapper(method) - # new in python 2.5 - if hasattr(text_type, 'partition'): - def partition(self, sep): - return tuple(map(self.__class__, - text_type.partition(self, self.escape(sep)))) - def rpartition(self, sep): - return tuple(map(self.__class__, - text_type.rpartition(self, self.escape(sep)))) + def partition(self, sep): + return tuple(map(self.__class__, text_type.partition(self, self.escape(sep)))) - # new in python 2.6 - if hasattr(text_type, 'format'): - def format(*args, **kwargs): - self, args = args[0], args[1:] - formatter = EscapeFormatter(self.escape) - kwargs = _MagicFormatMapping(args, kwargs) - return self.__class__(formatter.vformat(self, args, kwargs)) + def rpartition(self, sep): + return tuple(map(self.__class__, text_type.rpartition(self, self.escape(sep)))) - def __html_format__(self, format_spec): - if format_spec: - raise ValueError('Unsupported format specification ' - 'for Markup.') - return self + def format(self, *args, **kwargs): + formatter = EscapeFormatter(self.escape) + kwargs = _MagicFormatMapping(args, kwargs) + return self.__class__(formatter.vformat(self, args, kwargs)) + + def __html_format__(self, format_spec): + if format_spec: + raise ValueError("Unsupported format specification " "for Markup.") + return self # not in python 3 - if hasattr(text_type, '__getslice__'): - __getslice__ = make_simple_escaping_wrapper('__getslice__') + if hasattr(text_type, "__getslice__"): + __getslice__ = make_simple_escaping_wrapper("__getslice__") del method, make_simple_escaping_wrapper @@ -229,7 +238,7 @@ class _MagicFormatMapping(Mapping): self._last_index = 0 def __getitem__(self, key): - if key == '': + if key == "": idx = self._last_index self._last_index += 1 try: @@ -246,35 +255,37 @@ class _MagicFormatMapping(Mapping): return len(self._kwargs) -if hasattr(text_type, 'format'): - class EscapeFormatter(string.Formatter): +if hasattr(text_type, "format"): + class EscapeFormatter(string.Formatter): def __init__(self, escape): self.escape = escape def format_field(self, value, format_spec): - if hasattr(value, '__html_format__'): + if hasattr(value, "__html_format__"): rv = value.__html_format__(format_spec) - elif hasattr(value, '__html__'): + elif hasattr(value, "__html__"): if format_spec: - raise ValueError('No format specification allowed ' - 'when formatting an object with ' - 'its __html__ method.') + raise ValueError( + "Format specifier {0} given, but {1} does not" + " define __html_format__. A class that defines" + " __html__ must define __html_format__ to work" + " with format specifiers.".format(format_spec, type(value)) + ) rv = value.__html__() else: # We need to make sure the format spec is unicode here as # otherwise the wrong callback methods are invoked. For # instance a byte string there would invoke __str__ and # not __unicode__. - rv = string.Formatter.format_field( - self, value, text_type(format_spec)) + rv = string.Formatter.format_field(self, value, text_type(format_spec)) return text_type(self.escape(rv)) def _escape_argspec(obj, iterable, escape): """Helper for various string-wrapped functions.""" for key, value in iterable: - if hasattr(value, '__html__') or isinstance(value, string_types): + if hasattr(value, "__html__") or isinstance(value, string_types): obj[key] = escape(value) return obj @@ -286,20 +297,31 @@ class _MarkupEscapeHelper(object): self.obj = obj self.escape = escape - __getitem__ = lambda s, x: _MarkupEscapeHelper(s.obj[x], s.escape) - __unicode__ = __str__ = lambda s: text_type(s.escape(s.obj)) - __repr__ = lambda s: str(s.escape(repr(s.obj))) - __int__ = lambda s: int(s.obj) - __float__ = lambda s: float(s.obj) + def __getitem__(self, item): + return _MarkupEscapeHelper(self.obj[item], self.escape) + + def __str__(self): + return text_type(self.escape(self.obj)) + + __unicode__ = __str__ + + def __repr__(self): + return str(self.escape(repr(self.obj))) + + def __int__(self): + return int(self.obj) + + def __float__(self): + return float(self.obj) # we have to import it down here as the speedups and native # modules imports the markup type which is define above. try: - from markupsafe._speedups import escape, escape_silent, soft_unicode + from ._speedups import escape, escape_silent, soft_unicode except ImportError: - from markupsafe._native import escape, escape_silent, soft_unicode + from ._native import escape, escape_silent, soft_unicode if not PY2: soft_str = soft_unicode - __all__.append('soft_str') + __all__.append("soft_str") diff --git a/pipenv/vendor/markupsafe/_compat.py b/pipenv/vendor/markupsafe/_compat.py index 62e5632a..bc05090f 100644 --- a/pipenv/vendor/markupsafe/_compat.py +++ b/pipenv/vendor/markupsafe/_compat.py @@ -1,12 +1,10 @@ # -*- coding: utf-8 -*- """ - markupsafe._compat - ~~~~~~~~~~~~~~~~~~ +markupsafe._compat +~~~~~~~~~~~~~~~~~~ - Compatibility module for different Python versions. - - :copyright: (c) 2013 by Armin Ronacher. - :license: BSD, see LICENSE for more details. +:copyright: 2010 Pallets +:license: BSD-3-Clause """ import sys @@ -17,10 +15,19 @@ if not PY2: string_types = (str,) unichr = chr int_types = (int,) - iteritems = lambda x: iter(x.items()) + + def iteritems(x): + return iter(x.items()) + + from collections.abc import Mapping + else: text_type = unicode string_types = (str, unicode) unichr = unichr int_types = (int, long) - iteritems = lambda x: x.iteritems() + + def iteritems(x): + return x.iteritems() + + from collections import Mapping diff --git a/pipenv/vendor/markupsafe/_constants.py b/pipenv/vendor/markupsafe/_constants.py index 919bf03c..7c57c2d2 100644 --- a/pipenv/vendor/markupsafe/_constants.py +++ b/pipenv/vendor/markupsafe/_constants.py @@ -1,267 +1,264 @@ # -*- coding: utf-8 -*- """ - markupsafe._constants - ~~~~~~~~~~~~~~~~~~~~~ +markupsafe._constants +~~~~~~~~~~~~~~~~~~~~~ - Highlevel implementation of the Markup string. - - :copyright: (c) 2010 by Armin Ronacher. - :license: BSD, see LICENSE for more details. +:copyright: 2010 Pallets +:license: BSD-3-Clause """ - HTML_ENTITIES = { - 'AElig': 198, - 'Aacute': 193, - 'Acirc': 194, - 'Agrave': 192, - 'Alpha': 913, - 'Aring': 197, - 'Atilde': 195, - 'Auml': 196, - 'Beta': 914, - 'Ccedil': 199, - 'Chi': 935, - 'Dagger': 8225, - 'Delta': 916, - 'ETH': 208, - 'Eacute': 201, - 'Ecirc': 202, - 'Egrave': 200, - 'Epsilon': 917, - 'Eta': 919, - 'Euml': 203, - 'Gamma': 915, - 'Iacute': 205, - 'Icirc': 206, - 'Igrave': 204, - 'Iota': 921, - 'Iuml': 207, - 'Kappa': 922, - 'Lambda': 923, - 'Mu': 924, - 'Ntilde': 209, - 'Nu': 925, - 'OElig': 338, - 'Oacute': 211, - 'Ocirc': 212, - 'Ograve': 210, - 'Omega': 937, - 'Omicron': 927, - 'Oslash': 216, - 'Otilde': 213, - 'Ouml': 214, - 'Phi': 934, - 'Pi': 928, - 'Prime': 8243, - 'Psi': 936, - 'Rho': 929, - 'Scaron': 352, - 'Sigma': 931, - 'THORN': 222, - 'Tau': 932, - 'Theta': 920, - 'Uacute': 218, - 'Ucirc': 219, - 'Ugrave': 217, - 'Upsilon': 933, - 'Uuml': 220, - 'Xi': 926, - 'Yacute': 221, - 'Yuml': 376, - 'Zeta': 918, - 'aacute': 225, - 'acirc': 226, - 'acute': 180, - 'aelig': 230, - 'agrave': 224, - 'alefsym': 8501, - 'alpha': 945, - 'amp': 38, - 'and': 8743, - 'ang': 8736, - 'apos': 39, - 'aring': 229, - 'asymp': 8776, - 'atilde': 227, - 'auml': 228, - 'bdquo': 8222, - 'beta': 946, - 'brvbar': 166, - 'bull': 8226, - 'cap': 8745, - 'ccedil': 231, - 'cedil': 184, - 'cent': 162, - 'chi': 967, - 'circ': 710, - 'clubs': 9827, - 'cong': 8773, - 'copy': 169, - 'crarr': 8629, - 'cup': 8746, - 'curren': 164, - 'dArr': 8659, - 'dagger': 8224, - 'darr': 8595, - 'deg': 176, - 'delta': 948, - 'diams': 9830, - 'divide': 247, - 'eacute': 233, - 'ecirc': 234, - 'egrave': 232, - 'empty': 8709, - 'emsp': 8195, - 'ensp': 8194, - 'epsilon': 949, - 'equiv': 8801, - 'eta': 951, - 'eth': 240, - 'euml': 235, - 'euro': 8364, - 'exist': 8707, - 'fnof': 402, - 'forall': 8704, - 'frac12': 189, - 'frac14': 188, - 'frac34': 190, - 'frasl': 8260, - 'gamma': 947, - 'ge': 8805, - 'gt': 62, - 'hArr': 8660, - 'harr': 8596, - 'hearts': 9829, - 'hellip': 8230, - 'iacute': 237, - 'icirc': 238, - 'iexcl': 161, - 'igrave': 236, - 'image': 8465, - 'infin': 8734, - 'int': 8747, - 'iota': 953, - 'iquest': 191, - 'isin': 8712, - 'iuml': 239, - 'kappa': 954, - 'lArr': 8656, - 'lambda': 955, - 'lang': 9001, - 'laquo': 171, - 'larr': 8592, - 'lceil': 8968, - 'ldquo': 8220, - 'le': 8804, - 'lfloor': 8970, - 'lowast': 8727, - 'loz': 9674, - 'lrm': 8206, - 'lsaquo': 8249, - 'lsquo': 8216, - 'lt': 60, - 'macr': 175, - 'mdash': 8212, - 'micro': 181, - 'middot': 183, - 'minus': 8722, - 'mu': 956, - 'nabla': 8711, - 'nbsp': 160, - 'ndash': 8211, - 'ne': 8800, - 'ni': 8715, - 'not': 172, - 'notin': 8713, - 'nsub': 8836, - 'ntilde': 241, - 'nu': 957, - 'oacute': 243, - 'ocirc': 244, - 'oelig': 339, - 'ograve': 242, - 'oline': 8254, - 'omega': 969, - 'omicron': 959, - 'oplus': 8853, - 'or': 8744, - 'ordf': 170, - 'ordm': 186, - 'oslash': 248, - 'otilde': 245, - 'otimes': 8855, - 'ouml': 246, - 'para': 182, - 'part': 8706, - 'permil': 8240, - 'perp': 8869, - 'phi': 966, - 'pi': 960, - 'piv': 982, - 'plusmn': 177, - 'pound': 163, - 'prime': 8242, - 'prod': 8719, - 'prop': 8733, - 'psi': 968, - 'quot': 34, - 'rArr': 8658, - 'radic': 8730, - 'rang': 9002, - 'raquo': 187, - 'rarr': 8594, - 'rceil': 8969, - 'rdquo': 8221, - 'real': 8476, - 'reg': 174, - 'rfloor': 8971, - 'rho': 961, - 'rlm': 8207, - 'rsaquo': 8250, - 'rsquo': 8217, - 'sbquo': 8218, - 'scaron': 353, - 'sdot': 8901, - 'sect': 167, - 'shy': 173, - 'sigma': 963, - 'sigmaf': 962, - 'sim': 8764, - 'spades': 9824, - 'sub': 8834, - 'sube': 8838, - 'sum': 8721, - 'sup': 8835, - 'sup1': 185, - 'sup2': 178, - 'sup3': 179, - 'supe': 8839, - 'szlig': 223, - 'tau': 964, - 'there4': 8756, - 'theta': 952, - 'thetasym': 977, - 'thinsp': 8201, - 'thorn': 254, - 'tilde': 732, - 'times': 215, - 'trade': 8482, - 'uArr': 8657, - 'uacute': 250, - 'uarr': 8593, - 'ucirc': 251, - 'ugrave': 249, - 'uml': 168, - 'upsih': 978, - 'upsilon': 965, - 'uuml': 252, - 'weierp': 8472, - 'xi': 958, - 'yacute': 253, - 'yen': 165, - 'yuml': 255, - 'zeta': 950, - 'zwj': 8205, - 'zwnj': 8204 + "AElig": 198, + "Aacute": 193, + "Acirc": 194, + "Agrave": 192, + "Alpha": 913, + "Aring": 197, + "Atilde": 195, + "Auml": 196, + "Beta": 914, + "Ccedil": 199, + "Chi": 935, + "Dagger": 8225, + "Delta": 916, + "ETH": 208, + "Eacute": 201, + "Ecirc": 202, + "Egrave": 200, + "Epsilon": 917, + "Eta": 919, + "Euml": 203, + "Gamma": 915, + "Iacute": 205, + "Icirc": 206, + "Igrave": 204, + "Iota": 921, + "Iuml": 207, + "Kappa": 922, + "Lambda": 923, + "Mu": 924, + "Ntilde": 209, + "Nu": 925, + "OElig": 338, + "Oacute": 211, + "Ocirc": 212, + "Ograve": 210, + "Omega": 937, + "Omicron": 927, + "Oslash": 216, + "Otilde": 213, + "Ouml": 214, + "Phi": 934, + "Pi": 928, + "Prime": 8243, + "Psi": 936, + "Rho": 929, + "Scaron": 352, + "Sigma": 931, + "THORN": 222, + "Tau": 932, + "Theta": 920, + "Uacute": 218, + "Ucirc": 219, + "Ugrave": 217, + "Upsilon": 933, + "Uuml": 220, + "Xi": 926, + "Yacute": 221, + "Yuml": 376, + "Zeta": 918, + "aacute": 225, + "acirc": 226, + "acute": 180, + "aelig": 230, + "agrave": 224, + "alefsym": 8501, + "alpha": 945, + "amp": 38, + "and": 8743, + "ang": 8736, + "apos": 39, + "aring": 229, + "asymp": 8776, + "atilde": 227, + "auml": 228, + "bdquo": 8222, + "beta": 946, + "brvbar": 166, + "bull": 8226, + "cap": 8745, + "ccedil": 231, + "cedil": 184, + "cent": 162, + "chi": 967, + "circ": 710, + "clubs": 9827, + "cong": 8773, + "copy": 169, + "crarr": 8629, + "cup": 8746, + "curren": 164, + "dArr": 8659, + "dagger": 8224, + "darr": 8595, + "deg": 176, + "delta": 948, + "diams": 9830, + "divide": 247, + "eacute": 233, + "ecirc": 234, + "egrave": 232, + "empty": 8709, + "emsp": 8195, + "ensp": 8194, + "epsilon": 949, + "equiv": 8801, + "eta": 951, + "eth": 240, + "euml": 235, + "euro": 8364, + "exist": 8707, + "fnof": 402, + "forall": 8704, + "frac12": 189, + "frac14": 188, + "frac34": 190, + "frasl": 8260, + "gamma": 947, + "ge": 8805, + "gt": 62, + "hArr": 8660, + "harr": 8596, + "hearts": 9829, + "hellip": 8230, + "iacute": 237, + "icirc": 238, + "iexcl": 161, + "igrave": 236, + "image": 8465, + "infin": 8734, + "int": 8747, + "iota": 953, + "iquest": 191, + "isin": 8712, + "iuml": 239, + "kappa": 954, + "lArr": 8656, + "lambda": 955, + "lang": 9001, + "laquo": 171, + "larr": 8592, + "lceil": 8968, + "ldquo": 8220, + "le": 8804, + "lfloor": 8970, + "lowast": 8727, + "loz": 9674, + "lrm": 8206, + "lsaquo": 8249, + "lsquo": 8216, + "lt": 60, + "macr": 175, + "mdash": 8212, + "micro": 181, + "middot": 183, + "minus": 8722, + "mu": 956, + "nabla": 8711, + "nbsp": 160, + "ndash": 8211, + "ne": 8800, + "ni": 8715, + "not": 172, + "notin": 8713, + "nsub": 8836, + "ntilde": 241, + "nu": 957, + "oacute": 243, + "ocirc": 244, + "oelig": 339, + "ograve": 242, + "oline": 8254, + "omega": 969, + "omicron": 959, + "oplus": 8853, + "or": 8744, + "ordf": 170, + "ordm": 186, + "oslash": 248, + "otilde": 245, + "otimes": 8855, + "ouml": 246, + "para": 182, + "part": 8706, + "permil": 8240, + "perp": 8869, + "phi": 966, + "pi": 960, + "piv": 982, + "plusmn": 177, + "pound": 163, + "prime": 8242, + "prod": 8719, + "prop": 8733, + "psi": 968, + "quot": 34, + "rArr": 8658, + "radic": 8730, + "rang": 9002, + "raquo": 187, + "rarr": 8594, + "rceil": 8969, + "rdquo": 8221, + "real": 8476, + "reg": 174, + "rfloor": 8971, + "rho": 961, + "rlm": 8207, + "rsaquo": 8250, + "rsquo": 8217, + "sbquo": 8218, + "scaron": 353, + "sdot": 8901, + "sect": 167, + "shy": 173, + "sigma": 963, + "sigmaf": 962, + "sim": 8764, + "spades": 9824, + "sub": 8834, + "sube": 8838, + "sum": 8721, + "sup": 8835, + "sup1": 185, + "sup2": 178, + "sup3": 179, + "supe": 8839, + "szlig": 223, + "tau": 964, + "there4": 8756, + "theta": 952, + "thetasym": 977, + "thinsp": 8201, + "thorn": 254, + "tilde": 732, + "times": 215, + "trade": 8482, + "uArr": 8657, + "uacute": 250, + "uarr": 8593, + "ucirc": 251, + "ugrave": 249, + "uml": 168, + "upsih": 978, + "upsilon": 965, + "uuml": 252, + "weierp": 8472, + "xi": 958, + "yacute": 253, + "yen": 165, + "yuml": 255, + "zeta": 950, + "zwj": 8205, + "zwnj": 8204, } diff --git a/pipenv/vendor/markupsafe/_native.py b/pipenv/vendor/markupsafe/_native.py index 5e83f10a..cd08752c 100644 --- a/pipenv/vendor/markupsafe/_native.py +++ b/pipenv/vendor/markupsafe/_native.py @@ -1,36 +1,49 @@ # -*- coding: utf-8 -*- """ - markupsafe._native - ~~~~~~~~~~~~~~~~~~ +markupsafe._native +~~~~~~~~~~~~~~~~~~ - Native Python implementation the C module is not compiled. +Native Python implementation used when the C module is not compiled. - :copyright: (c) 2010 by Armin Ronacher. - :license: BSD, see LICENSE for more details. +:copyright: 2010 Pallets +:license: BSD-3-Clause """ -from markupsafe import Markup -from markupsafe._compat import text_type +from . import Markup +from ._compat import text_type def escape(s): - """Convert the characters &, <, >, ' and " in string s to HTML-safe - sequences. Use this if you need to display text that might contain - such characters in HTML. Marks return value as markup string. + """Replace the characters ``&``, ``<``, ``>``, ``'``, and ``"`` in + the string with HTML-safe sequences. Use this if you need to display + text that might contain such characters in HTML. + + If the object has an ``__html__`` method, it is called and the + return value is assumed to already be safe for HTML. + + :param s: An object to be converted to a string and escaped. + :return: A :class:`Markup` string with the escaped text. """ - if hasattr(s, '__html__'): - return s.__html__() - return Markup(text_type(s) - .replace('&', '&') - .replace('>', '>') - .replace('<', '<') - .replace("'", ''') - .replace('"', '"') + if hasattr(s, "__html__"): + return Markup(s.__html__()) + return Markup( + text_type(s) + .replace("&", "&") + .replace(">", ">") + .replace("<", "<") + .replace("'", "'") + .replace('"', """) ) def escape_silent(s): - """Like :func:`escape` but converts `None` into an empty - markup string. + """Like :func:`escape` but treats ``None`` as the empty string. + Useful with optional values, as otherwise you get the string + ``'None'`` when the value is ``None``. + + >>> escape(None) + Markup('None') + >>> escape_silent(None) + Markup('') """ if s is None: return Markup() @@ -38,8 +51,18 @@ def escape_silent(s): def soft_unicode(s): - """Make a string unicode if it isn't already. That way a markup - string is not converted back to unicode. + """Convert an object to a string if it isn't already. This preserves + a :class:`Markup` string rather than converting it back to a basic + string, so it will still be marked as safe and won't be escaped + again. + + >>> value = escape('') + >>> value + Markup('<User 1>') + >>> escape(str(value)) + Markup('&lt;User 1&gt;') + >>> escape(soft_unicode(value)) + Markup('<User 1>') """ if not isinstance(s, text_type): s = text_type(s) diff --git a/pipenv/vendor/markupsafe/_speedups.c b/pipenv/vendor/markupsafe/_speedups.c index d779a68c..12d2c4a7 100644 --- a/pipenv/vendor/markupsafe/_speedups.c +++ b/pipenv/vendor/markupsafe/_speedups.c @@ -2,33 +2,30 @@ * markupsafe._speedups * ~~~~~~~~~~~~~~~~~~~~ * - * This module implements functions for automatic escaping in C for better - * performance. + * C implementation of escaping for better performance. Used instead of + * the native Python implementation when compiled. * - * :copyright: (c) 2010 by Armin Ronacher. - * :license: BSD. + * :copyright: 2010 Pallets + * :license: BSD-3-Clause */ - #include +#if PY_MAJOR_VERSION < 3 #define ESCAPED_CHARS_TABLE_SIZE 63 #define UNICHR(x) (PyUnicode_AS_UNICODE((PyUnicodeObject*)PyUnicode_DecodeASCII(x, strlen(x), NULL))); -#if PY_VERSION_HEX < 0x02050000 && !defined(PY_SSIZE_T_MIN) -typedef int Py_ssize_t; -#define PY_SSIZE_T_MAX INT_MAX -#define PY_SSIZE_T_MIN INT_MIN -#endif - - -static PyObject* markup; static Py_ssize_t escaped_chars_delta_len[ESCAPED_CHARS_TABLE_SIZE]; static Py_UNICODE *escaped_chars_repl[ESCAPED_CHARS_TABLE_SIZE]; +#endif + +static PyObject* markup; static int init_constants(void) { PyObject *module; + +#if PY_MAJOR_VERSION < 3 /* mapping of characters to replace */ escaped_chars_repl['"'] = UNICHR("""); escaped_chars_repl['\''] = UNICHR("'"); @@ -41,6 +38,7 @@ init_constants(void) escaped_chars_delta_len['"'] = escaped_chars_delta_len['\''] = \ escaped_chars_delta_len['&'] = 4; escaped_chars_delta_len['<'] = escaped_chars_delta_len['>'] = 3; +#endif /* import markup type so that we can mark the return value */ module = PyImport_ImportModule("markupsafe"); @@ -52,6 +50,7 @@ init_constants(void) return 1; } +#if PY_MAJOR_VERSION < 3 static PyObject* escape_unicode(PyUnicodeObject *in) { @@ -112,13 +111,192 @@ escape_unicode(PyUnicodeObject *in) return (PyObject*)out; } +#else /* PY_MAJOR_VERSION < 3 */ +#define GET_DELTA(inp, inp_end, delta) \ + while (inp < inp_end) { \ + switch (*inp++) { \ + case '"': \ + case '\'': \ + case '&': \ + delta += 4; \ + break; \ + case '<': \ + case '>': \ + delta += 3; \ + break; \ + } \ + } + +#define DO_ESCAPE(inp, inp_end, outp) \ + { \ + Py_ssize_t ncopy = 0; \ + while (inp < inp_end) { \ + switch (*inp) { \ + case '"': \ + memcpy(outp, inp-ncopy, sizeof(*outp)*ncopy); \ + outp += ncopy; ncopy = 0; \ + *outp++ = '&'; \ + *outp++ = '#'; \ + *outp++ = '3'; \ + *outp++ = '4'; \ + *outp++ = ';'; \ + break; \ + case '\'': \ + memcpy(outp, inp-ncopy, sizeof(*outp)*ncopy); \ + outp += ncopy; ncopy = 0; \ + *outp++ = '&'; \ + *outp++ = '#'; \ + *outp++ = '3'; \ + *outp++ = '9'; \ + *outp++ = ';'; \ + break; \ + case '&': \ + memcpy(outp, inp-ncopy, sizeof(*outp)*ncopy); \ + outp += ncopy; ncopy = 0; \ + *outp++ = '&'; \ + *outp++ = 'a'; \ + *outp++ = 'm'; \ + *outp++ = 'p'; \ + *outp++ = ';'; \ + break; \ + case '<': \ + memcpy(outp, inp-ncopy, sizeof(*outp)*ncopy); \ + outp += ncopy; ncopy = 0; \ + *outp++ = '&'; \ + *outp++ = 'l'; \ + *outp++ = 't'; \ + *outp++ = ';'; \ + break; \ + case '>': \ + memcpy(outp, inp-ncopy, sizeof(*outp)*ncopy); \ + outp += ncopy; ncopy = 0; \ + *outp++ = '&'; \ + *outp++ = 'g'; \ + *outp++ = 't'; \ + *outp++ = ';'; \ + break; \ + default: \ + ncopy++; \ + } \ + inp++; \ + } \ + memcpy(outp, inp-ncopy, sizeof(*outp)*ncopy); \ + } + +static PyObject* +escape_unicode_kind1(PyUnicodeObject *in) +{ + Py_UCS1 *inp = PyUnicode_1BYTE_DATA(in); + Py_UCS1 *inp_end = inp + PyUnicode_GET_LENGTH(in); + Py_UCS1 *outp; + PyObject *out; + Py_ssize_t delta = 0; + + GET_DELTA(inp, inp_end, delta); + if (!delta) { + Py_INCREF(in); + return (PyObject*)in; + } + + out = PyUnicode_New(PyUnicode_GET_LENGTH(in) + delta, + PyUnicode_IS_ASCII(in) ? 127 : 255); + if (!out) + return NULL; + + inp = PyUnicode_1BYTE_DATA(in); + outp = PyUnicode_1BYTE_DATA(out); + DO_ESCAPE(inp, inp_end, outp); + return out; +} + +static PyObject* +escape_unicode_kind2(PyUnicodeObject *in) +{ + Py_UCS2 *inp = PyUnicode_2BYTE_DATA(in); + Py_UCS2 *inp_end = inp + PyUnicode_GET_LENGTH(in); + Py_UCS2 *outp; + PyObject *out; + Py_ssize_t delta = 0; + + GET_DELTA(inp, inp_end, delta); + if (!delta) { + Py_INCREF(in); + return (PyObject*)in; + } + + out = PyUnicode_New(PyUnicode_GET_LENGTH(in) + delta, 65535); + if (!out) + return NULL; + + inp = PyUnicode_2BYTE_DATA(in); + outp = PyUnicode_2BYTE_DATA(out); + DO_ESCAPE(inp, inp_end, outp); + return out; +} + + +static PyObject* +escape_unicode_kind4(PyUnicodeObject *in) +{ + Py_UCS4 *inp = PyUnicode_4BYTE_DATA(in); + Py_UCS4 *inp_end = inp + PyUnicode_GET_LENGTH(in); + Py_UCS4 *outp; + PyObject *out; + Py_ssize_t delta = 0; + + GET_DELTA(inp, inp_end, delta); + if (!delta) { + Py_INCREF(in); + return (PyObject*)in; + } + + out = PyUnicode_New(PyUnicode_GET_LENGTH(in) + delta, 1114111); + if (!out) + return NULL; + + inp = PyUnicode_4BYTE_DATA(in); + outp = PyUnicode_4BYTE_DATA(out); + DO_ESCAPE(inp, inp_end, outp); + return out; +} + +static PyObject* +escape_unicode(PyUnicodeObject *in) +{ + if (PyUnicode_READY(in)) + return NULL; + + switch (PyUnicode_KIND(in)) { + case PyUnicode_1BYTE_KIND: + return escape_unicode_kind1(in); + case PyUnicode_2BYTE_KIND: + return escape_unicode_kind2(in); + case PyUnicode_4BYTE_KIND: + return escape_unicode_kind4(in); + } + assert(0); /* shouldn't happen */ + return NULL; +} +#endif /* PY_MAJOR_VERSION < 3 */ static PyObject* escape(PyObject *self, PyObject *text) { + static PyObject *id_html; PyObject *s = NULL, *rv = NULL, *html; + if (id_html == NULL) { +#if PY_MAJOR_VERSION < 3 + id_html = PyString_InternFromString("__html__"); +#else + id_html = PyUnicode_InternFromString("__html__"); +#endif + if (id_html == NULL) { + return NULL; + } + } + /* we don't have to escape integers, bools or floats */ if (PyLong_CheckExact(text) || #if PY_MAJOR_VERSION < 3 @@ -129,10 +307,16 @@ escape(PyObject *self, PyObject *text) return PyObject_CallFunctionObjArgs(markup, text, NULL); /* if the object has an __html__ method that performs the escaping */ - html = PyObject_GetAttrString(text, "__html__"); + html = PyObject_GetAttr(text ,id_html); if (html) { - rv = PyObject_CallObject(html, NULL); + s = PyObject_CallObject(html, NULL); Py_DECREF(html); + if (s == NULL) { + return NULL; + } + /* Convert to Markup object */ + rv = PyObject_CallFunctionObjArgs(markup, (PyObject*)s, NULL); + Py_DECREF(s); return rv; } diff --git a/pipenv/vendor/orderedmultidict/LICENSE.md b/pipenv/vendor/orderedmultidict/LICENSE.md new file mode 100644 index 00000000..210e8658 --- /dev/null +++ b/pipenv/vendor/orderedmultidict/LICENSE.md @@ -0,0 +1,31 @@ +Build Amazing Things. + +*** + +### Unlicense + +This is free and unencumbered software released into the public\ +domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or distribute\ +this software, either in source code form or as a compiled binary, for any\ +purpose, commercial or non-commercial, and by any means. + +In jurisdictions that recognize copyright laws, the author or authors of\ +this software dedicate any and all copyright interest in the software to the\ +public domain. We make this dedication for the benefit of the public at\ +large and to the detriment of our heirs and successors. We intend this\ +dedication to be an overt act of relinquishment in perpetuity of all\ +present and future rights to this software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF\ +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED\ +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A\ +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT\ +SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR\ +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT\ +OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION\ +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\ +SOFTWARE. + +For more information, please refer to diff --git a/pipenv/vendor/orderedmultidict/__init__.py b/pipenv/vendor/orderedmultidict/__init__.py new file mode 100755 index 00000000..4f0ce2f7 --- /dev/null +++ b/pipenv/vendor/orderedmultidict/__init__.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- + +# +# omdict - Ordered Multivalue Dictionary. +# +# Ansgar Grunseid +# grunseid.com +# grunseid@gmail.com +# +# License: Build Amazing Things (Unlicense) + +from __future__ import absolute_import + +from .orderedmultidict import * # noqa + +__title__ = 'orderedmultidict' +__version__ = '1.0' +__author__ = 'Ansgar Grunseid' +__contact__ = 'grunseid@gmail.com' +__license__ = 'Unlicense' +__url__ = 'https://github.com/gruns/orderedmultidict' diff --git a/pipenv/vendor/orderedmultidict/itemlist.py b/pipenv/vendor/orderedmultidict/itemlist.py new file mode 100755 index 00000000..e9e96c72 --- /dev/null +++ b/pipenv/vendor/orderedmultidict/itemlist.py @@ -0,0 +1,157 @@ +# -*- coding: utf-8 -*- + +# +# omdict - Ordered Multivalue Dictionary. +# +# Ansgar Grunseid +# grunseid.com +# grunseid@gmail.com +# +# License: Build Amazing Things (Unlicense) + +from __future__ import absolute_import + +from six.moves import zip_longest + +_absent = object() # Marker that means no parameter was provided. + + +class itemnode(object): + + """ + Dictionary key:value items wrapped in a node to be members of itemlist, the + doubly linked list defined below. + """ + + def __init__(self, prev=None, next=None, key=_absent, value=_absent): + self.prev = prev + self.next = next + self.key = key + self.value = value + + +class itemlist(object): + + """ + Doubly linked list of itemnodes. + + This class is used as the key:value item storage of orderedmultidict. + Methods below were only added as needed for use with orderedmultidict, so + some otherwise common list methods may be missing. + """ + + def __init__(self, items=[]): + self.root = itemnode() + self.root.next = self.root.prev = self.root + self.size = 0 + + for key, value in items: + self.append(key, value) + + def append(self, key, value): + tail = self.root.prev if self.root.prev is not self.root else self.root + node = itemnode(tail, self.root, key=key, value=value) + tail.next = node + self.root.prev = node + self.size += 1 + return node + + def removenode(self, node): + node.prev.next = node.next + node.next.prev = node.prev + self.size -= 1 + return self + + def clear(self): + for node, key, value in self: + self.removenode(node) + return self + + def items(self): + return list(self.iteritems()) + + def keys(self): + return list(self.iterkeys()) + + def values(self): + return list(self.itervalues()) + + def iteritems(self): + for node, key, value in self: + yield key, value + + def iterkeys(self): + for node, key, value in self: + yield key + + def itervalues(self): + for node, key, value in self: + yield value + + def reverse(self): + for node, key, value in self: + node.prev, node.next = node.next, node.prev + self.root.prev, self.root.next = self.root.next, self.root.prev + return self + + def __len__(self): + return self.size + + def __iter__(self): + current = self.root.next + while current and current is not self.root: + # Record current.next here in case current.next changes after the + # yield and before we return for the next iteration. For example, + # methods like reverse() will change current.next() before yield + # gets executed again. + nextnode = current.next + yield current, current.key, current.value + current = nextnode + + def __contains__(self, item): + """ + Params: + item: Can either be a (key,value) tuple or an itemnode reference. + """ + node = key = value = _absent + if hasattr(item, '__len__') and callable(item.__len__): + if len(item) == 2: + key, value = item + elif len(item) == 3: + node, key, value = item + else: + node = item + + if node is not _absent or _absent not in [key, value]: + for selfnode, selfkey, selfvalue in self: + if ((node is _absent and key == selfkey and value == selfvalue) + or (node is not _absent and node == selfnode)): + return True + return False + + def __getitem__(self, index): + # Only support direct access to the first or last element, as this is + # all orderedmultidict needs for now. + if index == 0 and self.root.next is not self.root: + return self.root.next + elif index == -1 and self.root.prev is not self.root: + return self.root.prev + raise IndexError(index) + + def __delitem__(self, index): + self.removenode(self[index]) + + def __eq__(self, other): + for (n1, key1, value1), (n2, key2, value2) in zip_longest(self, other): + if key1 != key2 or value1 != value2: + return False + return True + + def __ne__(self, other): + return not self.__eq__(other) + + def __nonzero__(self): + return self.size > 0 + + def __str__(self): + return '[%s]' % self.items() diff --git a/pipenv/vendor/orderedmultidict/orderedmultidict.py b/pipenv/vendor/orderedmultidict/orderedmultidict.py new file mode 100755 index 00000000..924dd8d2 --- /dev/null +++ b/pipenv/vendor/orderedmultidict/orderedmultidict.py @@ -0,0 +1,811 @@ +# -*- coding: utf-8 -*- + +# +# omdict - Ordered Multivalue Dictionary. +# +# Ansgar Grunseid +# grunseid.com +# grunseid@gmail.com +# +# License: Build Amazing Things (Unlicense) + +from __future__ import absolute_import + +from itertools import chain +from collections import MutableMapping + +import six +from six.moves import map, zip_longest + +from .itemlist import itemlist + +try: + from collections import OrderedDict as odict # Python 2.7 and later. +except ImportError: + from ordereddict import OrderedDict as odict # Python 2.6 and earlier. + +import sys +items_attr = 'items' if sys.version_info[0] >= 3 else 'iteritems' + +_absent = object() # Marker that means no parameter was provided. + + +def callable_attr(obj, attr): + return hasattr(obj, attr) and callable(getattr(obj, attr)) + + +# +# TODO(grun): Create a subclass of list that values(), getlist(), allitems(), +# etc return that the user can manipulate directly to control the omdict() +# object. +# +# For example, users should be able to do things like +# +# omd = omdict([(1,1), (1,11)]) +# omd.values(1).append('sup') +# omd.allitems() == [(1,1), (1,11), (1,'sup')] +# omd.values(1).remove(11) +# omd.allitems() == [(1,1), (1,'sup')] +# omd.values(1).extend(['two', 'more']) +# omd.allitems() == [(1,1), (1,'sup'), (1,'two'), (1,'more')] +# +# or +# +# omd = omdict([(1,1), (1,11)]) +# omd.allitems().extend([(2,2), (2,22)]) +# omd.allitems() == [(1,1), (1,11), (2,2), (2,22)]) +# +# or +# +# omd = omdict() +# omd.values(1) = [1, 11] +# omd.allitems() == [(1,1), (1,11)] +# omd.values(1) = list(map(lambda i: i * -10, omd.values(1))) +# omd.allitems() == [(1,-10), (1,-110)] +# omd.allitems() = filter(lambda (k,v): v > -100, omd.allitems()) +# omd.allitems() == [(1,-10)] +# +# etc. +# +# To accomplish this, subclass list in such a manner that each list element is +# really a two tuple, where the first tuple value is the actual value and the +# second tuple value is a reference to the itemlist node for that value. Users +# only interact with the first tuple values, the actual values, but behind the +# scenes when an element is modified, deleted, inserted, etc, the according +# itemlist nodes are modified, deleted, inserted, etc accordingly. In this +# manner, users can manipulate omdict objects directly through direct list +# manipulation. +# +# Once accomplished, some methods become redundant and should be removed in +# favor of the more intuitive direct value list manipulation. Such redundant +# methods include getlist() (removed in favor of values()?), addlist(), and +# setlist(). +# +# With the removal of many of the 'list' methods, think about renaming all +# remaining 'list' methods to 'values' methods, like poplist() -> popvalues(), +# poplistitem() -> popvaluesitem(), etc. This would be an easy switch for most +# methods, but wouldn't fit others so well. For example, iterlists() would +# become itervalues(), a name extremely similar to iterallvalues() but quite +# different in function. +# + + +class omdict(MutableMapping): + + """ + Ordered Multivalue Dictionary. + + A multivalue dictionary is a dictionary that can store multiple values per + key. An ordered multivalue dictionary is a multivalue dictionary that + retains the order of insertions and deletions. + + Internally, items are stored in a doubly linked list, self._items. A + dictionary, self._map, is also maintained and stores an ordered list of + linked list node references, one for each value associated with that key. + + Standard dict methods interact with the first value associated with a given + key. This means that omdict retains method parity with dict, and a dict + object can be replaced with an omdict object and all interaction will + behave identically. All dict methods that retain parity with omdict are: + + get(), setdefault(), pop(), popitem(), + clear(), copy(), update(), fromkeys(), len() + __getitem__(), __setitem__(), __delitem__(), __contains__(), + items(), keys(), values(), iteritems(), iterkeys(), itervalues(), + + Optional parameters have been added to some dict methods, but because the + added parameters are optional, existing use remains unaffected. An optional + parameter has been added to these methods: + + items(), values(), iteritems(), itervalues() + + New methods have also been added to omdict. Methods with 'list' in their + name interact with lists of values, and methods with 'all' in their name + interact with all items in the dictionary, including multiple items with + the same key. + + The new omdict methods are: + + load(), size(), reverse(), + getlist(), add(), addlist(), set(), setlist(), setdefaultlist(), + poplist(), popvalue(), popvalues(), popitem(), poplistitem(), + allitems(), allkeys(), allvalues(), lists(), listitems(), + iterallitems(), iterallkeys(), iterallvalues(), iterlists(), + iterlistitems() + + Explanations and examples of the new methods above can be found in the + function comments below and online at + + https://github.com/gruns/orderedmultidict + + Additional omdict information and documentation can also be found at the + above url. + """ + + def __init__(self, *args, **kwargs): + # Doubly linked list of itemnodes. Each itemnode stores a key:value + # item. + self._items = itemlist() + + # Ordered dictionary of keys and itemnode references. Each itemnode + # reference points to one of that keys values. + self._map = odict() + + self.load(*args, **kwargs) + + def load(self, *args, **kwargs): + """ + Clear all existing key:value items and import all key:value items from + . If multiple values exist for the same key in , they + are all be imported. + + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.load([(4,4), (4,44), (5,5)]) + omd.allitems() == [(4,4), (4,44), (5,5)] + + Returns: . + """ + self.clear() + self.updateall(*args, **kwargs) + return self + + def copy(self): + return self.__class__(self.allitems()) + + def clear(self): + self._map.clear() + self._items.clear() + + def size(self): + """ + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.size() == 5 + + Returns: Total number of items, including multiple items with the same + key. + """ + return len(self._items) + + @classmethod + def fromkeys(cls, iterable, value=None): + return cls([(key, value) for key in iterable]) + + def has_key(self, key): + return key in self + + def update(self, *args, **kwargs): + self._update_updateall(True, *args, **kwargs) + + def updateall(self, *args, **kwargs): + """ + Update this dictionary with the items from , replacing + existing key:value items with shared keys before adding new key:value + items. + + Example: + omd = omdict([(1,1), (2,2)]) + omd.updateall([(2,'two'), (1,'one'), (2,222), (1,111)]) + omd.allitems() == [(1, 'one'), (2, 'two'), (2, 222), (1, 111)] + + Returns: . + """ + self._update_updateall(False, *args, **kwargs) + return self + + def _update_updateall(self, replace_at_most_one, *args, **kwargs): + # Bin the items in and into or + # . Items in are new values to replace old + # values for a given key, and items in are new items to be + # added. + replacements, leftovers = dict(), [] + for mapping in chain(args, [kwargs]): + self._bin_update_items( + self._items_iterator(mapping), replace_at_most_one, + replacements, leftovers) + + # First, replace existing values for each key. + for key, values in six.iteritems(replacements): + self.setlist(key, values) + # Then, add the leftover items to the end of the list of all items. + for key, value in leftovers: + self.add(key, value) + + def _bin_update_items(self, items, replace_at_most_one, + replacements, leftovers): + """ + are modified directly, ala pass by + reference. + """ + for key, value in items: + # If there are existing items with key that have yet to be + # marked for replacement, mark that item's value to be replaced by + # by appending it to . + if key in self and key not in replacements: + replacements[key] = [value] + elif (key in self and not replace_at_most_one and + len(replacements[key]) < len(self.values(key))): + replacements[key].append(value) + else: + if replace_at_most_one: + replacements[key] = [value] + else: + leftovers.append((key, value)) + + def _items_iterator(self, container): + cont = container + iterator = iter(cont) + if callable_attr(cont, 'iterallitems'): + iterator = cont.iterallitems() + elif callable_attr(cont, 'allitems'): + iterator = iter(cont.allitems()) + elif callable_attr(cont, 'iteritems'): + iterator = cont.iteritems() + elif callable_attr(cont, 'items'): + iterator = iter(cont.items()) + return iterator + + def get(self, key, default=None): + if key in self: + return self._map[key][0].value + return default + + def getlist(self, key, default=[]): + """ + Returns: The list of values for if is in the dictionary, + else . If is not provided, an empty list is + returned. + """ + if key in self: + return [node.value for node in self._map[key]] + return default + + def setdefault(self, key, default=None): + if key in self: + return self[key] + self.add(key, default) + return default + + def setdefaultlist(self, key, defaultlist=[None]): + """ + Similar to setdefault() except is a list of values to set + for . If already exists, its existing list of values is + returned. + + If isn't a key and is an empty list, [], no values + are added for and will not be added as a key. + + Returns: List of 's values if exists in the dictionary, + otherwise . + """ + if key in self: + return self.getlist(key) + self.addlist(key, defaultlist) + return defaultlist + + def add(self, key, value=None): + """ + Add to the list of values for . If is not in the + dictionary, then is added as the sole value for . + + Example: + omd = omdict() + omd.add(1, 1) # omd.allitems() == [(1,1)] + omd.add(1, 11) # omd.allitems() == [(1,1), (1,11)] + omd.add(2, 2) # omd.allitems() == [(1,1), (1,11), (2,2)] + + Returns: . + """ + self._map.setdefault(key, []) + node = self._items.append(key, value) + self._map[key].append(node) + return self + + def addlist(self, key, valuelist=[]): + """ + Add the values in to the list of values for . If + is not in the dictionary, the values in become the values + for . + + Example: + omd = omdict([(1,1)]) + omd.addlist(1, [11, 111]) + omd.allitems() == [(1, 1), (1, 11), (1, 111)] + omd.addlist(2, [2]) + omd.allitems() == [(1, 1), (1, 11), (1, 111), (2, 2)] + + Returns: . + """ + for value in valuelist: + self.add(key, value) + return self + + def set(self, key, value=None): + """ + Sets 's value to . Identical in function to __setitem__(). + + Returns: . + """ + self[key] = value + return self + + def setlist(self, key, values): + """ + Sets 's list of values to . Existing items with key + are first replaced with new values from . Any remaining old + items that haven't been replaced with new values are deleted, and any + new values from that don't have corresponding items with + to replace are appended to the end of the list of all items. + + If values is an empty list, [], is deleted, equivalent in action + to del self[]. + + Example: + omd = omdict([(1,1), (2,2)]) + omd.setlist(1, [11, 111]) + omd.allitems() == [(1,11), (2,2), (1,111)] + + omd = omdict([(1,1), (1,11), (2,2), (1,111)]) + omd.setlist(1, [None]) + omd.allitems() == [(1,None), (2,2)] + + omd = omdict([(1,1), (1,11), (2,2), (1,111)]) + omd.setlist(1, []) + omd.allitems() == [(2,2)] + + Returns: . + """ + if not values and key in self: + self.pop(key) + else: + it = zip_longest( + list(self._map.get(key, [])), values, fillvalue=_absent) + for node, value in it: + if node is not _absent and value is not _absent: + node.value = value + elif node is _absent: + self.add(key, value) + elif value is _absent: + self._map[key].remove(node) + self._items.removenode(node) + return self + + def removevalues(self, key, values): + """ + Removes all from the values of . If has no + remaining values after removevalues(), the key is popped. + + Example: + omd = omdict([(1, 1), (1, 11), (1, 1), (1, 111)]) + omd.removevalues(1, [1, 111]) + omd.allitems() == [(1, 11)] + + Returns: . + """ + self.setlist(key, [v for v in self.getlist(key) if v not in values]) + return self + + def pop(self, key, default=_absent): + if key in self: + return self.poplist(key)[0] + elif key not in self._map and default is not _absent: + return default + raise KeyError(key) + + def poplist(self, key, default=_absent): + """ + If is in the dictionary, pop it and return its list of values. If + is not in the dictionary, return . KeyError is raised if + is not provided and is not in the dictionary. + + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.poplist(1) == [1, 11, 111] + omd.allitems() == [(2,2), (3,3)] + omd.poplist(2) == [2] + omd.allitems() == [(3,3)] + + Raises: KeyError if isn't in the dictionary and isn't + provided. + Returns: List of 's values. + """ + if key in self: + values = self.getlist(key) + del self._map[key] + for node, nodekey, nodevalue in self._items: + if nodekey == key: + self._items.removenode(node) + return values + elif key not in self._map and default is not _absent: + return default + raise KeyError(key) + + def popvalue(self, key, value=_absent, default=_absent, last=True): + """ + If is provided, pops the first or last (key,value) item in the + dictionary if is in the dictionary. + + If is not provided, pops the first or last value for if + is in the dictionary. + + If no longer has any values after a popvalue() call, is + removed from the dictionary. If isn't in the dictionary and + was provided, return default. KeyError is raised if + is not provided and is not in the dictionary. ValueError is + raised if is provided but isn't a value for . + + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3), (2,22)]) + omd.popvalue(1) == 111 + omd.allitems() == [(1,11), (1,111), (2,2), (3,3), (2,22)] + omd.popvalue(1, last=False) == 1 + omd.allitems() == [(1,11), (2,2), (3,3), (2,22)] + omd.popvalue(2, 2) == 2 + omd.allitems() == [(1,11), (3,3), (2,22)] + omd.popvalue(1, 11) == 11 + omd.allitems() == [(3,3), (2,22)] + omd.popvalue('not a key', default='sup') == 'sup' + + Params: + last: Boolean whether to return 's first value ( is False) + or last value ( is True). + Raises: + KeyError if isn't in the dictionary and isn't + provided. + ValueError if isn't a value for . + Returns: The first or last of 's values. + """ + def pop_node_with_index(key, index): + node = self._map[key].pop(index) + if not self._map[key]: + del self._map[key] + self._items.removenode(node) + return node + + if key in self: + if value is not _absent: + if last: + pos = self.values(key)[::-1].index(value) + else: + pos = self.values(key).index(value) + if pos == -1: + raise ValueError(value) + else: + index = (len(self.values(key)) - 1 - pos) if last else pos + return pop_node_with_index(key, index).value + else: + return pop_node_with_index(key, -1 if last else 0).value + elif key not in self._map and default is not _absent: + return default + raise KeyError(key) + + def popitem(self, fromall=False, last=True): + """ + Pop and return a key:value item. + + If is False, items()[0] is popped if is False or + items()[-1] is popped if is True. All remaining items with the + same key are removed. + + If is True, allitems()[0] is popped if is False or + allitems()[-1] is popped if is True. Any remaining items with + the same key remain. + + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.popitem() == (3,3) + omd.popitem(fromall=False, last=False) == (1,1) + omd.popitem(fromall=False, last=False) == (2,2) + + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.popitem(fromall=True, last=False) == (1,1) + omd.popitem(fromall=True, last=False) == (1,11) + omd.popitem(fromall=True, last=True) == (3,3) + omd.popitem(fromall=True, last=False) == (1,111) + + Params: + fromall: Whether to pop an item from items() ( is True) or + allitems() ( is False). + last: Boolean whether to pop the first item or last item of items() + or allitems(). + Raises: KeyError if the dictionary is empty. + Returns: The first or last item from item() or allitem(). + """ + if not self._items: + raise KeyError('popitem(): %s is empty' % self.__class__.__name__) + + if fromall: + node = self._items[-1 if last else 0] + key = node.key + return key, self.popvalue(key, last=last) + else: + key = list(self._map.keys())[-1 if last else 0] + return key, self.pop(key) + + def poplistitem(self, last=True): + """ + Pop and return a key:valuelist item comprised of a key and that key's + list of values. If is False, a key:valuelist item comprised of + keys()[0] and its list of values is popped and returned. If is + True, a key:valuelist item comprised of keys()[-1] and its list of + values is popped and returned. + + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.poplistitem(last=True) == (3,[3]) + omd.poplistitem(last=False) == (1,[1,11,111]) + + Params: + last: Boolean whether to pop the first or last key and its associated + list of values. + Raises: KeyError if the dictionary is empty. + Returns: A two-tuple comprised of the first or last key and its + associated list of values. + """ + if not self._items: + s = 'poplistitem(): %s is empty' % self.__class__.__name__ + raise KeyError(s) + + key = self.keys()[-1 if last else 0] + return key, self.poplist(key) + + def items(self, key=_absent): + """ + Raises: KeyError if is provided and not in the dictionary. + Returns: List created from iteritems(). Only items with key + are returned if is provided and is a dictionary key. + """ + return list(self.iteritems(key)) + + def keys(self): + return list(self.iterkeys()) + + def values(self, key=_absent): + """ + Raises: KeyError if is provided and not in the dictionary. + Returns: List created from itervalues().If is provided and + is a dictionary key, only values of items with key are + returned. + """ + if key is not _absent and key in self._map: + return self.getlist(key) + return list(self.itervalues()) + + def lists(self): + """ + Returns: List created from iterlists(). + """ + return list(self.iterlists()) + + def listitems(self): + """ + Returns: List created from iterlistitems(). + """ + return list(self.iterlistitems()) + + def iteritems(self, key=_absent): + """ + Parity with dict.iteritems() except the optional parameter has + been added. If is provided, only items with the provided key are + iterated over. KeyError is raised if is provided and not in the + dictionary. + + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.iteritems(1) -> (1,1) -> (1,11) -> (1,111) + omd.iteritems() -> (1,1) -> (2,2) -> (3,3) + + Raises: KeyError if is provided and not in the dictionary. + Returns: An iterator over the items() of the dictionary, or only items + with the key if is provided. + """ + if key is not _absent: + if key in self: + items = [(node.key, node.value) for node in self._map[key]] + return iter(items) + raise KeyError(key) + items = six.iteritems(self._map) + return iter((key, nodes[0].value) for (key, nodes) in items) + + def iterkeys(self): + return six.iterkeys(self._map) + + def itervalues(self, key=_absent): + """ + Parity with dict.itervalues() except the optional parameter has + been added. If is provided, only values from items with the + provided key are iterated over. KeyError is raised if is provided + and not in the dictionary. + + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.itervalues(1) -> 1 -> 11 -> 111 + omd.itervalues() -> 1 -> 11 -> 111 -> 2 -> 3 + + Raises: KeyError if is provided and isn't in the dictionary. + Returns: An iterator over the values() of the dictionary, or only the + values of key if is provided. + """ + if key is not _absent: + if key in self: + return iter([node.value for node in self._map[key]]) + raise KeyError(key) + return iter([nodes[0].value for nodes in six.itervalues(self._map)]) + + def allitems(self, key=_absent): + ''' + Raises: KeyError if is provided and not in the dictionary. + Returns: List created from iterallitems(). + ''' + return list(self.iterallitems(key)) + + def allkeys(self): + ''' + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.allkeys() == [1,1,1,2,3] + + Returns: List created from iterallkeys(). + ''' + return list(self.iterallkeys()) + + def allvalues(self, key=_absent): + ''' + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.allvalues() == [1,11,111,2,3] + omd.allvalues(1) == [1,11,111] + + Raises: KeyError if is provided and not in the dictionary. + Returns: List created from iterallvalues(). + ''' + return list(self.iterallvalues(key)) + + def iterallitems(self, key=_absent): + ''' + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.iterallitems() == (1,1) -> (1,11) -> (1,111) -> (2,2) -> (3,3) + omd.iterallitems(1) == (1,1) -> (1,11) -> (1,111) + + Raises: KeyError if is provided and not in the dictionary. + Returns: An iterator over every item in the diciontary. If is + provided, only items with the key are iterated over. + ''' + if key is not _absent: + # Raises KeyError if is not in self._map. + return self.iteritems(key) + return self._items.iteritems() + + def iterallkeys(self): + ''' + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.iterallkeys() == 1 -> 1 -> 1 -> 2 -> 3 + + Returns: An iterator over the keys of every item in the dictionary. + ''' + return self._items.iterkeys() + + def iterallvalues(self, key=_absent): + ''' + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.iterallvalues() == 1 -> 11 -> 111 -> 2 -> 3 + + Returns: An iterator over the values of every item in the dictionary. + ''' + if key is not _absent: + if key in self: + return iter(self.getlist(key)) + raise KeyError(key) + return self._items.itervalues() + + def iterlists(self): + ''' + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.iterlists() -> [1,11,111] -> [2] -> [3] + + Returns: An iterator over the list comprised of the lists of values for + each key. + ''' + return map(lambda key: self.getlist(key), self) + + def iterlistitems(self): + """ + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.iterlistitems() -> (1,[1,11,111]) -> (2,[2]) -> (3,[3]) + + Returns: An iterator over the list of key:valuelist items. + """ + return map(lambda key: (key, self.getlist(key)), self) + + def reverse(self): + """ + Reverse the order of all items in the dictionary. + + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.reverse() + omd.allitems() == [(3,3), (2,2), (1,111), (1,11), (1,1)] + + Returns: . + """ + for key in six.iterkeys(self._map): + self._map[key].reverse() + self._items.reverse() + return self + + def __eq__(self, other): + if callable_attr(other, 'iterallitems'): + myiter, otheriter = self.iterallitems(), other.iterallitems() + for i1, i2 in zip_longest(myiter, otheriter, fillvalue=_absent): + if i1 != i2 or i1 is _absent or i2 is _absent: + return False + elif not hasattr(other, '__len__') or not hasattr(other, items_attr): + return False + # Ignore order so we can compare ordered omdicts with unordered dicts. + else: + if len(self) != len(other): + return False + for key, value in six.iteritems(other): + if self.get(key, _absent) != value: + return False + return True + + def __ne__(self, other): + return not self.__eq__(other) + + def __len__(self): + return len(self._map) + + def __iter__(self): + for key in self.iterkeys(): + yield key + + def __contains__(self, key): + return key in self._map + + def __getitem__(self, key): + if key in self: + return self.get(key) + raise KeyError(key) + + def __setitem__(self, key, value): + self.setlist(key, [value]) + + def __delitem__(self, key): + return self.pop(key) + + def __nonzero__(self): + return bool(self._map) + + def __str__(self): + return '{%s}' % ', '.join( + map(lambda p: '%r: %r' % (p[0], p[1]), self.iterallitems())) + + def __repr__(self): + return '%s(%s)' % (self.__class__.__name__, self.allitems()) diff --git a/pipenv/vendor/packaging/__about__.py b/pipenv/vendor/packaging/__about__.py index 21fc6ce3..7481c9e2 100644 --- a/pipenv/vendor/packaging/__about__.py +++ b/pipenv/vendor/packaging/__about__.py @@ -4,18 +4,24 @@ from __future__ import absolute_import, division, print_function __all__ = [ - "__title__", "__summary__", "__uri__", "__version__", "__author__", - "__email__", "__license__", "__copyright__", + "__title__", + "__summary__", + "__uri__", + "__version__", + "__author__", + "__email__", + "__license__", + "__copyright__", ] __title__ = "packaging" __summary__ = "Core utilities for Python packages" __uri__ = "https://github.com/pypa/packaging" -__version__ = "18.0" +__version__ = "19.0" __author__ = "Donald Stufft and individual contributors" __email__ = "donald@stufft.io" __license__ = "BSD or Apache License, Version 2.0" -__copyright__ = "Copyright 2014-2018 %s" % __author__ +__copyright__ = "Copyright 2014-2019 %s" % __author__ diff --git a/pipenv/vendor/packaging/__init__.py b/pipenv/vendor/packaging/__init__.py index 5ee62202..a0cf67df 100644 --- a/pipenv/vendor/packaging/__init__.py +++ b/pipenv/vendor/packaging/__init__.py @@ -4,11 +4,23 @@ from __future__ import absolute_import, division, print_function from .__about__ import ( - __author__, __copyright__, __email__, __license__, __summary__, __title__, - __uri__, __version__ + __author__, + __copyright__, + __email__, + __license__, + __summary__, + __title__, + __uri__, + __version__, ) __all__ = [ - "__title__", "__summary__", "__uri__", "__version__", "__author__", - "__email__", "__license__", "__copyright__", + "__title__", + "__summary__", + "__uri__", + "__version__", + "__author__", + "__email__", + "__license__", + "__copyright__", ] diff --git a/pipenv/vendor/packaging/_compat.py b/pipenv/vendor/packaging/_compat.py index 210bb80b..25da473c 100644 --- a/pipenv/vendor/packaging/_compat.py +++ b/pipenv/vendor/packaging/_compat.py @@ -12,9 +12,9 @@ PY3 = sys.version_info[0] == 3 # flake8: noqa if PY3: - string_types = str, + string_types = (str,) else: - string_types = basestring, + string_types = (basestring,) def with_metaclass(meta, *bases): @@ -27,4 +27,5 @@ def with_metaclass(meta, *bases): class metaclass(meta): def __new__(cls, name, this_bases, d): return meta(name, bases, d) - return type.__new__(metaclass, 'temporary_class', (), {}) + + return type.__new__(metaclass, "temporary_class", (), {}) diff --git a/pipenv/vendor/packaging/_structures.py b/pipenv/vendor/packaging/_structures.py index e9fc4a04..68dcca63 100644 --- a/pipenv/vendor/packaging/_structures.py +++ b/pipenv/vendor/packaging/_structures.py @@ -5,7 +5,6 @@ from __future__ import absolute_import, division, print_function class Infinity(object): - def __repr__(self): return "Infinity" @@ -38,7 +37,6 @@ Infinity = Infinity() class NegativeInfinity(object): - def __repr__(self): return "-Infinity" diff --git a/pipenv/vendor/packaging/markers.py b/pipenv/vendor/packaging/markers.py index 5fdf510c..eff5abbb 100644 --- a/pipenv/vendor/packaging/markers.py +++ b/pipenv/vendor/packaging/markers.py @@ -17,8 +17,11 @@ from .specifiers import Specifier, InvalidSpecifier __all__ = [ - "InvalidMarker", "UndefinedComparison", "UndefinedEnvironmentName", - "Marker", "default_environment", + "InvalidMarker", + "UndefinedComparison", + "UndefinedEnvironmentName", + "Marker", + "default_environment", ] @@ -42,7 +45,6 @@ class UndefinedEnvironmentName(ValueError): class Node(object): - def __init__(self, value): self.value = value @@ -57,62 +59,52 @@ class Node(object): class Variable(Node): - def serialize(self): return str(self) class Value(Node): - def serialize(self): return '"{0}"'.format(self) class Op(Node): - def serialize(self): return str(self) VARIABLE = ( - L("implementation_version") | - L("platform_python_implementation") | - L("implementation_name") | - L("python_full_version") | - L("platform_release") | - L("platform_version") | - L("platform_machine") | - L("platform_system") | - L("python_version") | - L("sys_platform") | - L("os_name") | - L("os.name") | # PEP-345 - L("sys.platform") | # PEP-345 - L("platform.version") | # PEP-345 - L("platform.machine") | # PEP-345 - L("platform.python_implementation") | # PEP-345 - L("python_implementation") | # undocumented setuptools legacy - L("extra") + L("implementation_version") + | L("platform_python_implementation") + | L("implementation_name") + | L("python_full_version") + | L("platform_release") + | L("platform_version") + | L("platform_machine") + | L("platform_system") + | L("python_version") + | L("sys_platform") + | L("os_name") + | L("os.name") + | L("sys.platform") # PEP-345 + | L("platform.version") # PEP-345 + | L("platform.machine") # PEP-345 + | L("platform.python_implementation") # PEP-345 + | L("python_implementation") # PEP-345 + | L("extra") # undocumented setuptools legacy ) ALIASES = { - 'os.name': 'os_name', - 'sys.platform': 'sys_platform', - 'platform.version': 'platform_version', - 'platform.machine': 'platform_machine', - 'platform.python_implementation': 'platform_python_implementation', - 'python_implementation': 'platform_python_implementation' + "os.name": "os_name", + "sys.platform": "sys_platform", + "platform.version": "platform_version", + "platform.machine": "platform_machine", + "platform.python_implementation": "platform_python_implementation", + "python_implementation": "platform_python_implementation", } VARIABLE.setParseAction(lambda s, l, t: Variable(ALIASES.get(t[0], t[0]))) VERSION_CMP = ( - L("===") | - L("==") | - L(">=") | - L("<=") | - L("!=") | - L("~=") | - L(">") | - L("<") + L("===") | L("==") | L(">=") | L("<=") | L("!=") | L("~=") | L(">") | L("<") ) MARKER_OP = VERSION_CMP | L("not in") | L("in") @@ -152,8 +144,11 @@ def _format_marker(marker, first=True): # where the single item is itself it's own list. In that case we want skip # the rest of this function so that we don't get extraneous () on the # outside. - if (isinstance(marker, list) and len(marker) == 1 and - isinstance(marker[0], (list, tuple))): + if ( + isinstance(marker, list) + and len(marker) == 1 + and isinstance(marker[0], (list, tuple)) + ): return _format_marker(marker[0]) if isinstance(marker, list): @@ -239,20 +234,20 @@ def _evaluate_markers(markers, environment): def format_full_version(info): - version = '{0.major}.{0.minor}.{0.micro}'.format(info) + version = "{0.major}.{0.minor}.{0.micro}".format(info) kind = info.releaselevel - if kind != 'final': + if kind != "final": version += kind[0] + str(info.serial) return version def default_environment(): - if hasattr(sys, 'implementation'): + if hasattr(sys, "implementation"): iver = format_full_version(sys.implementation.version) implementation_name = sys.implementation.name else: - iver = '0' - implementation_name = '' + iver = "0" + implementation_name = "" return { "implementation_name": implementation_name, @@ -270,13 +265,13 @@ def default_environment(): class Marker(object): - def __init__(self, marker): try: self._markers = _coerce_parse_result(MARKER.parseString(marker)) except ParseException as e: err_str = "Invalid marker: {0!r}, parse error at {1!r}".format( - marker, marker[e.loc:e.loc + 8]) + marker, marker[e.loc : e.loc + 8] + ) raise InvalidMarker(err_str) def __str__(self): diff --git a/pipenv/vendor/packaging/requirements.py b/pipenv/vendor/packaging/requirements.py index e8008a6d..4d9688b9 100644 --- a/pipenv/vendor/packaging/requirements.py +++ b/pipenv/vendor/packaging/requirements.py @@ -38,8 +38,8 @@ IDENTIFIER = Combine(ALPHANUM + ZeroOrMore(IDENTIFIER_END)) NAME = IDENTIFIER("name") EXTRA = IDENTIFIER -URI = Regex(r'[^ ]+')("url") -URL = (AT + URI) +URI = Regex(r"[^ ]+")("url") +URL = AT + URI EXTRAS_LIST = EXTRA + ZeroOrMore(COMMA + EXTRA) EXTRAS = (LBRACKET + Optional(EXTRAS_LIST) + RBRACKET)("extras") @@ -48,17 +48,18 @@ VERSION_PEP440 = Regex(Specifier._regex_str, re.VERBOSE | re.IGNORECASE) VERSION_LEGACY = Regex(LegacySpecifier._regex_str, re.VERBOSE | re.IGNORECASE) VERSION_ONE = VERSION_PEP440 ^ VERSION_LEGACY -VERSION_MANY = Combine(VERSION_ONE + ZeroOrMore(COMMA + VERSION_ONE), - joinString=",", adjacent=False)("_raw_spec") +VERSION_MANY = Combine( + VERSION_ONE + ZeroOrMore(COMMA + VERSION_ONE), joinString=",", adjacent=False +)("_raw_spec") _VERSION_SPEC = Optional(((LPAREN + VERSION_MANY + RPAREN) | VERSION_MANY)) -_VERSION_SPEC.setParseAction(lambda s, l, t: t._raw_spec or '') +_VERSION_SPEC.setParseAction(lambda s, l, t: t._raw_spec or "") VERSION_SPEC = originalTextFor(_VERSION_SPEC)("specifier") VERSION_SPEC.setParseAction(lambda s, l, t: t[1]) MARKER_EXPR = originalTextFor(MARKER_EXPR())("marker") MARKER_EXPR.setParseAction( - lambda s, l, t: Marker(s[t._original_start:t._original_end]) + lambda s, l, t: Marker(s[t._original_start : t._original_end]) ) MARKER_SEPARATOR = SEMICOLON MARKER = MARKER_SEPARATOR + MARKER_EXPR @@ -66,8 +67,7 @@ MARKER = MARKER_SEPARATOR + MARKER_EXPR VERSION_AND_MARKER = VERSION_SPEC + Optional(MARKER) URL_AND_MARKER = URL + Optional(MARKER) -NAMED_REQUIREMENT = \ - NAME + Optional(EXTRAS) + (URL_AND_MARKER | VERSION_AND_MARKER) +NAMED_REQUIREMENT = NAME + Optional(EXTRAS) + (URL_AND_MARKER | VERSION_AND_MARKER) REQUIREMENT = stringStart + NAMED_REQUIREMENT + stringEnd # pyparsing isn't thread safe during initialization, so we do it eagerly, see @@ -92,15 +92,21 @@ class Requirement(object): try: req = REQUIREMENT.parseString(requirement_string) except ParseException as e: - raise InvalidRequirement("Parse error at \"{0!r}\": {1}".format( - requirement_string[e.loc:e.loc + 8], e.msg - )) + raise InvalidRequirement( + 'Parse error at "{0!r}": {1}'.format( + requirement_string[e.loc : e.loc + 8], e.msg + ) + ) self.name = req.name if req.url: parsed_url = urlparse.urlparse(req.url) - if not (parsed_url.scheme and parsed_url.netloc) or ( - not parsed_url.scheme and not parsed_url.netloc): + if parsed_url.scheme == "file": + if urlparse.urlunparse(parsed_url) != req.url: + raise InvalidRequirement("Invalid URL given") + elif not (parsed_url.scheme and parsed_url.netloc) or ( + not parsed_url.scheme and not parsed_url.netloc + ): raise InvalidRequirement("Invalid URL: {0}".format(req.url)) self.url = req.url else: @@ -120,6 +126,8 @@ class Requirement(object): if self.url: parts.append("@ {0}".format(self.url)) + if self.marker: + parts.append(" ") if self.marker: parts.append("; {0}".format(self.marker)) diff --git a/pipenv/vendor/packaging/specifiers.py b/pipenv/vendor/packaging/specifiers.py index 4c798999..743576a0 100644 --- a/pipenv/vendor/packaging/specifiers.py +++ b/pipenv/vendor/packaging/specifiers.py @@ -19,7 +19,6 @@ class InvalidSpecifier(ValueError): class BaseSpecifier(with_metaclass(abc.ABCMeta, object)): - @abc.abstractmethod def __str__(self): """ @@ -84,10 +83,7 @@ class _IndividualSpecifier(BaseSpecifier): if not match: raise InvalidSpecifier("Invalid specifier: '{0}'".format(spec)) - self._spec = ( - match.group("operator").strip(), - match.group("version").strip(), - ) + self._spec = (match.group("operator").strip(), match.group("version").strip()) # Store whether or not this Specifier should accept prereleases self._prereleases = prereleases @@ -99,11 +95,7 @@ class _IndividualSpecifier(BaseSpecifier): else "" ) - return "<{0}({1!r}{2})>".format( - self.__class__.__name__, - str(self), - pre, - ) + return "<{0}({1!r}{2})>".format(self.__class__.__name__, str(self), pre) def __str__(self): return "{0}{1}".format(*self._spec) @@ -194,8 +186,9 @@ class _IndividualSpecifier(BaseSpecifier): # If our version is a prerelease, and we were not set to allow # prereleases, then we'll store it for later incase nothing # else matches this specifier. - if (parsed_version.is_prerelease and not - (prereleases or self.prereleases)): + if parsed_version.is_prerelease and not ( + prereleases or self.prereleases + ): found_prereleases.append(version) # Either this is not a prerelease, or we should have been # accepting prereleases from the beginning. @@ -213,8 +206,7 @@ class _IndividualSpecifier(BaseSpecifier): class LegacySpecifier(_IndividualSpecifier): - _regex_str = ( - r""" + _regex_str = r""" (?P(==|!=|<=|>=|<|>)) \s* (?P @@ -225,10 +217,8 @@ class LegacySpecifier(_IndividualSpecifier): # them, and a comma since it's a version separator. ) """ - ) - _regex = re.compile( - r"^\s*" + _regex_str + r"\s*$", re.VERBOSE | re.IGNORECASE) + _regex = re.compile(r"^\s*" + _regex_str + r"\s*$", re.VERBOSE | re.IGNORECASE) _operators = { "==": "equal", @@ -269,13 +259,13 @@ def _require_version_compare(fn): if not isinstance(prospective, Version): return False return fn(self, prospective, spec) + return wrapped class Specifier(_IndividualSpecifier): - _regex_str = ( - r""" + _regex_str = r""" (?P(~=|==|!=|<=|>=|<|>|===)) (?P (?: @@ -367,10 +357,8 @@ class Specifier(_IndividualSpecifier): ) ) """ - ) - _regex = re.compile( - r"^\s*" + _regex_str + r"\s*$", re.VERBOSE | re.IGNORECASE) + _regex = re.compile(r"^\s*" + _regex_str + r"\s*$", re.VERBOSE | re.IGNORECASE) _operators = { "~=": "compatible", @@ -397,8 +385,7 @@ class Specifier(_IndividualSpecifier): prefix = ".".join( list( itertools.takewhile( - lambda x: (not x.startswith("post") and not - x.startswith("dev")), + lambda x: (not x.startswith("post") and not x.startswith("dev")), _version_split(spec), ) )[:-1] @@ -407,8 +394,9 @@ class Specifier(_IndividualSpecifier): # Add the prefix notation to the end of our string prefix += ".*" - return (self._get_operator(">=")(prospective, spec) and - self._get_operator("==")(prospective, prefix)) + return self._get_operator(">=")(prospective, spec) and self._get_operator("==")( + prospective, prefix + ) @_require_version_compare def _compare_equal(self, prospective, spec): @@ -428,7 +416,7 @@ class Specifier(_IndividualSpecifier): # Shorten the prospective version to be the same length as the spec # so that we can determine if the specifier is a prefix of the # prospective version or not. - prospective = prospective[:len(spec)] + prospective = prospective[: len(spec)] # Pad out our two sides with zeros so that they both equal the same # length. @@ -567,27 +555,17 @@ def _pad_version(left, right): right_split.append(list(itertools.takewhile(lambda x: x.isdigit(), right))) # Get the rest of our versions - left_split.append(left[len(left_split[0]):]) - right_split.append(right[len(right_split[0]):]) + left_split.append(left[len(left_split[0]) :]) + right_split.append(right[len(right_split[0]) :]) # Insert our padding - left_split.insert( - 1, - ["0"] * max(0, len(right_split[0]) - len(left_split[0])), - ) - right_split.insert( - 1, - ["0"] * max(0, len(left_split[0]) - len(right_split[0])), - ) + left_split.insert(1, ["0"] * max(0, len(right_split[0]) - len(left_split[0]))) + right_split.insert(1, ["0"] * max(0, len(left_split[0]) - len(right_split[0]))) - return ( - list(itertools.chain(*left_split)), - list(itertools.chain(*right_split)), - ) + return (list(itertools.chain(*left_split)), list(itertools.chain(*right_split))) class SpecifierSet(BaseSpecifier): - def __init__(self, specifiers="", prereleases=None): # Split on , to break each indidivual specifier into it's own item, and # strip each item to remove leading/trailing whitespace. @@ -721,10 +699,7 @@ class SpecifierSet(BaseSpecifier): # given version is contained within all of them. # Note: This use of all() here means that an empty set of specifiers # will always return True, this is an explicit design decision. - return all( - s.contains(item, prereleases=prereleases) - for s in self._specs - ) + return all(s.contains(item, prereleases=prereleases) for s in self._specs) def filter(self, iterable, prereleases=None): # Determine if we're forcing a prerelease or not, if we're not forcing diff --git a/pipenv/vendor/packaging/utils.py b/pipenv/vendor/packaging/utils.py index 4b94a82f..88418786 100644 --- a/pipenv/vendor/packaging/utils.py +++ b/pipenv/vendor/packaging/utils.py @@ -36,13 +36,7 @@ def canonicalize_version(version): # Release segment # NB: This strips trailing '.0's to normalize - parts.append( - re.sub( - r'(\.0)+$', - '', - ".".join(str(x) for x in version.release) - ) - ) + parts.append(re.sub(r"(\.0)+$", "", ".".join(str(x) for x in version.release))) # Pre-release if version.pre is not None: diff --git a/pipenv/vendor/packaging/version.py b/pipenv/vendor/packaging/version.py index 6ed5cbbd..95157a1f 100644 --- a/pipenv/vendor/packaging/version.py +++ b/pipenv/vendor/packaging/version.py @@ -10,14 +10,11 @@ import re from ._structures import Infinity -__all__ = [ - "parse", "Version", "LegacyVersion", "InvalidVersion", "VERSION_PATTERN" -] +__all__ = ["parse", "Version", "LegacyVersion", "InvalidVersion", "VERSION_PATTERN"] _Version = collections.namedtuple( - "_Version", - ["epoch", "release", "dev", "pre", "post", "local"], + "_Version", ["epoch", "release", "dev", "pre", "post", "local"] ) @@ -40,7 +37,6 @@ class InvalidVersion(ValueError): class _BaseVersion(object): - def __hash__(self): return hash(self._key) @@ -70,7 +66,6 @@ class _BaseVersion(object): class LegacyVersion(_BaseVersion): - def __init__(self, version): self._version = str(version) self._key = _legacy_cmpkey(self._version) @@ -126,12 +121,14 @@ class LegacyVersion(_BaseVersion): return False -_legacy_version_component_re = re.compile( - r"(\d+ | [a-z]+ | \.| -)", re.VERBOSE, -) +_legacy_version_component_re = re.compile(r"(\d+ | [a-z]+ | \.| -)", re.VERBOSE) _legacy_version_replacement_map = { - "pre": "c", "preview": "c", "-": "final-", "rc": "c", "dev": "@", + "pre": "c", + "preview": "c", + "-": "final-", + "rc": "c", + "dev": "@", } @@ -215,10 +212,7 @@ VERSION_PATTERN = r""" class Version(_BaseVersion): - _regex = re.compile( - r"^\s*" + VERSION_PATTERN + r"\s*$", - re.VERBOSE | re.IGNORECASE, - ) + _regex = re.compile(r"^\s*" + VERSION_PATTERN + r"\s*$", re.VERBOSE | re.IGNORECASE) def __init__(self, version): # Validate the version and parse it into pieces @@ -230,18 +224,11 @@ class Version(_BaseVersion): self._version = _Version( epoch=int(match.group("epoch")) if match.group("epoch") else 0, release=tuple(int(i) for i in match.group("release").split(".")), - pre=_parse_letter_version( - match.group("pre_l"), - match.group("pre_n"), - ), + pre=_parse_letter_version(match.group("pre_l"), match.group("pre_n")), post=_parse_letter_version( - match.group("post_l"), - match.group("post_n1") or match.group("post_n2"), - ), - dev=_parse_letter_version( - match.group("dev_l"), - match.group("dev_n"), + match.group("post_l"), match.group("post_n1") or match.group("post_n2") ), + dev=_parse_letter_version(match.group("dev_l"), match.group("dev_n")), local=_parse_local_version(match.group("local")), ) @@ -395,12 +382,7 @@ def _cmpkey(epoch, release, pre, post, dev, local): # re-reverse it back into the correct order and make it a tuple and use # that for our sorting key. release = tuple( - reversed(list( - itertools.dropwhile( - lambda x: x == 0, - reversed(release), - ) - )) + reversed(list(itertools.dropwhile(lambda x: x == 0, reversed(release)))) ) # We need to "trick" the sorting algorithm to put 1.0.dev0 before 1.0a0. @@ -433,9 +415,6 @@ def _cmpkey(epoch, release, pre, post, dev, local): # - Numeric segments sort numerically # - Shorter versions sort before longer versions when the prefixes # match exactly - local = tuple( - (i, "") if isinstance(i, int) else (-Infinity, i) - for i in local - ) + local = tuple((i, "") if isinstance(i, int) else (-Infinity, i) for i in local) return epoch, release, pre, post, dev, local diff --git a/pipenv/vendor/parse.py b/pipenv/vendor/parse.py index 7f9f0786..0b5cce23 100644 --- a/pipenv/vendor/parse.py +++ b/pipenv/vendor/parse.py @@ -3,7 +3,7 @@ r'''Parse strings using a specification based on the Python format() syntax. ``parse()`` is the opposite of ``format()`` The module is set up to only export ``parse()``, ``search()``, ``findall()``, -and ``with_pattern()`` when ``import *`` is used: +and ``with_pattern()`` when ``import \*`` is used: >>> from parse import * @@ -78,9 +78,11 @@ Some simple parse() format string examples: {'item': 'hand grenade'} >>> print(r['item']) hand grenade +>>> 'item' in r +True -Dotted names and indexes are possible though the application must make -additional sense of the result: +Note that `in` only works if you have named fields. Dotted names and indexes +are possible though the application must make additional sense of the result: >>> r = parse("Mmm, {food.type}, I love it!", "Mmm, spam, I love it!") >>> print(r) @@ -132,38 +134,39 @@ The differences between `parse()` and `format()` are: ===== =========================================== ======== Type Characters Matched Output ===== =========================================== ======== - w Letters and underscore str - W Non-letter and underscore str - s Whitespace str - S Non-whitespace str - d Digits (effectively integer numbers) int - D Non-digit str - n Numbers with thousands separators (, or .) int - % Percentage (converted to value/100.0) float - f Fixed-point numbers float - F Decimal numbers Decimal - e Floating-point numbers with exponent float +l Letters (ASCII) str +w Letters, numbers and underscore str +W Not letters, numbers and underscore str +s Whitespace str +S Non-whitespace str +d Digits (effectively integer numbers) int +D Non-digit str +n Numbers with thousands separators (, or .) int +% Percentage (converted to value/100.0) float +f Fixed-point numbers float +F Decimal numbers Decimal +e Floating-point numbers with exponent float e.g. 1.1e-10, NAN (all case insensitive) - g General number format (either d, f or e) float - b Binary numbers int - o Octal numbers int - x Hexadecimal numbers (lower and upper case) int - ti ISO 8601 format date/time datetime +g General number format (either d, f or e) float +b Binary numbers int +o Octal numbers int +x Hexadecimal numbers (lower and upper case) int +ti ISO 8601 format date/time datetime e.g. 1972-01-20T10:21:36Z ("T" and "Z" optional) - te RFC2822 e-mail format date/time datetime +te RFC2822 e-mail format date/time datetime e.g. Mon, 20 Jan 1972 10:21:36 +1000 - tg Global (day/month) format date/time datetime +tg Global (day/month) format date/time datetime e.g. 20/1/1972 10:21:36 AM +1:00 - ta US (month/day) format date/time datetime +ta US (month/day) format date/time datetime e.g. 1/20/1972 10:21:36 PM +10:30 - tc ctime() format date/time datetime +tc ctime() format date/time datetime e.g. Sun Sep 16 01:03:52 1973 - th HTTP log format date/time datetime +th HTTP log format date/time datetime e.g. 21/Nov/2011:00:07:11 +0000 - ts Linux system log format date/time datetime +ts Linux system log format date/time datetime e.g. Nov 9 03:37:44 - tt Time time +tt Time time e.g. 10:21:36 PM -5:30 ===== =========================================== ======== @@ -342,6 +345,14 @@ the pattern, the actual match represents the shortest successful match for **Version history (in brief)**: +- 1.12.0 Do not assume closing brace when an opening one is found (thanks @mattsep) +- 1.11.1 Revert having unicode char in docstring, it breaks Bamboo builds(?!) +- 1.11.0 Implement `__contains__` for Result instances. +- 1.10.0 Introduce a "letters" matcher, since "w" matches numbers + also. +- 1.9.1 Fix deprecation warnings around backslashes in regex strings + (thanks Mickael Schoentgen). Also fix some documentation formatting + issues. - 1.9.0 We now honor precision and width specifiers when parsing numbers and strings, allowing parsing of concatenated elements of fixed width (thanks Julia Signell) @@ -400,12 +411,12 @@ the pattern, the actual match represents the shortest successful match for and removed the restriction on mixing fixed-position and named fields - 1.0.0 initial release -This code is copyright 2012-2017 Richard Jones +This code is copyright 2012-2019 Richard Jones See the end of the source file for the license of use. ''' from __future__ import absolute_import -__version__ = '1.9.0' +__version__ = '1.12.0' # yes, I now have two problems import re @@ -421,7 +432,7 @@ log = logging.getLogger(__name__) def with_pattern(pattern, regex_group_count=None): - """Attach a regular expression pattern matcher to a custom type converter + r"""Attach a regular expression pattern matcher to a custom type converter function. This annotates the type converter with the :attr:`pattern` attribute. @@ -530,9 +541,9 @@ MONTHS_MAP = dict( Nov=11, November=11, Dec=12, December=12 ) -DAYS_PAT = '(Mon|Tue|Wed|Thu|Fri|Sat|Sun)' -MONTHS_PAT = '(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)' -ALL_MONTHS_PAT = '(%s)' % '|'.join(MONTHS_MAP) +DAYS_PAT = r'(Mon|Tue|Wed|Thu|Fri|Sat|Sun)' +MONTHS_PAT = r'(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)' +ALL_MONTHS_PAT = r'(%s)' % '|'.join(MONTHS_MAP) TIME_PAT = r'(\d{1,2}:\d{1,2}(:\d{1,2}(\.\d+)?)?)' AM_PAT = r'(\s+[AP]M)' TZ_PAT = r'(\s+[-+]\d\d?:?\d\d)' @@ -550,11 +561,11 @@ def date_convert(string, match, ymd=None, mdy=None, dmy=None, m=groups[mm] d=groups[dd] elif ymd is not None: - y, m, d = re.split('[-/\s]', groups[ymd]) + y, m, d = re.split(r'[-/\s]', groups[ymd]) elif mdy is not None: - m, d, y = re.split('[-/\s]', groups[mdy]) + m, d, y = re.split(r'[-/\s]', groups[mdy]) elif dmy is not None: - d, m, y = re.split('[-/\s]', groups[dmy]) + d, m, y = re.split(r'[-/\s]', groups[dmy]) elif d_m_y is not None: d, m, y = d_m_y d = groups[d] @@ -636,10 +647,10 @@ class RepeatedNameError(ValueError): # note: {} are handled separately # note: I don't use r'' here because Sublime Text 2 syntax highlight has a fit -REGEX_SAFETY = re.compile('([?\\\\.[\]()*+\^$!\|])') +REGEX_SAFETY = re.compile(r'([?\\\\.[\]()*+\^$!\|])') # allowed field types -ALLOWED_TYPES = set(list('nbox%fFegwWdDsS') + +ALLOWED_TYPES = set(list('nbox%fFegwWdDsSl') + ['t' + c for c in 'ieahgcts']) @@ -745,7 +756,7 @@ class Parser(object): @property def _match_re(self): if self.__match_re is None: - expression = '^%s$' % self._expression + expression = r'^%s$' % self._expression try: self.__match_re = re.compile(expression, self._re_flags) except AssertionError: @@ -875,7 +886,7 @@ class Parser(object): e.append(r'\{') elif part == '}}': e.append(r'\}') - elif part[0] == '{': + elif part[0] == '{' and part[-1] == '}': # this will be a braces-delimited field to handle e.append(self._handle_field(part)) else: @@ -923,16 +934,16 @@ class Parser(object): name, self._name_types[name])) group = self._name_to_group_map[name] # match previously-seen value - return '(?P=%s)' % group + return r'(?P=%s)' % group else: group = self._to_group_name(name) self._name_types[name] = format self._named_fields.append(group) # this will become a group, which must not contain dots - wrap = '(?P<%s>%%s)' % group + wrap = r'(?P<%s>%%s)' % group else: self._fixed_fields.append(self._group_index) - wrap = '(%s)' + wrap = r'(%s)' if ':' in field: format = field[1:] group = self._group_index @@ -940,7 +951,7 @@ class Parser(object): # simplest case: no type specifier ({} or {name}) if not format: self._group_index += 1 - return wrap % '.+?' + return wrap % r'.+?' # decode the format specification format = extract_format(format, self._extra_types) @@ -960,19 +971,19 @@ class Parser(object): return type_converter(string) self._type_conversions[group] = f elif type == 'n': - s = '\d{1,3}([,.]\d{3})*' + s = r'\d{1,3}([,.]\d{3})*' self._group_index += 1 self._type_conversions[group] = int_convert(10) elif type == 'b': - s = '(0[bB])?[01]+' + s = r'(0[bB])?[01]+' self._type_conversions[group] = int_convert(2) self._group_index += 1 elif type == 'o': - s = '(0[oO])?[0-7]+' + s = r'(0[oO])?[0-7]+' self._type_conversions[group] = int_convert(8) self._group_index += 1 elif type == 'x': - s = '(0[xX])?[0-9a-fA-F]+' + s = r'(0[xX])?[0-9a-fA-F]+' self._type_conversions[group] = int_convert(16) self._group_index += 1 elif type == '%': @@ -994,10 +1005,10 @@ class Parser(object): self._type_conversions[group] = lambda s, m: float(s) elif type == 'd': if format.get('width'): - width = '{1,%s}' % int(format['width']) + width = r'{1,%s}' % int(format['width']) else: width = '+' - s = '\\d{w}|0[xX][0-9a-fA-F]{w}|0[bB][01]{w}|0[oO][0-7]{w}'.format(w=width) + s = r'\d{w}|0[xX][0-9a-fA-F]{w}|0[bB][01]{w}|0[oO][0-7]{w}'.format(w=width) self._type_conversions[group] = int_convert(10) elif type == 'ti': s = r'(\d{4}-\d\d-\d\d)((\s+|T)%s)?(Z|\s*[-+]\d\d:?\d\d)?' % \ @@ -1055,18 +1066,19 @@ class Parser(object): self._type_conversions[group] = partial(date_convert, mm=n+1, dd=n+3, hms=n + 5) self._group_index += 5 - + elif type == 'l': + s = r'[A-Za-z]+' elif type: s = r'\%s+' % type elif format.get('precision'): if format.get('width'): - s = '.{%s,%s}?' % (format['width'], format['precision']) + s = r'.{%s,%s}?' % (format['width'], format['precision']) else: - s = '.{1,%s}?' % format['precision'] + s = r'.{1,%s}?' % format['precision'] elif format.get('width'): - s = '.{%s,}?' % format['width'] + s = r'.{%s,}?' % format['width'] else: - s = '.+?' + s = r'.+?' align = format['align'] fill = format['fill'] @@ -1079,7 +1091,7 @@ class Parser(object): # configurable fill defaulting to "0" if not fill: fill = '0' - s = '%s*' % fill + s + s = r'%s*' % fill + s # allow numbers to be prefixed with a sign s = r'[-+ ]?' + s @@ -1101,7 +1113,7 @@ class Parser(object): if not align: align = '>' - if fill in '.\+?*[](){}^$': + if fill in r'.\+?*[](){}^$': fill = '\\' + fill # align "=" has been handled @@ -1118,8 +1130,11 @@ class Parser(object): class Result(object): '''The result of a parse() or search(). - Fixed results may be looked up using result[index]. Named results may be - looked up using result['name']. + Fixed results may be looked up using `result[index]`. + + Named results may be looked up using `result['name']`. + + Named results may be tested for existence using `'name' in result`. ''' def __init__(self, fixed, named, spans): self.fixed = fixed @@ -1135,6 +1150,9 @@ class Result(object): return '<%s %r %r>' % (self.__class__.__name__, self.fixed, self.named) + def __contains__(self, name): + return name in self.named + class Match(object): '''The result of a parse() or search() if no results are generated. @@ -1295,7 +1313,7 @@ def compile(format, extra_types=None, case_sensitive=False): return Parser(format, extra_types=extra_types) -# Copyright (c) 2012-2013 Richard Jones +# Copyright (c) 2012-2019 Richard Jones # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/pipenv/vendor/passa/cli/options.py b/pipenv/vendor/passa/cli/options.py index f8ba1fe7..f20b612a 100644 --- a/pipenv/vendor/passa/cli/options.py +++ b/pipenv/vendor/passa/cli/options.py @@ -20,13 +20,13 @@ class Project(passa.models.projects.Project): pipfile = root.joinpath("Pipfile") if not pipfile.is_file(): raise argparse.ArgumentError( - "{0!r} is not a Pipfile project".format(root), + "project", "{0!r} is not a Pipfile project".format(root), ) try: super(Project, self).__init__(root.as_posix(), *args, **kwargs) except tomlkit.exceptions.ParseError as e: raise argparse.ArgumentError( - "failed to parse Pipfile: {0!r}".format(str(e)), + "project", "failed to parse Pipfile: {0!r}".format(str(e)), ) def __name__(self): diff --git a/pipenv/vendor/pep517/LICENSE b/pipenv/vendor/pep517/LICENSE new file mode 100644 index 00000000..b0ae9dbc --- /dev/null +++ b/pipenv/vendor/pep517/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2017 Thomas Kluyver + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/pipenv/vendor/pep517/__init__.py b/pipenv/vendor/pep517/__init__.py new file mode 100644 index 00000000..9c1a098f --- /dev/null +++ b/pipenv/vendor/pep517/__init__.py @@ -0,0 +1,4 @@ +"""Wrappers to build Python packages using PEP 517 hooks +""" + +__version__ = '0.5.0' diff --git a/pipenv/vendor/pep517/_in_process.py b/pipenv/vendor/pep517/_in_process.py new file mode 100644 index 00000000..d6524b66 --- /dev/null +++ b/pipenv/vendor/pep517/_in_process.py @@ -0,0 +1,207 @@ +"""This is invoked in a subprocess to call the build backend hooks. + +It expects: +- Command line args: hook_name, control_dir +- Environment variable: PEP517_BUILD_BACKEND=entry.point:spec +- control_dir/input.json: + - {"kwargs": {...}} + +Results: +- control_dir/output.json + - {"return_val": ...} +""" +from glob import glob +from importlib import import_module +import os +from os.path import join as pjoin +import re +import shutil +import sys + +# This is run as a script, not a module, so it can't do a relative import +import compat + + +class BackendUnavailable(Exception): + """Raised if we cannot import the backend""" + + +def _build_backend(): + """Find and load the build backend""" + ep = os.environ['PEP517_BUILD_BACKEND'] + mod_path, _, obj_path = ep.partition(':') + try: + obj = import_module(mod_path) + except ImportError: + raise BackendUnavailable + if obj_path: + for path_part in obj_path.split('.'): + obj = getattr(obj, path_part) + return obj + + +def get_requires_for_build_wheel(config_settings): + """Invoke the optional get_requires_for_build_wheel hook + + Returns [] if the hook is not defined. + """ + backend = _build_backend() + try: + hook = backend.get_requires_for_build_wheel + except AttributeError: + return [] + else: + return hook(config_settings) + + +def prepare_metadata_for_build_wheel(metadata_directory, config_settings): + """Invoke optional prepare_metadata_for_build_wheel + + Implements a fallback by building a wheel if the hook isn't defined. + """ + backend = _build_backend() + try: + hook = backend.prepare_metadata_for_build_wheel + except AttributeError: + return _get_wheel_metadata_from_wheel(backend, metadata_directory, + config_settings) + else: + return hook(metadata_directory, config_settings) + + +WHEEL_BUILT_MARKER = 'PEP517_ALREADY_BUILT_WHEEL' + + +def _dist_info_files(whl_zip): + """Identify the .dist-info folder inside a wheel ZipFile.""" + res = [] + for path in whl_zip.namelist(): + m = re.match(r'[^/\\]+-[^/\\]+\.dist-info/', path) + if m: + res.append(path) + if res: + return res + raise Exception("No .dist-info folder found in wheel") + + +def _get_wheel_metadata_from_wheel( + backend, metadata_directory, config_settings): + """Build a wheel and extract the metadata from it. + + Fallback for when the build backend does not + define the 'get_wheel_metadata' hook. + """ + from zipfile import ZipFile + whl_basename = backend.build_wheel(metadata_directory, config_settings) + with open(os.path.join(metadata_directory, WHEEL_BUILT_MARKER), 'wb'): + pass # Touch marker file + + whl_file = os.path.join(metadata_directory, whl_basename) + with ZipFile(whl_file) as zipf: + dist_info = _dist_info_files(zipf) + zipf.extractall(path=metadata_directory, members=dist_info) + return dist_info[0].split('/')[0] + + +def _find_already_built_wheel(metadata_directory): + """Check for a wheel already built during the get_wheel_metadata hook. + """ + if not metadata_directory: + return None + metadata_parent = os.path.dirname(metadata_directory) + if not os.path.isfile(pjoin(metadata_parent, WHEEL_BUILT_MARKER)): + return None + + whl_files = glob(os.path.join(metadata_parent, '*.whl')) + if not whl_files: + print('Found wheel built marker, but no .whl files') + return None + if len(whl_files) > 1: + print('Found multiple .whl files; unspecified behaviour. ' + 'Will call build_wheel.') + return None + + # Exactly one .whl file + return whl_files[0] + + +def build_wheel(wheel_directory, config_settings, metadata_directory=None): + """Invoke the mandatory build_wheel hook. + + If a wheel was already built in the + prepare_metadata_for_build_wheel fallback, this + will copy it rather than rebuilding the wheel. + """ + prebuilt_whl = _find_already_built_wheel(metadata_directory) + if prebuilt_whl: + shutil.copy2(prebuilt_whl, wheel_directory) + return os.path.basename(prebuilt_whl) + + return _build_backend().build_wheel(wheel_directory, config_settings, + metadata_directory) + + +def get_requires_for_build_sdist(config_settings): + """Invoke the optional get_requires_for_build_wheel hook + + Returns [] if the hook is not defined. + """ + backend = _build_backend() + try: + hook = backend.get_requires_for_build_sdist + except AttributeError: + return [] + else: + return hook(config_settings) + + +class _DummyException(Exception): + """Nothing should ever raise this exception""" + + +class GotUnsupportedOperation(Exception): + """For internal use when backend raises UnsupportedOperation""" + + +def build_sdist(sdist_directory, config_settings): + """Invoke the mandatory build_sdist hook.""" + backend = _build_backend() + try: + return backend.build_sdist(sdist_directory, config_settings) + except getattr(backend, 'UnsupportedOperation', _DummyException): + raise GotUnsupportedOperation + + +HOOK_NAMES = { + 'get_requires_for_build_wheel', + 'prepare_metadata_for_build_wheel', + 'build_wheel', + 'get_requires_for_build_sdist', + 'build_sdist', +} + + +def main(): + if len(sys.argv) < 3: + sys.exit("Needs args: hook_name, control_dir") + hook_name = sys.argv[1] + control_dir = sys.argv[2] + if hook_name not in HOOK_NAMES: + sys.exit("Unknown hook: %s" % hook_name) + hook = globals()[hook_name] + + hook_input = compat.read_json(pjoin(control_dir, 'input.json')) + + json_out = {'unsupported': False, 'return_val': None} + try: + json_out['return_val'] = hook(**hook_input['kwargs']) + except BackendUnavailable: + json_out['no_backend'] = True + except GotUnsupportedOperation: + json_out['unsupported'] = True + + compat.write_json(json_out, pjoin(control_dir, 'output.json'), indent=2) + + +if __name__ == '__main__': + main() diff --git a/pipenv/vendor/pep517/build.py b/pipenv/vendor/pep517/build.py new file mode 100644 index 00000000..6fca39a8 --- /dev/null +++ b/pipenv/vendor/pep517/build.py @@ -0,0 +1,108 @@ +"""Build a project using PEP 517 hooks. +""" +import argparse +import logging +import os +import contextlib +import pytoml +import shutil +import errno +import tempfile + +from .envbuild import BuildEnvironment +from .wrappers import Pep517HookCaller + +log = logging.getLogger(__name__) + + +@contextlib.contextmanager +def tempdir(): + td = tempfile.mkdtemp() + try: + yield td + finally: + shutil.rmtree(td) + + +def _do_build(hooks, env, dist, dest): + get_requires_name = 'get_requires_for_build_{dist}'.format(**locals()) + get_requires = getattr(hooks, get_requires_name) + reqs = get_requires({}) + log.info('Got build requires: %s', reqs) + + env.pip_install(reqs) + log.info('Installed dynamic build dependencies') + + with tempdir() as td: + log.info('Trying to build %s in %s', dist, td) + build_name = 'build_{dist}'.format(**locals()) + build = getattr(hooks, build_name) + filename = build(td, {}) + source = os.path.join(td, filename) + shutil.move(source, os.path.join(dest, os.path.basename(filename))) + + +def mkdir_p(*args, **kwargs): + """Like `mkdir`, but does not raise an exception if the + directory already exists. + """ + try: + return os.mkdir(*args, **kwargs) + except OSError as exc: + if exc.errno != errno.EEXIST: + raise + + +def build(source_dir, dist, dest=None): + pyproject = os.path.join(source_dir, 'pyproject.toml') + dest = os.path.join(source_dir, dest or 'dist') + mkdir_p(dest) + + with open(pyproject) as f: + pyproject_data = pytoml.load(f) + # Ensure the mandatory data can be loaded + buildsys = pyproject_data['build-system'] + requires = buildsys['requires'] + backend = buildsys['build-backend'] + + hooks = Pep517HookCaller(source_dir, backend) + + with BuildEnvironment() as env: + env.pip_install(requires) + _do_build(hooks, env, dist, dest) + + +parser = argparse.ArgumentParser() +parser.add_argument( + 'source_dir', + help="A directory containing pyproject.toml", +) +parser.add_argument( + '--binary', '-b', + action='store_true', + default=False, +) +parser.add_argument( + '--source', '-s', + action='store_true', + default=False, +) +parser.add_argument( + '--out-dir', '-o', + help="Destination in which to save the builds relative to source dir", +) + + +def main(args): + # determine which dists to build + dists = list(filter(None, ( + 'sdist' if args.source or not args.binary else None, + 'wheel' if args.binary or not args.source else None, + ))) + + for dist in dists: + build(args.source_dir, dist, args.out_dir) + + +if __name__ == '__main__': + main(parser.parse_args()) diff --git a/pipenv/vendor/pep517/check.py b/pipenv/vendor/pep517/check.py new file mode 100644 index 00000000..fc82cca7 --- /dev/null +++ b/pipenv/vendor/pep517/check.py @@ -0,0 +1,202 @@ +"""Check a project and backend by attempting to build using PEP 517 hooks. +""" +import argparse +import logging +import os +from os.path import isfile, join as pjoin +from pytoml import TomlError, load as toml_load +import shutil +from subprocess import CalledProcessError +import sys +import tarfile +from tempfile import mkdtemp +import zipfile + +from .colorlog import enable_colourful_output +from .envbuild import BuildEnvironment +from .wrappers import Pep517HookCaller + +log = logging.getLogger(__name__) + + +def check_build_sdist(hooks, build_sys_requires): + with BuildEnvironment() as env: + try: + env.pip_install(build_sys_requires) + log.info('Installed static build dependencies') + except CalledProcessError: + log.error('Failed to install static build dependencies') + return False + + try: + reqs = hooks.get_requires_for_build_sdist({}) + log.info('Got build requires: %s', reqs) + except Exception: + log.error('Failure in get_requires_for_build_sdist', exc_info=True) + return False + + try: + env.pip_install(reqs) + log.info('Installed dynamic build dependencies') + except CalledProcessError: + log.error('Failed to install dynamic build dependencies') + return False + + td = mkdtemp() + log.info('Trying to build sdist in %s', td) + try: + try: + filename = hooks.build_sdist(td, {}) + log.info('build_sdist returned %r', filename) + except Exception: + log.info('Failure in build_sdist', exc_info=True) + return False + + if not filename.endswith('.tar.gz'): + log.error( + "Filename %s doesn't have .tar.gz extension", filename) + return False + + path = pjoin(td, filename) + if isfile(path): + log.info("Output file %s exists", path) + else: + log.error("Output file %s does not exist", path) + return False + + if tarfile.is_tarfile(path): + log.info("Output file is a tar file") + else: + log.error("Output file is not a tar file") + return False + + finally: + shutil.rmtree(td) + + return True + + +def check_build_wheel(hooks, build_sys_requires): + with BuildEnvironment() as env: + try: + env.pip_install(build_sys_requires) + log.info('Installed static build dependencies') + except CalledProcessError: + log.error('Failed to install static build dependencies') + return False + + try: + reqs = hooks.get_requires_for_build_wheel({}) + log.info('Got build requires: %s', reqs) + except Exception: + log.error('Failure in get_requires_for_build_sdist', exc_info=True) + return False + + try: + env.pip_install(reqs) + log.info('Installed dynamic build dependencies') + except CalledProcessError: + log.error('Failed to install dynamic build dependencies') + return False + + td = mkdtemp() + log.info('Trying to build wheel in %s', td) + try: + try: + filename = hooks.build_wheel(td, {}) + log.info('build_wheel returned %r', filename) + except Exception: + log.info('Failure in build_wheel', exc_info=True) + return False + + if not filename.endswith('.whl'): + log.error("Filename %s doesn't have .whl extension", filename) + return False + + path = pjoin(td, filename) + if isfile(path): + log.info("Output file %s exists", path) + else: + log.error("Output file %s does not exist", path) + return False + + if zipfile.is_zipfile(path): + log.info("Output file is a zip file") + else: + log.error("Output file is not a zip file") + return False + + finally: + shutil.rmtree(td) + + return True + + +def check(source_dir): + pyproject = pjoin(source_dir, 'pyproject.toml') + if isfile(pyproject): + log.info('Found pyproject.toml') + else: + log.error('Missing pyproject.toml') + return False + + try: + with open(pyproject) as f: + pyproject_data = toml_load(f) + # Ensure the mandatory data can be loaded + buildsys = pyproject_data['build-system'] + requires = buildsys['requires'] + backend = buildsys['build-backend'] + log.info('Loaded pyproject.toml') + except (TomlError, KeyError): + log.error("Invalid pyproject.toml", exc_info=True) + return False + + hooks = Pep517HookCaller(source_dir, backend) + + sdist_ok = check_build_sdist(hooks, requires) + wheel_ok = check_build_wheel(hooks, requires) + + if not sdist_ok: + log.warning('Sdist checks failed; scroll up to see') + if not wheel_ok: + log.warning('Wheel checks failed') + + return sdist_ok + + +def main(argv=None): + ap = argparse.ArgumentParser() + ap.add_argument( + 'source_dir', + help="A directory containing pyproject.toml") + args = ap.parse_args(argv) + + enable_colourful_output() + + ok = check(args.source_dir) + + if ok: + print(ansi('Checks passed', 'green')) + else: + print(ansi('Checks failed', 'red')) + sys.exit(1) + + +ansi_codes = { + 'reset': '\x1b[0m', + 'bold': '\x1b[1m', + 'red': '\x1b[31m', + 'green': '\x1b[32m', +} + + +def ansi(s, attr): + if os.name != 'nt' and sys.stdout.isatty(): + return ansi_codes[attr] + str(s) + ansi_codes['reset'] + else: + return str(s) + + +if __name__ == '__main__': + main() diff --git a/pipenv/vendor/pep517/colorlog.py b/pipenv/vendor/pep517/colorlog.py new file mode 100644 index 00000000..69c8a59d --- /dev/null +++ b/pipenv/vendor/pep517/colorlog.py @@ -0,0 +1,115 @@ +"""Nicer log formatting with colours. + +Code copied from Tornado, Apache licensed. +""" +# Copyright 2012 Facebook +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import sys + +try: + import curses +except ImportError: + curses = None + + +def _stderr_supports_color(): + color = False + if curses and hasattr(sys.stderr, 'isatty') and sys.stderr.isatty(): + try: + curses.setupterm() + if curses.tigetnum("colors") > 0: + color = True + except Exception: + pass + return color + + +class LogFormatter(logging.Formatter): + """Log formatter with colour support + """ + DEFAULT_COLORS = { + logging.INFO: 2, # Green + logging.WARNING: 3, # Yellow + logging.ERROR: 1, # Red + logging.CRITICAL: 1, + } + + def __init__(self, color=True, datefmt=None): + r""" + :arg bool color: Enables color support. + :arg string fmt: Log message format. + It will be applied to the attributes dict of log records. The + text between ``%(color)s`` and ``%(end_color)s`` will be colored + depending on the level if color support is on. + :arg dict colors: color mappings from logging level to terminal color + code + :arg string datefmt: Datetime format. + Used for formatting ``(asctime)`` placeholder in ``prefix_fmt``. + .. versionchanged:: 3.2 + Added ``fmt`` and ``datefmt`` arguments. + """ + logging.Formatter.__init__(self, datefmt=datefmt) + self._colors = {} + if color and _stderr_supports_color(): + # The curses module has some str/bytes confusion in + # python3. Until version 3.2.3, most methods return + # bytes, but only accept strings. In addition, we want to + # output these strings with the logging module, which + # works with unicode strings. The explicit calls to + # unicode() below are harmless in python2 but will do the + # right conversion in python 3. + fg_color = (curses.tigetstr("setaf") or + curses.tigetstr("setf") or "") + if (3, 0) < sys.version_info < (3, 2, 3): + fg_color = str(fg_color, "ascii") + + for levelno, code in self.DEFAULT_COLORS.items(): + self._colors[levelno] = str( + curses.tparm(fg_color, code), "ascii") + self._normal = str(curses.tigetstr("sgr0"), "ascii") + + scr = curses.initscr() + self.termwidth = scr.getmaxyx()[1] + curses.endwin() + else: + self._normal = '' + # Default width is usually 80, but too wide is + # worse than too narrow + self.termwidth = 70 + + def formatMessage(self, record): + mlen = len(record.message) + right_text = '{initial}-{name}'.format(initial=record.levelname[0], + name=record.name) + if mlen + len(right_text) < self.termwidth: + space = ' ' * (self.termwidth - (mlen + len(right_text))) + else: + space = ' ' + + if record.levelno in self._colors: + start_color = self._colors[record.levelno] + end_color = self._normal + else: + start_color = end_color = '' + + return record.message + space + start_color + right_text + end_color + + +def enable_colourful_output(level=logging.INFO): + handler = logging.StreamHandler() + handler.setFormatter(LogFormatter()) + logging.root.addHandler(handler) + logging.root.setLevel(level) diff --git a/pipenv/vendor/pep517/compat.py b/pipenv/vendor/pep517/compat.py new file mode 100644 index 00000000..01c66fc7 --- /dev/null +++ b/pipenv/vendor/pep517/compat.py @@ -0,0 +1,23 @@ +"""Handle reading and writing JSON in UTF-8, on Python 3 and 2.""" +import json +import sys + +if sys.version_info[0] >= 3: + # Python 3 + def write_json(obj, path, **kwargs): + with open(path, 'w', encoding='utf-8') as f: + json.dump(obj, f, **kwargs) + + def read_json(path): + with open(path, 'r', encoding='utf-8') as f: + return json.load(f) + +else: + # Python 2 + def write_json(obj, path, **kwargs): + with open(path, 'wb') as f: + json.dump(obj, f, encoding='utf-8', **kwargs) + + def read_json(path): + with open(path, 'rb') as f: + return json.load(f) diff --git a/pipenv/vendor/pep517/envbuild.py b/pipenv/vendor/pep517/envbuild.py new file mode 100644 index 00000000..61253f4d --- /dev/null +++ b/pipenv/vendor/pep517/envbuild.py @@ -0,0 +1,158 @@ +"""Build wheels/sdists by installing build deps to a temporary environment. +""" + +import os +import logging +import pytoml +import shutil +from subprocess import check_call +import sys +from sysconfig import get_paths +from tempfile import mkdtemp + +from .wrappers import Pep517HookCaller + +log = logging.getLogger(__name__) + + +def _load_pyproject(source_dir): + with open(os.path.join(source_dir, 'pyproject.toml')) as f: + pyproject_data = pytoml.load(f) + buildsys = pyproject_data['build-system'] + return buildsys['requires'], buildsys['build-backend'] + + +class BuildEnvironment(object): + """Context manager to install build deps in a simple temporary environment + + Based on code I wrote for pip, which is MIT licensed. + """ + # Copyright (c) 2008-2016 The pip developers (see AUTHORS.txt file) + # + # Permission is hereby granted, free of charge, to any person obtaining + # a copy of this software and associated documentation files (the + # "Software"), to deal in the Software without restriction, including + # without limitation the rights to use, copy, modify, merge, publish, + # distribute, sublicense, and/or sell copies of the Software, and to + # permit persons to whom the Software is furnished to do so, subject to + # the following conditions: + # + # The above copyright notice and this permission notice shall be + # included in all copies or substantial portions of the Software. + # + # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + path = None + + def __init__(self, cleanup=True): + self._cleanup = cleanup + + def __enter__(self): + self.path = mkdtemp(prefix='pep517-build-env-') + log.info('Temporary build environment: %s', self.path) + + self.save_path = os.environ.get('PATH', None) + self.save_pythonpath = os.environ.get('PYTHONPATH', None) + + install_scheme = 'nt' if (os.name == 'nt') else 'posix_prefix' + install_dirs = get_paths(install_scheme, vars={ + 'base': self.path, + 'platbase': self.path, + }) + + scripts = install_dirs['scripts'] + if self.save_path: + os.environ['PATH'] = scripts + os.pathsep + self.save_path + else: + os.environ['PATH'] = scripts + os.pathsep + os.defpath + + if install_dirs['purelib'] == install_dirs['platlib']: + lib_dirs = install_dirs['purelib'] + else: + lib_dirs = install_dirs['purelib'] + os.pathsep + \ + install_dirs['platlib'] + if self.save_pythonpath: + os.environ['PYTHONPATH'] = lib_dirs + os.pathsep + \ + self.save_pythonpath + else: + os.environ['PYTHONPATH'] = lib_dirs + + return self + + def pip_install(self, reqs): + """Install dependencies into this env by calling pip in a subprocess""" + if not reqs: + return + log.info('Calling pip to install %s', reqs) + check_call([ + sys.executable, '-m', 'pip', 'install', '--ignore-installed', + '--prefix', self.path] + list(reqs)) + + def __exit__(self, exc_type, exc_val, exc_tb): + needs_cleanup = ( + self._cleanup and + self.path is not None and + os.path.isdir(self.path) + ) + if needs_cleanup: + shutil.rmtree(self.path) + + if self.save_path is None: + os.environ.pop('PATH', None) + else: + os.environ['PATH'] = self.save_path + + if self.save_pythonpath is None: + os.environ.pop('PYTHONPATH', None) + else: + os.environ['PYTHONPATH'] = self.save_pythonpath + + +def build_wheel(source_dir, wheel_dir, config_settings=None): + """Build a wheel from a source directory using PEP 517 hooks. + + :param str source_dir: Source directory containing pyproject.toml + :param str wheel_dir: Target directory to create wheel in + :param dict config_settings: Options to pass to build backend + + This is a blocking function which will run pip in a subprocess to install + build requirements. + """ + if config_settings is None: + config_settings = {} + requires, backend = _load_pyproject(source_dir) + hooks = Pep517HookCaller(source_dir, backend) + + with BuildEnvironment() as env: + env.pip_install(requires) + reqs = hooks.get_requires_for_build_wheel(config_settings) + env.pip_install(reqs) + return hooks.build_wheel(wheel_dir, config_settings) + + +def build_sdist(source_dir, sdist_dir, config_settings=None): + """Build an sdist from a source directory using PEP 517 hooks. + + :param str source_dir: Source directory containing pyproject.toml + :param str sdist_dir: Target directory to place sdist in + :param dict config_settings: Options to pass to build backend + + This is a blocking function which will run pip in a subprocess to install + build requirements. + """ + if config_settings is None: + config_settings = {} + requires, backend = _load_pyproject(source_dir) + hooks = Pep517HookCaller(source_dir, backend) + + with BuildEnvironment() as env: + env.pip_install(requires) + reqs = hooks.get_requires_for_build_sdist(config_settings) + env.pip_install(reqs) + return hooks.build_sdist(sdist_dir, config_settings) diff --git a/pipenv/vendor/pep517/wrappers.py b/pipenv/vendor/pep517/wrappers.py new file mode 100644 index 00000000..b14b8991 --- /dev/null +++ b/pipenv/vendor/pep517/wrappers.py @@ -0,0 +1,163 @@ +from contextlib import contextmanager +import os +from os.path import dirname, abspath, join as pjoin +import shutil +from subprocess import check_call +import sys +from tempfile import mkdtemp + +from . import compat + +_in_proc_script = pjoin(dirname(abspath(__file__)), '_in_process.py') + + +@contextmanager +def tempdir(): + td = mkdtemp() + try: + yield td + finally: + shutil.rmtree(td) + + +class BackendUnavailable(Exception): + """Will be raised if the backend cannot be imported in the hook process.""" + + +class UnsupportedOperation(Exception): + """May be raised by build_sdist if the backend indicates that it can't.""" + + +def default_subprocess_runner(cmd, cwd=None, extra_environ=None): + """The default method of calling the wrapper subprocess.""" + env = os.environ.copy() + if extra_environ: + env.update(extra_environ) + + check_call(cmd, cwd=cwd, env=env) + + +class Pep517HookCaller(object): + """A wrapper around a source directory to be built with a PEP 517 backend. + + source_dir : The path to the source directory, containing pyproject.toml. + backend : The build backend spec, as per PEP 517, from pyproject.toml. + """ + def __init__(self, source_dir, build_backend): + self.source_dir = abspath(source_dir) + self.build_backend = build_backend + self._subprocess_runner = default_subprocess_runner + + # TODO: Is this over-engineered? Maybe frontends only need to + # set this when creating the wrapper, not on every call. + @contextmanager + def subprocess_runner(self, runner): + prev = self._subprocess_runner + self._subprocess_runner = runner + yield + self._subprocess_runner = prev + + def get_requires_for_build_wheel(self, config_settings=None): + """Identify packages required for building a wheel + + Returns a list of dependency specifications, e.g.: + ["wheel >= 0.25", "setuptools"] + + This does not include requirements specified in pyproject.toml. + It returns the result of calling the equivalently named hook in a + subprocess. + """ + return self._call_hook('get_requires_for_build_wheel', { + 'config_settings': config_settings + }) + + def prepare_metadata_for_build_wheel( + self, metadata_directory, config_settings=None): + """Prepare a *.dist-info folder with metadata for this project. + + Returns the name of the newly created folder. + + If the build backend defines a hook with this name, it will be called + in a subprocess. If not, the backend will be asked to build a wheel, + and the dist-info extracted from that. + """ + return self._call_hook('prepare_metadata_for_build_wheel', { + 'metadata_directory': abspath(metadata_directory), + 'config_settings': config_settings, + }) + + def build_wheel( + self, wheel_directory, config_settings=None, + metadata_directory=None): + """Build a wheel from this project. + + Returns the name of the newly created file. + + In general, this will call the 'build_wheel' hook in the backend. + However, if that was previously called by + 'prepare_metadata_for_build_wheel', and the same metadata_directory is + used, the previously built wheel will be copied to wheel_directory. + """ + if metadata_directory is not None: + metadata_directory = abspath(metadata_directory) + return self._call_hook('build_wheel', { + 'wheel_directory': abspath(wheel_directory), + 'config_settings': config_settings, + 'metadata_directory': metadata_directory, + }) + + def get_requires_for_build_sdist(self, config_settings=None): + """Identify packages required for building a wheel + + Returns a list of dependency specifications, e.g.: + ["setuptools >= 26"] + + This does not include requirements specified in pyproject.toml. + It returns the result of calling the equivalently named hook in a + subprocess. + """ + return self._call_hook('get_requires_for_build_sdist', { + 'config_settings': config_settings + }) + + def build_sdist(self, sdist_directory, config_settings=None): + """Build an sdist from this project. + + Returns the name of the newly created file. + + This calls the 'build_sdist' backend hook in a subprocess. + """ + return self._call_hook('build_sdist', { + 'sdist_directory': abspath(sdist_directory), + 'config_settings': config_settings, + }) + + def _call_hook(self, hook_name, kwargs): + # On Python 2, pytoml returns Unicode values (which is correct) but the + # environment passed to check_call needs to contain string values. We + # convert here by encoding using ASCII (the backend can only contain + # letters, digits and _, . and : characters, and will be used as a + # Python identifier, so non-ASCII content is wrong on Python 2 in + # any case). + if sys.version_info[0] == 2: + build_backend = self.build_backend.encode('ASCII') + else: + build_backend = self.build_backend + + with tempdir() as td: + compat.write_json({'kwargs': kwargs}, pjoin(td, 'input.json'), + indent=2) + + # Run the hook in a subprocess + self._subprocess_runner( + [sys.executable, _in_proc_script, hook_name, td], + cwd=self.source_dir, + extra_environ={'PEP517_BUILD_BACKEND': build_backend} + ) + + data = compat.read_json(pjoin(td, 'output.json')) + if data.get('unsupported'): + raise UnsupportedOperation + if data.get('no_backend'): + raise BackendUnavailable + return data['return_val'] diff --git a/pipenv/vendor/pexpect/__init__.py b/pipenv/vendor/pexpect/__init__.py index 2a18d191..cf7a70d0 100644 --- a/pipenv/vendor/pexpect/__init__.py +++ b/pipenv/vendor/pexpect/__init__.py @@ -75,7 +75,7 @@ if sys.platform != 'win32': from .pty_spawn import spawn, spawnu from .run import run, runu -__version__ = '4.6.0' +__version__ = '4.7.0' __revision__ = '' __all__ = ['ExceptionPexpect', 'EOF', 'TIMEOUT', 'spawn', 'spawnu', 'run', 'runu', 'which', 'split_command_line', '__version__', '__revision__'] diff --git a/pipenv/vendor/pexpect/_async.py b/pipenv/vendor/pexpect/_async.py index bdd515b1..ca2044e1 100644 --- a/pipenv/vendor/pexpect/_async.py +++ b/pipenv/vendor/pexpect/_async.py @@ -1,5 +1,6 @@ import asyncio import errno +import signal from pexpect import EOF @@ -29,6 +30,23 @@ def expect_async(expecter, timeout=None): transport.pause_reading() return expecter.timeout(e) +@asyncio.coroutine +def repl_run_command_async(repl, cmdlines, timeout=-1): + res = [] + repl.child.sendline(cmdlines[0]) + for line in cmdlines[1:]: + yield from repl._expect_prompt(timeout=timeout, async_=True) + res.append(repl.child.before) + repl.child.sendline(line) + + # Command was fully submitted, now wait for the next prompt + prompt_idx = yield from repl._expect_prompt(timeout=timeout, async_=True) + if prompt_idx == 1: + # We got the continuation prompt - command was incomplete + repl.child.kill(signal.SIGINT) + yield from repl._expect_prompt(timeout=1, async_=True) + raise ValueError("Continuation prompt found - input was incomplete:") + return u''.join(res + [repl.child.before]) class PatternWaiter(asyncio.Protocol): transport = None @@ -41,7 +59,7 @@ class PatternWaiter(asyncio.Protocol): if not self.fut.done(): self.fut.set_result(result) self.transport.pause_reading() - + def error(self, exc): if not self.fut.done(): self.fut.set_exception(exc) @@ -49,7 +67,7 @@ class PatternWaiter(asyncio.Protocol): def connection_made(self, transport): self.transport = transport - + def data_received(self, data): spawn = self.expecter.spawn s = spawn._decoder.decode(data) @@ -67,7 +85,7 @@ class PatternWaiter(asyncio.Protocol): except Exception as e: self.expecter.errored() self.error(e) - + def eof_received(self): # N.B. If this gets called, async will close the pipe (the spawn object) # for us @@ -78,7 +96,7 @@ class PatternWaiter(asyncio.Protocol): self.error(e) else: self.found(index) - + def connection_lost(self, exc): if isinstance(exc, OSError) and exc.errno == errno.EIO: # We may get here without eof_received being called, e.g on Linux diff --git a/pipenv/vendor/pexpect/expect.py b/pipenv/vendor/pexpect/expect.py index 1c0275b4..db376d59 100644 --- a/pipenv/vendor/pexpect/expect.py +++ b/pipenv/vendor/pexpect/expect.py @@ -244,7 +244,7 @@ class searcher_re(object): self.eof_index = -1 self.timeout_index = -1 self._searches = [] - for n, s in zip(list(range(len(patterns))), patterns): + for n, s in enumerate(patterns): if s is EOF: self.eof_index = n continue diff --git a/pipenv/vendor/pexpect/pty_spawn.py b/pipenv/vendor/pexpect/pty_spawn.py index e0e2b54f..691c2c63 100644 --- a/pipenv/vendor/pexpect/pty_spawn.py +++ b/pipenv/vendor/pexpect/pty_spawn.py @@ -430,61 +430,83 @@ class spawn(SpawnBase): available right away then one character will be returned immediately. It will not wait for 30 seconds for another 99 characters to come in. - This is a wrapper around os.read(). It uses select.select() to - implement the timeout. ''' + On the other hand, if there are bytes available to read immediately, + all those bytes will be read (up to the buffer size). So, if the + buffer size is 1 megabyte and there is 1 megabyte of data available + to read, the buffer will be filled, regardless of timeout. + + This is a wrapper around os.read(). It uses select.select() or + select.poll() to implement the timeout. ''' if self.closed: raise ValueError('I/O operation on closed file.') + if self.use_poll: + def select(timeout): + return poll_ignore_interrupts([self.child_fd], timeout) + else: + def select(timeout): + return select_ignore_interrupts([self.child_fd], [], [], timeout)[0] + + # If there is data available to read right now, read as much as + # we can. We do this to increase performance if there are a lot + # of bytes to be read. This also avoids calling isalive() too + # often. See also: + # * https://github.com/pexpect/pexpect/pull/304 + # * http://trac.sagemath.org/ticket/10295 + if select(0): + try: + incoming = super(spawn, self).read_nonblocking(size) + except EOF: + # Maybe the child is dead: update some attributes in that case + self.isalive() + raise + while len(incoming) < size and select(0): + try: + incoming += super(spawn, self).read_nonblocking(size - len(incoming)) + except EOF: + # Maybe the child is dead: update some attributes in that case + self.isalive() + # Don't raise EOF, just return what we read so far. + return incoming + return incoming + if timeout == -1: timeout = self.timeout - # Note that some systems such as Solaris do not give an EOF when - # the child dies. In fact, you can still try to read - # from the child_fd -- it will block forever or until TIMEOUT. - # For this case, I test isalive() before doing any reading. - # If isalive() is false, then I pretend that this is the same as EOF. if not self.isalive(): - # timeout of 0 means "poll" - if self.use_poll: - r = poll_ignore_interrupts([self.child_fd], timeout) - else: - r, w, e = select_ignore_interrupts([self.child_fd], [], [], 0) - if not r: - self.flag_eof = True - raise EOF('End Of File (EOF). Braindead platform.') + # The process is dead, but there may or may not be data + # available to read. Note that some systems such as Solaris + # do not give an EOF when the child dies. In fact, you can + # still try to read from the child_fd -- it will block + # forever or until TIMEOUT. For that reason, it's important + # to do this check before calling select() with timeout. + if select(0): + return super(spawn, self).read_nonblocking(size) + self.flag_eof = True + raise EOF('End Of File (EOF). Braindead platform.') elif self.__irix_hack: # Irix takes a long time before it realizes a child was terminated. + # Make sure that the timeout is at least 2 seconds. # FIXME So does this mean Irix systems are forced to always have # FIXME a 2 second delay when calling read_nonblocking? That sucks. - if self.use_poll: - r = poll_ignore_interrupts([self.child_fd], timeout) - else: - r, w, e = select_ignore_interrupts([self.child_fd], [], [], 2) - if not r and not self.isalive(): - self.flag_eof = True - raise EOF('End Of File (EOF). Slow platform.') - if self.use_poll: - r = poll_ignore_interrupts([self.child_fd], timeout) - else: - r, w, e = select_ignore_interrupts( - [self.child_fd], [], [], timeout - ) + if timeout is not None and timeout < 2: + timeout = 2 - if not r: - if not self.isalive(): - # Some platforms, such as Irix, will claim that their - # processes are alive; timeout on the select; and - # then finally admit that they are not alive. - self.flag_eof = True - raise EOF('End of File (EOF). Very slow platform.') - else: - raise TIMEOUT('Timeout exceeded.') - - if self.child_fd in r: + # Because of the select(0) check above, we know that no data + # is available right now. But if a non-zero timeout is given + # (possibly timeout=None), we call select() with a timeout. + if (timeout != 0) and select(timeout): return super(spawn, self).read_nonblocking(size) - raise ExceptionPexpect('Reached an unexpected state.') # pragma: no cover + if not self.isalive(): + # Some platforms, such as Irix, will claim that their + # processes are alive; timeout on the select; and + # then finally admit that they are not alive. + self.flag_eof = True + raise EOF('End of File (EOF). Very slow platform.') + else: + raise TIMEOUT('Timeout exceeded.') def write(self, s): '''This is similar to send() except that there is no return value. diff --git a/pipenv/vendor/pexpect/pxssh.py b/pipenv/vendor/pexpect/pxssh.py index ef2e9118..3d53bd97 100644 --- a/pipenv/vendor/pexpect/pxssh.py +++ b/pipenv/vendor/pexpect/pxssh.py @@ -109,7 +109,7 @@ class pxssh (spawn): username = raw_input('username: ') password = getpass.getpass('password: ') s.login (hostname, username, password) - + `debug_command_string` is only for the test suite to confirm that the string generated for SSH is correct, using this will not allow you to do anything other than get a string back from `pxssh.pxssh.login()`. @@ -118,12 +118,12 @@ class pxssh (spawn): def __init__ (self, timeout=30, maxread=2000, searchwindowsize=None, logfile=None, cwd=None, env=None, ignore_sighup=True, echo=True, options={}, encoding=None, codec_errors='strict', - debug_command_string=False): + debug_command_string=False, use_poll=False): spawn.__init__(self, None, timeout=timeout, maxread=maxread, searchwindowsize=searchwindowsize, logfile=logfile, cwd=cwd, env=env, ignore_sighup=ignore_sighup, echo=echo, - encoding=encoding, codec_errors=codec_errors) + encoding=encoding, codec_errors=codec_errors, use_poll=use_poll) self.name = '' @@ -154,7 +154,7 @@ class pxssh (spawn): # Unsetting SSH_ASKPASS on the remote side doesn't disable it! Annoying! #self.SSH_OPTS = "-x -o'RSAAuthentication=no' -o 'PubkeyAuthentication=no'" self.force_password = False - + self.debug_command_string = debug_command_string # User defined SSH options, eg, @@ -220,7 +220,7 @@ class pxssh (spawn): can take 12 seconds. Low latency connections are more likely to fail with a low sync_multiplier. Best case sync time gets worse with a high sync multiplier (500 ms with default). ''' - + # All of these timing pace values are magic. # I came up with these based on what seemed reliable for # connecting to a heavily loaded machine I have. @@ -253,20 +253,19 @@ class pxssh (spawn): ### TODO: This is getting messy and I'm pretty sure this isn't perfect. ### TODO: I need to draw a flow chart for this. ### TODO: Unit tests for SSH tunnels, remote SSH command exec, disabling original prompt sync - def login (self, server, username, password='', terminal_type='ansi', + def login (self, server, username=None, password='', terminal_type='ansi', original_prompt=r"[#$]", login_timeout=10, port=None, auto_prompt_reset=True, ssh_key=None, quiet=True, sync_multiplier=1, check_local_ip=True, password_regex=r'(?i)(?:password:)|(?:passphrase for key)', ssh_tunnels={}, spawn_local_ssh=True, - sync_original_prompt=True, ssh_config=None): + sync_original_prompt=True, ssh_config=None, cmd='ssh'): '''This logs the user into the given server. - It uses - 'original_prompt' to try to find the prompt right after login. When it - finds the prompt it immediately tries to reset the prompt to something - more easily matched. The default 'original_prompt' is very optimistic - and is easily fooled. It's more reliable to try to match the original + It uses 'original_prompt' to try to find the prompt right after login. + When it finds the prompt it immediately tries to reset the prompt to + something more easily matched. The default 'original_prompt' is very + optimistic and is easily fooled. It's more reliable to try to match the original prompt as exactly as possible to prevent false matches by server strings such as the "Message Of The Day". On many systems you can disable the MOTD on the remote server by creating a zero-length file @@ -284,27 +283,31 @@ class pxssh (spawn): uses a unique prompt in the :meth:`prompt` method. If the original prompt is not reset then this will disable the :meth:`prompt` method unless you manually set the :attr:`PROMPT` attribute. - + Set ``password_regex`` if there is a MOTD message with `password` in it. Changing this is like playing in traffic, don't (p)expect it to match straight away. - + If you require to connect to another SSH server from the your original SSH connection set ``spawn_local_ssh`` to `False` and this will use your current session to do so. Setting this option to `False` and not having an active session will trigger an error. - + Set ``ssh_key`` to a file path to an SSH private key to use that SSH key for the session authentication. Set ``ssh_key`` to `True` to force passing the current SSH authentication socket to the desired ``hostname``. - + Set ``ssh_config`` to a file path string of an SSH client config file to pass that file to the client to handle itself. You may set any options you wish in here, however doing so will require you to post extra information that you may not want to if you run into issues. + + Alter the ``cmd`` to change the ssh client used, or to prepend it with network + namespaces. For example ```cmd="ip netns exec vlan2 ssh"``` to execute the ssh in + network namespace named ```vlan```. ''' - + session_regex_array = ["(?i)are you sure you want to continue connecting", original_prompt, password_regex, "(?i)permission denied", "(?i)terminal type", TIMEOUT] session_init_regex_array = [] session_init_regex_array.extend(session_regex_array) @@ -320,7 +323,7 @@ class pxssh (spawn): if ssh_config is not None: if spawn_local_ssh and not os.path.isfile(ssh_config): raise ExceptionPxssh('SSH config does not exist or is not a file.') - ssh_options = ssh_options + '-F ' + ssh_config + ssh_options = ssh_options + ' -F ' + ssh_config if port is not None: ssh_options = ssh_options + ' -p %s'%(str(port)) if ssh_key is not None: @@ -331,7 +334,7 @@ class pxssh (spawn): if spawn_local_ssh and not os.path.isfile(ssh_key): raise ExceptionPxssh('private ssh key does not exist or is not a file.') ssh_options = ssh_options + ' -i %s' % (ssh_key) - + # SSH tunnels, make sure you know what you're putting into the lists # under each heading. Do not expect these to open 100% of the time, # The port you're requesting might be bound. @@ -354,7 +357,42 @@ class pxssh (spawn): if spawn_local_ssh==False: tunnel = quote(str(tunnel)) ssh_options = ssh_options + ' -' + cmd_type + ' ' + str(tunnel) - cmd = "ssh %s -l %s %s" % (ssh_options, username, server) + + if username is not None: + ssh_options = ssh_options + ' -l ' + username + elif ssh_config is None: + raise TypeError('login() needs either a username or an ssh_config') + else: # make sure ssh_config has an entry for the server with a username + with open(ssh_config, 'rt') as f: + lines = [l.strip() for l in f.readlines()] + + server_regex = r'^Host\s+%s\s*$' % server + user_regex = r'^User\s+\w+\s*$' + config_has_server = False + server_has_username = False + for line in lines: + if not config_has_server and re.match(server_regex, line, re.IGNORECASE): + config_has_server = True + elif config_has_server and 'hostname' in line.lower(): + pass + elif config_has_server and 'host' in line.lower(): + server_has_username = False # insurance + break # we have left the relevant section + elif config_has_server and re.match(user_regex, line, re.IGNORECASE): + server_has_username = True + break + + if lines: + del line + + del lines + + if not config_has_server: + raise TypeError('login() ssh_config has no Host entry for %s' % server) + elif not server_has_username: + raise TypeError('login() ssh_config has no user entry for %s' % server) + + cmd += " %s %s" % (ssh_options, server) if self.debug_command_string: return(cmd) diff --git a/pipenv/vendor/pexpect/replwrap.py b/pipenv/vendor/pexpect/replwrap.py index ed0e657d..c930f1e4 100644 --- a/pipenv/vendor/pexpect/replwrap.py +++ b/pipenv/vendor/pexpect/replwrap.py @@ -61,11 +61,11 @@ class REPLWrapper(object): self.child.expect(orig_prompt) self.child.sendline(prompt_change) - def _expect_prompt(self, timeout=-1): + def _expect_prompt(self, timeout=-1, async_=False): return self.child.expect_exact([self.prompt, self.continuation_prompt], - timeout=timeout) + timeout=timeout, async_=async_) - def run_command(self, command, timeout=-1): + def run_command(self, command, timeout=-1, async_=False): """Send a command to the REPL, wait for and return output. :param str command: The command to send. Trailing newlines are not needed. @@ -75,6 +75,10 @@ class REPLWrapper(object): :param int timeout: How long to wait for the next prompt. -1 means the default from the :class:`pexpect.spawn` object (default 30 seconds). None means to wait indefinitely. + :param bool async_: On Python 3.4, or Python 3.3 with asyncio + installed, passing ``async_=True`` will make this return an + :mod:`asyncio` Future, which you can yield from to get the same + result that this method would normally give directly. """ # Split up multiline commands and feed them in bit-by-bit cmdlines = command.splitlines() @@ -84,6 +88,10 @@ class REPLWrapper(object): if not cmdlines: raise ValueError("No command was given") + if async_: + from ._async import repl_run_command_async + return repl_run_command_async(self, cmdlines, timeout) + res = [] self.child.sendline(cmdlines[0]) for line in cmdlines[1:]: diff --git a/pipenv/vendor/pip_shims/__init__.py b/pipenv/vendor/pip_shims/__init__.py index 22bf130f..9320a437 100644 --- a/pipenv/vendor/pip_shims/__init__.py +++ b/pipenv/vendor/pip_shims/__init__.py @@ -3,7 +3,7 @@ from __future__ import absolute_import import sys -__version__ = '0.3.2' +__version__ = "0.3.3" from . import shims diff --git a/pipenv/vendor/pip_shims/shims.py b/pipenv/vendor/pip_shims/shims.py index 34de6906..4cad473a 100644 --- a/pipenv/vendor/pip_shims/shims.py +++ b/pipenv/vendor/pip_shims/shims.py @@ -1,26 +1,38 @@ # -*- coding=utf-8 -*- -from collections import namedtuple -from contextlib import contextmanager import importlib import os import sys - +from collections import namedtuple +from contextlib import contextmanager import six -six.add_move(six.MovedAttribute("Callable", "collections", "collections.abc")) -from six.moves import Callable + +# format: off +six.add_move(six.MovedAttribute("Callable", "collections", "collections.abc")) # noqa +from six.moves import Callable # type: ignore # noqa # isort:skip + +# format: on class _shims(object): - CURRENT_PIP_VERSION = "18.1" + CURRENT_PIP_VERSION = "19.1.1" BASE_IMPORT_PATH = os.environ.get("PIP_SHIMS_BASE_MODULE", "pip") path_info = namedtuple("PathInfo", "path start_version end_version") def __dir__(self): result = list(self._locations.keys()) + list(self.__dict__.keys()) - result.extend(('__file__', '__doc__', '__all__', - '__docformat__', '__name__', '__path__', - '__package__', '__version__')) + result.extend( + ( + "__file__", + "__doc__", + "__all__", + "__docformat__", + "__name__", + "__path__", + "__package__", + "__version__", + ) + ) return result @classmethod @@ -34,13 +46,14 @@ class _shims(object): def __init__(self): # from .utils import _parse, get_package, STRING_TYPES from . import utils + self.utils = utils self._parse = utils._parse self.get_package = utils.get_package self.STRING_TYPES = utils.STRING_TYPES self._modules = { "pip": importlib.import_module(self.BASE_IMPORT_PATH), - "pip_shims.utils": utils + "pip_shims.utils": utils, } self.pip_version = getattr(self._modules["pip"], "__version__") version_types = ["post", "pre", "dev", "rc"] @@ -62,15 +75,15 @@ class _shims(object): ), "cmdoptions": ( ("cli.cmdoptions", "18.1", "9999"), - ("cmdoptions", "7.0.0", "18.0") + ("cmdoptions", "7.0.0", "18.0"), ), "Command": ( ("cli.base_command.Command", "18.1", "9999"), - ("basecommand.Command", "7.0.0", "18.0") + ("basecommand.Command", "7.0.0", "18.0"), ), "ConfigOptionParser": ( ("cli.parser.ConfigOptionParser", "18.1", "9999"), - ("baseparser.ConfigOptionParser", "7.0.0", "18.0") + ("baseparser.ConfigOptionParser", "7.0.0", "18.0"), ), "DistributionNotFound": ("exceptions.DistributionNotFound", "7.0.0", "9999"), "FAVORITE_HASH": ("utils.hashes.FAVORITE_HASH", "7.0.0", "9999"), @@ -80,70 +93,130 @@ class _shims(object): ), "FrozenRequirement": ( ("FrozenRequirement", "7.0.0", "9.0.3"), - ("operations.freeze.FrozenRequirement", "10.0.0", "9999") + ("operations.freeze.FrozenRequirement", "10.0.0", "9999"), ), "get_installed_distributions": ( ("utils.misc.get_installed_distributions", "10", "9999"), - ("utils.get_installed_distributions", "7", "9.0.3") + ("utils.get_installed_distributions", "7", "9.0.3"), ), "index_group": ( ("cli.cmdoptions.index_group", "18.1", "9999"), - ("cmdoptions.index_group", "7.0.0", "18.0") + ("cmdoptions.index_group", "7.0.0", "18.0"), ), "InstallRequirement": ("req.req_install.InstallRequirement", "7.0.0", "9999"), "InstallationError": ("exceptions.InstallationError", "7.0.0", "9999"), "UninstallationError": ("exceptions.UninstallationError", "7.0.0", "9999"), "DistributionNotFound": ("exceptions.DistributionNotFound", "7.0.0", "9999"), - "RequirementsFileParseError": ("exceptions.RequirementsFileParseError", "7.0.0", "9999"), - "BestVersionAlreadyInstalled": ("exceptions.BestVersionAlreadyInstalled", "7.0.0", "9999"), + "RequirementsFileParseError": ( + "exceptions.RequirementsFileParseError", + "7.0.0", + "9999", + ), + "BestVersionAlreadyInstalled": ( + "exceptions.BestVersionAlreadyInstalled", + "7.0.0", + "9999", + ), "BadCommand": ("exceptions.BadCommand", "7.0.0", "9999"), "CommandError": ("exceptions.CommandError", "7.0.0", "9999"), - "PreviousBuildDirError": ("exceptions.PreviousBuildDirError", "7.0.0", "9999"), + "PreviousBuildDirError": ( + "exceptions.PreviousBuildDirError", + "7.0.0", + "9999", + ), "install_req_from_editable": ( ("req.constructors.install_req_from_editable", "18.1", "9999"), - ("req.req_install.InstallRequirement.from_editable", "7.0.0", "18.0") + ("req.req_install.InstallRequirement.from_editable", "7.0.0", "18.0"), ), "install_req_from_line": ( ("req.constructors.install_req_from_line", "18.1", "9999"), - ("req.req_install.InstallRequirement.from_line", "7.0.0", "18.0") + ("req.req_install.InstallRequirement.from_line", "7.0.0", "18.0"), ), "is_archive_file": ("download.is_archive_file", "7.0.0", "9999"), "is_file_url": ("download.is_file_url", "7.0.0", "9999"), "unpack_url": ("download.unpack_url", "7.0.0", "9999"), "is_installable_dir": ( ("utils.misc.is_installable_dir", "10.0.0", "9999"), - ("utils.is_installable_dir", "7.0.0", "9.0.3") + ("utils.is_installable_dir", "7.0.0", "9.0.3"), ), "Link": ("index.Link", "7.0.0", "9999"), "make_abstract_dist": ( - ("operations.prepare.make_abstract_dist", "10.0.0", "9999"), - ("req.req_set.make_abstract_dist", "7.0.0", "9.0.3") + ( + "distributions.make_distribution_for_install_requirement", + "19.1.2", + "9999", + ), + ("operations.prepare.make_abstract_dist", "10.0.0", "19.1.1"), + ("req.req_set.make_abstract_dist", "7.0.0", "9.0.3"), + ), + "make_distribution_for_install_requirement": ( + "distributions.make_distribution_for_install_requirement", + "19.1.2", + "9999", ), "make_option_group": ( ("cli.cmdoptions.make_option_group", "18.1", "9999"), - ("cmdoptions.make_option_group", "7.0.0", "18.0") + ("cmdoptions.make_option_group", "7.0.0", "18.0"), ), "PackageFinder": ("index.PackageFinder", "7.0.0", "9999"), + "CandidateEvaluator": ("index.CandidateEvaluator", "19.1", "9999"), "parse_requirements": ("req.req_file.parse_requirements", "7.0.0", "9999"), - "parse_version": ("index.parse_version", "7.0.0", "9999"), "path_to_url": ("download.path_to_url", "7.0.0", "9999"), "PipError": ("exceptions.PipError", "7.0.0", "9999"), - "RequirementPreparer": ("operations.prepare.RequirementPreparer", "7", "9999"), + "RequirementPreparer": ( + "operations.prepare.RequirementPreparer", + "7", + "9999", + ), "RequirementSet": ("req.req_set.RequirementSet", "7.0.0", "9999"), "RequirementTracker": ("req.req_tracker.RequirementTracker", "7.0.0", "9999"), - "Resolver": ("resolve.Resolver", "7.0.0", "9999"), + "Resolver": ( + ("resolve.Resolver", "7.0.0", "19.1.1"), + ("legacy_resolve.Resolver", "19.1.2", "9999"), + ), "SafeFileCache": ("download.SafeFileCache", "7.0.0", "9999"), "UninstallPathSet": ("req.req_uninstall.UninstallPathSet", "7.0.0", "9999"), "url_to_path": ("download.url_to_path", "7.0.0", "9999"), "USER_CACHE_DIR": ("locations.USER_CACHE_DIR", "7.0.0", "9999"), - "VcsSupport": ("vcs.VcsSupport", "7.0.0", "9999"), + "VcsSupport": ( + ("vcs.VcsSupport", "7.0.0", "19.1.1"), + ("vcs.versioncontrol.VcsSupport", "19.2", "9999"), + ), "Wheel": ("wheel.Wheel", "7.0.0", "9999"), "WheelCache": ( ("cache.WheelCache", "10.0.0", "9999"), - ("wheel.WheelCache", "7", "9.0.3") + ("wheel.WheelCache", "7", "9.0.3"), ), "WheelBuilder": ("wheel.WheelBuilder", "7.0.0", "9999"), + "AbstractDistribution": ( + "distributions.base.AbstractDistribution", + "19.1.2", + "9999", + ), + "InstalledDistribution": ( + "distributions.installed.InstalledDistribution", + "19.1.2", + "9999", + ), + "SourceDistribution": ( + ("req.req_set.IsSDist", "7.0.0", "9.0.3"), + ("operations.prepare.IsSDist", "10.0.0", "19.1.1"), + ("distributions.source.SourceDistribution", "19.1.2", "9999"), + ), + "WheelDistribution": ( + "distributions.wheel.WheelDistribution", + "19.1.2", + "9999", + ), "PyPI": ("models.index.PyPI", "7.0.0", "9999"), + "stdlib_pkgs": ( + ("utils.compat.stdlib_pkgs", "18.1", "9999"), + ("compat.stdlib_pkgs", "7", "18.0"), + ), + "DEV_PKGS": ( + ("commands.freeze.DEV_PKGS", "9.0.0", "9999"), + ({"setuptools", "pip", "distribute", "wheel"}, "7.0.0", "8.1.2"), + ), } def _ensure_methods(self, cls, classname, *methods): @@ -151,6 +224,7 @@ class _shims(object): if all(getattr(cls, m, None) for m in method_names): return cls new_functions = {} + class BaseFunc(Callable): def __init__(self, func_base, name, *args, **kwargs): self.func = func_base @@ -163,11 +237,7 @@ class _shims(object): new_functions[method_name] = classmethod(BaseFunc(fn, method_name)) if six.PY2: classname = classname.encode(sys.getdefaultencoding()) - type_ = type( - classname, - (cls,), - new_functions - ) + type_ = type(classname, (cls,), new_functions) return type_ def _get_module_paths(self, module, base_path=None): @@ -187,22 +257,23 @@ class _shims(object): new_to_old = {} for method_name, new_method_name in self._moves.get(original_target, {}).items(): module_paths = self._get_module_paths(new_method_name) - target = next(iter( - sorted(set([ - tgt for mod, tgt in map(self.get_package, module_paths) - ]))), None + target = next( + iter( + sorted(set([tgt for mod, tgt in map(self.get_package, module_paths)])) + ), + None, ) old_to_new[method_name] = { "target": target, "name": new_method_name, "location": self._locations[new_method_name], - "module": self._import(self._locations[new_method_name]) + "module": self._import(self._locations[new_method_name]), } new_to_old[new_method_name] = { "target": original_target, "name": method_name, "location": self._locations[original_target], - "module": original_import + "module": original_import, } return (old_to_new, new_to_old) @@ -218,9 +289,7 @@ class _shims(object): imported = self._modules[new_target] = new_to_old[new_name]["module"] method_map.append((old_method, remapped["module"])) if getattr(imported, "__class__", "") == type: - imported = self._ensure_methods( - imported, new_target, *method_map - ) + imported = self._ensure_methods(imported, new_target, *method_map) self._modules[new_target] = imported if imported: return imported @@ -231,8 +300,7 @@ class _shims(object): self.get_package(pth) for pth in self._get_module_paths(search_pth) ] moved_methods = [ - (base, target_cls) for base, target_cls - in module_paths if target_cls in moves + (base, target_cls) for base, target_cls in module_paths if target_cls in moves ] return next(iter(moved_methods), None) @@ -280,6 +348,8 @@ class _shims(object): def import_module(self, module): if module in self._modules: return self._modules[module] + if not isinstance(module, six.string_types): + return module try: imported = importlib.import_module(module) except ImportError: @@ -299,7 +369,8 @@ class _shims(object): for m, package_name in map(self.get_package, modules) ] imports = [ - getattr(m, pkg, self.none_or_ctxmanager(pkg)) for pkg, m in modules + getattr(m, pkg, self.none_or_ctxmanager(pkg)) + for pkg, m in modules if m is not None ] return next(iter(imports), None) @@ -327,15 +398,19 @@ class _shims(object): def get_pathinfo(self, module_path): assert isinstance(module_path, (list, tuple)) module_path, start_version, end_version = module_path - return self.path_info(module_path, self._parse(start_version), self._parse(end_version)) + return self.path_info( + module_path, self._parse(start_version), self._parse(end_version) + ) old_module = sys.modules[__name__] if __name__ in sys.modules else None module = sys.modules[__name__] = _shims() -module.__dict__.update({ - '__file__': __file__, - '__package__': __package__, - '__doc__': __doc__, - '__all__': module.__all__, - '__name__': __name__, -}) +module.__dict__.update( + { + "__file__": __file__, + "__package__": __package__, + "__doc__": __doc__, + "__all__": module.__all__, + "__name__": __name__, + } +) diff --git a/pipenv/vendor/pipdeptree.py b/pipenv/vendor/pipdeptree.py index 2082fc8a..899118cc 100644 --- a/pipenv/vendor/pipdeptree.py +++ b/pipenv/vendor/pipdeptree.py @@ -22,7 +22,7 @@ import pkg_resources # from graphviz import backend, Digraph -__version__ = '0.13.0' +__version__ = '0.13.2' flatten = chain.from_iterable @@ -127,6 +127,13 @@ def guess_version(pkg_key, default='?'): return getattr(m, '__version__', default) +def frozen_req_from_dist(dist): + try: + return FrozenRequirement.from_dist(dist) + except TypeError: + return FrozenRequirement.from_dist(dist, []) + + class Package(object): """Abstract class for wrappers around objects that pip returns. @@ -154,7 +161,7 @@ class Package(object): @staticmethod def frozen_repr(obj): - fr = FrozenRequirement.from_dist(obj, []) + fr = frozen_req_from_dist(obj) return str(fr).strip() def __getattr__(self, key): @@ -563,7 +570,7 @@ def _get_args(): def main(): args = _get_args() pkgs = get_installed_distributions(local_only=args.local_only, - user_only=args.user_only) + user_only=args.user_only) dist_index = build_dist_index(pkgs) tree = construct_tree(dist_index) diff --git a/pipenv/vendor/plette/__init__.py b/pipenv/vendor/plette/__init__.py index 8099f0b1..5daf460c 100644 --- a/pipenv/vendor/plette/__init__.py +++ b/pipenv/vendor/plette/__init__.py @@ -3,7 +3,7 @@ __all__ = [ "Lockfile", "Pipfile", ] -__version__ = '0.2.2' +__version__ = '0.2.3.dev0' from .lockfiles import Lockfile from .pipfiles import Pipfile diff --git a/pipenv/vendor/plette/models/base.py b/pipenv/vendor/plette/models/base.py index d70752ee..fad0d09e 100644 --- a/pipenv/vendor/plette/models/base.py +++ b/pipenv/vendor/plette/models/base.py @@ -22,7 +22,7 @@ def validate(cls, data): v = VALIDATORS[key] except KeyError: v = VALIDATORS[key] = cerberus.Validator(schema, allow_unknown=True) - if v.validate(data, normalize=False): + if v.validate(dict(data), normalize=False): return raise ValidationError(data, v) diff --git a/pipenv/vendor/pyparsing.py b/pipenv/vendor/pyparsing.py index cf38419b..ab804d53 100644 --- a/pipenv/vendor/pyparsing.py +++ b/pipenv/vendor/pyparsing.py @@ -1,6 +1,7 @@ +#-*- coding: utf-8 -*- # module pyparsing.py # -# Copyright (c) 2003-2018 Paul T. McGuire +# Copyright (c) 2003-2019 Paul T. McGuire # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -27,15 +28,18 @@ __doc__ = \ pyparsing module - Classes and methods to define and execute parsing grammars ============================================================================= -The pyparsing module is an alternative approach to creating and executing simple grammars, -vs. the traditional lex/yacc approach, or the use of regular expressions. With pyparsing, you -don't need to learn a new syntax for defining grammars or matching expressions - the parsing module -provides a library of classes that you use to construct the grammar directly in Python. +The pyparsing module is an alternative approach to creating and +executing simple grammars, vs. the traditional lex/yacc approach, or the +use of regular expressions. With pyparsing, you don't need to learn +a new syntax for defining grammars or matching expressions - the parsing +module provides a library of classes that you use to construct the +grammar directly in Python. -Here is a program to parse "Hello, World!" (or any greeting of the form -C{", !"}), built up using L{Word}, L{Literal}, and L{And} elements -(L{'+'} operator gives L{And} expressions, strings are auto-converted to -L{Literal} expressions):: +Here is a program to parse "Hello, World!" (or any greeting of the form +``", !"``), built up using :class:`Word`, +:class:`Literal`, and :class:`And` elements +(the :class:`'+'` operators create :class:`And` expressions, +and the strings are auto-converted to :class:`Literal` expressions):: from pyparsing import Word, alphas @@ -49,33 +53,48 @@ The program outputs the following:: Hello, World! -> ['Hello', ',', 'World', '!'] -The Python representation of the grammar is quite readable, owing to the self-explanatory -class names, and the use of '+', '|' and '^' operators. +The Python representation of the grammar is quite readable, owing to the +self-explanatory class names, and the use of '+', '|' and '^' operators. -The L{ParseResults} object returned from L{ParserElement.parseString} can be accessed as a nested list, a dictionary, or an -object with named attributes. +The :class:`ParseResults` object returned from +:class:`ParserElement.parseString` can be +accessed as a nested list, a dictionary, or an object with named +attributes. -The pyparsing module handles some of the problems that are typically vexing when writing text parsers: - - extra or missing whitespace (the above program will also handle "Hello,World!", "Hello , World !", etc.) - - quoted strings - - embedded comments +The pyparsing module handles some of the problems that are typically +vexing when writing text parsers: + + - extra or missing whitespace (the above program will also handle + "Hello,World!", "Hello , World !", etc.) + - quoted strings + - embedded comments Getting Started - ----------------- -Visit the classes L{ParserElement} and L{ParseResults} to see the base classes that most other pyparsing +Visit the classes :class:`ParserElement` and :class:`ParseResults` to +see the base classes that most other pyparsing classes inherit from. Use the docstrings for examples of how to: - - construct literal match expressions from L{Literal} and L{CaselessLiteral} classes - - construct character word-group expressions using the L{Word} class - - see how to create repetitive expressions using L{ZeroOrMore} and L{OneOrMore} classes - - use L{'+'}, L{'|'}, L{'^'}, and L{'&'} operators to combine simple expressions into more complex ones - - associate names with your parsed results using L{ParserElement.setResultsName} - - find some helpful expression short-cuts like L{delimitedList} and L{oneOf} - - find more useful common expressions in the L{pyparsing_common} namespace class + + - construct literal match expressions from :class:`Literal` and + :class:`CaselessLiteral` classes + - construct character word-group expressions using the :class:`Word` + class + - see how to create repetitive expressions using :class:`ZeroOrMore` + and :class:`OneOrMore` classes + - use :class:`'+'`, :class:`'|'`, :class:`'^'`, + and :class:`'&'` operators to combine simple expressions into + more complex ones + - associate names with your parsed results using + :class:`ParserElement.setResultsName` + - find some helpful expression short-cuts like :class:`delimitedList` + and :class:`oneOf` + - find more useful common expressions in the :class:`pyparsing_common` + namespace class """ -__version__ = "2.2.2" -__versionTime__ = "29 Sep 2018 15:58 UTC" +__version__ = "2.3.1" +__versionTime__ = "09 Jan 2019 23:26 UTC" __author__ = "Paul McGuire " import string @@ -91,6 +110,12 @@ import traceback import types from datetime import datetime +try: + # Python 3 + from itertools import filterfalse +except ImportError: + from itertools import ifilterfalse as filterfalse + try: from _thread import RLock except ImportError: @@ -113,27 +138,33 @@ except ImportError: except ImportError: _OrderedDict = None +try: + from types import SimpleNamespace +except ImportError: + class SimpleNamespace: pass + + #~ sys.stderr.write( "testing pyparsing module, version %s, %s\n" % (__version__,__versionTime__ ) ) __all__ = [ 'And', 'CaselessKeyword', 'CaselessLiteral', 'CharsNotIn', 'Combine', 'Dict', 'Each', 'Empty', 'FollowedBy', 'Forward', 'GoToColumn', 'Group', 'Keyword', 'LineEnd', 'LineStart', 'Literal', -'MatchFirst', 'NoMatch', 'NotAny', 'OneOrMore', 'OnlyOnce', 'Optional', 'Or', +'PrecededBy', 'MatchFirst', 'NoMatch', 'NotAny', 'OneOrMore', 'OnlyOnce', 'Optional', 'Or', 'ParseBaseException', 'ParseElementEnhance', 'ParseException', 'ParseExpression', 'ParseFatalException', 'ParseResults', 'ParseSyntaxException', 'ParserElement', 'QuotedString', 'RecursiveGrammarException', -'Regex', 'SkipTo', 'StringEnd', 'StringStart', 'Suppress', 'Token', 'TokenConverter', -'White', 'Word', 'WordEnd', 'WordStart', 'ZeroOrMore', +'Regex', 'SkipTo', 'StringEnd', 'StringStart', 'Suppress', 'Token', 'TokenConverter', +'White', 'Word', 'WordEnd', 'WordStart', 'ZeroOrMore', 'Char', 'alphanums', 'alphas', 'alphas8bit', 'anyCloseTag', 'anyOpenTag', 'cStyleComment', 'col', 'commaSeparatedList', 'commonHTMLEntity', 'countedArray', 'cppStyleComment', 'dblQuotedString', 'dblSlashComment', 'delimitedList', 'dictOf', 'downcaseTokens', 'empty', 'hexnums', 'htmlComment', 'javaStyleComment', 'line', 'lineEnd', 'lineStart', 'lineno', 'makeHTMLTags', 'makeXMLTags', 'matchOnlyAtCol', 'matchPreviousExpr', 'matchPreviousLiteral', 'nestedExpr', 'nullDebugAction', 'nums', 'oneOf', 'opAssoc', 'operatorPrecedence', 'printables', -'punc8bit', 'pythonStyleComment', 'quotedString', 'removeQuotes', 'replaceHTMLEntity', +'punc8bit', 'pythonStyleComment', 'quotedString', 'removeQuotes', 'replaceHTMLEntity', 'replaceWith', 'restOfLine', 'sglQuotedString', 'srange', 'stringEnd', 'stringStart', 'traceParseAction', 'unicodeString', 'upcaseTokens', 'withAttribute', 'indentedBlock', 'originalTextFor', 'ungroup', 'infixNotation','locatedExpr', 'withClass', -'CloseMatch', 'tokenMap', 'pyparsing_common', +'CloseMatch', 'tokenMap', 'pyparsing_common', 'pyparsing_unicode', 'unicode_set', ] system_version = tuple(sys.version_info)[:3] @@ -142,6 +173,7 @@ if PY_3: _MAX_INT = sys.maxsize basestring = str unichr = chr + unicode = str _ustr = str # build list of single arg builtins, that can be used as parse actions @@ -152,9 +184,11 @@ else: range = xrange def _ustr(obj): - """Drop-in replacement for str(obj) that tries to be Unicode friendly. It first tries - str(obj). If that fails with a UnicodeEncodeError, then it tries unicode(obj). It - then < returns the unicode object | encodes it with the default encoding | ... >. + """Drop-in replacement for str(obj) that tries to be Unicode + friendly. It first tries str(obj). If that fails with + a UnicodeEncodeError, then it tries unicode(obj). It then + < returns the unicode object | encodes it with the default + encoding | ... >. """ if isinstance(obj,unicode): return obj @@ -179,9 +213,9 @@ else: singleArgBuiltins.append(getattr(__builtin__,fname)) except AttributeError: continue - + _generatorType = type((y for y in range(1))) - + def _xml_escape(data): """Escape &, <, >, ", ', etc. in a string of data.""" @@ -192,9 +226,6 @@ def _xml_escape(data): data = data.replace(from_, to_) return data -class _Constants(object): - pass - alphas = string.ascii_uppercase + string.ascii_lowercase nums = "0123456789" hexnums = nums + "ABCDEFabcdef" @@ -220,16 +251,16 @@ class ParseBaseException(Exception): @classmethod def _from_exception(cls, pe): """ - internal factory method to simplify creating one type of ParseException + internal factory method to simplify creating one type of ParseException from another - avoids having __init__ signature conflicts among subclasses """ return cls(pe.pstr, pe.loc, pe.msg, pe.parserElement) def __getattr__( self, aname ): """supported attributes by name are: - - lineno - returns the line number of the exception text - - col - returns the column number of the exception text - - line - returns the line containing the exception text + - lineno - returns the line number of the exception text + - col - returns the column number of the exception text + - line - returns the line containing the exception text """ if( aname == "lineno" ): return lineno( self.loc, self.pstr ) @@ -262,22 +293,94 @@ class ParseException(ParseBaseException): """ Exception thrown when parse expressions don't match class; supported attributes by name are: - - lineno - returns the line number of the exception text - - col - returns the column number of the exception text - - line - returns the line containing the exception text - + - lineno - returns the line number of the exception text + - col - returns the column number of the exception text + - line - returns the line containing the exception text + Example:: + try: Word(nums).setName("integer").parseString("ABC") except ParseException as pe: print(pe) print("column: {}".format(pe.col)) - + prints:: + Expected integer (at char 0), (line:1, col:1) column: 1 + """ - pass + + @staticmethod + def explain(exc, depth=16): + """ + Method to take an exception and translate the Python internal traceback into a list + of the pyparsing expressions that caused the exception to be raised. + + Parameters: + + - exc - exception raised during parsing (need not be a ParseException, in support + of Python exceptions that might be raised in a parse action) + - depth (default=16) - number of levels back in the stack trace to list expression + and function names; if None, the full stack trace names will be listed; if 0, only + the failing input line, marker, and exception string will be shown + + Returns a multi-line string listing the ParserElements and/or function names in the + exception's stack trace. + + Note: the diagnostic output will include string representations of the expressions + that failed to parse. These representations will be more helpful if you use `setName` to + give identifiable names to your expressions. Otherwise they will use the default string + forms, which may be cryptic to read. + + explain() is only supported under Python 3. + """ + import inspect + + if depth is None: + depth = sys.getrecursionlimit() + ret = [] + if isinstance(exc, ParseBaseException): + ret.append(exc.line) + ret.append(' ' * (exc.col - 1) + '^') + ret.append("{0}: {1}".format(type(exc).__name__, exc)) + + if depth > 0: + callers = inspect.getinnerframes(exc.__traceback__, context=depth) + seen = set() + for i, ff in enumerate(callers[-depth:]): + frm = ff.frame + + f_self = frm.f_locals.get('self', None) + if isinstance(f_self, ParserElement): + if frm.f_code.co_name not in ('parseImpl', '_parseNoCache'): + continue + if f_self in seen: + continue + seen.add(f_self) + + self_type = type(f_self) + ret.append("{0}.{1} - {2}".format(self_type.__module__, + self_type.__name__, + f_self)) + elif f_self is not None: + self_type = type(f_self) + ret.append("{0}.{1}".format(self_type.__module__, + self_type.__name__)) + else: + code = frm.f_code + if code.co_name in ('wrapper', ''): + continue + + ret.append("{0}".format(code.co_name)) + + depth -= 1 + if not depth: + break + + return '\n'.join(ret) + class ParseFatalException(ParseBaseException): """user-throwable exception thrown when inconsistent parse content @@ -285,9 +388,11 @@ class ParseFatalException(ParseBaseException): pass class ParseSyntaxException(ParseFatalException): - """just like L{ParseFatalException}, but thrown internally when an - L{ErrorStop} ('-' operator) indicates that parsing is to stop - immediately because an unbacktrackable syntax error has been found""" + """just like :class:`ParseFatalException`, but thrown internally + when an :class:`ErrorStop` ('-' operator) indicates + that parsing is to stop immediately because an unbacktrackable + syntax error has been found. + """ pass #~ class ReparseException(ParseBaseException): @@ -304,7 +409,9 @@ class ParseSyntaxException(ParseFatalException): #~ self.reparseLoc = restartLoc class RecursiveGrammarException(Exception): - """exception thrown by L{ParserElement.validate} if the grammar could be improperly recursive""" + """exception thrown by :class:`ParserElement.validate` if the + grammar could be improperly recursive + """ def __init__( self, parseElementList ): self.parseElementTrace = parseElementList @@ -322,16 +429,18 @@ class _ParseResultsWithOffset(object): self.tup = (self.tup[0],i) class ParseResults(object): - """ - Structured parse results, to provide multiple means of access to the parsed data: - - as a list (C{len(results)}) - - by list index (C{results[0], results[1]}, etc.) - - by attribute (C{results.} - see L{ParserElement.setResultsName}) + """Structured parse results, to provide multiple means of access to + the parsed data: + + - as a list (``len(results)``) + - by list index (``results[0], results[1]``, etc.) + - by attribute (``results.`` - see :class:`ParserElement.setResultsName`) Example:: + integer = Word(nums) - date_str = (integer.setResultsName("year") + '/' - + integer.setResultsName("month") + '/' + date_str = (integer.setResultsName("year") + '/' + + integer.setResultsName("month") + '/' + integer.setResultsName("day")) # equivalent form: # date_str = integer("year") + '/' + integer("month") + '/' + integer("day") @@ -348,7 +457,9 @@ class ParseResults(object): test("'month' in result") test("'minutes' in result") test("result.dump()", str) + prints:: + list(result) -> ['1999', '/', '12', '/', '31'] result[0] -> '1999' result['month'] -> '12' @@ -398,7 +509,7 @@ class ParseResults(object): toklist = [ toklist ] if asList: if isinstance(toklist,ParseResults): - self[name] = _ParseResultsWithOffset(toklist.copy(),0) + self[name] = _ParseResultsWithOffset(ParseResults(toklist.__toklist), 0) else: self[name] = _ParseResultsWithOffset(ParseResults(toklist[0]),0) self[name].__name = name @@ -467,19 +578,19 @@ class ParseResults(object): def _itervalues( self ): return (self[k] for k in self._iterkeys()) - + def _iteritems( self ): return ((k, self[k]) for k in self._iterkeys()) if PY_3: - keys = _iterkeys - """Returns an iterator of all named result keys (Python 3.x only).""" + keys = _iterkeys + """Returns an iterator of all named result keys.""" values = _itervalues - """Returns an iterator of all named result values (Python 3.x only).""" + """Returns an iterator of all named result values.""" items = _iteritems - """Returns an iterator of all named result key-value tuples (Python 3.x only).""" + """Returns an iterator of all named result key-value tuples.""" else: iterkeys = _iterkeys @@ -498,7 +609,7 @@ class ParseResults(object): def values( self ): """Returns all named result values (as a list in Python 2.x, as an iterator in Python 3.x).""" return list(self.itervalues()) - + def items( self ): """Returns all named result key-values (as a list of tuples in Python 2.x, as an iterator in Python 3.x).""" return list(self.iteritems()) @@ -507,19 +618,20 @@ class ParseResults(object): """Since keys() returns an iterator, this method is helpful in bypassing code that looks for the existence of any defined results names.""" return bool(self.__tokdict) - + def pop( self, *args, **kwargs): """ - Removes and returns item at specified index (default=C{last}). - Supports both C{list} and C{dict} semantics for C{pop()}. If passed no - argument or an integer argument, it will use C{list} semantics - and pop tokens from the list of parsed tokens. If passed a - non-integer argument (most likely a string), it will use C{dict} - semantics and pop the corresponding value from any defined - results names. A second default return value argument is - supported, just as in C{dict.pop()}. + Removes and returns item at specified index (default= ``last``). + Supports both ``list`` and ``dict`` semantics for ``pop()``. If + passed no argument or an integer argument, it will use ``list`` + semantics and pop tokens from the list of parsed tokens. If passed + a non-integer argument (most likely a string), it will use ``dict`` + semantics and pop the corresponding value from any defined results + names. A second default return value argument is supported, just as in + ``dict.pop()``. Example:: + def remove_first(tokens): tokens.pop(0) print(OneOrMore(Word(nums)).parseString("0 123 321")) # -> ['0', '123', '321'] @@ -536,7 +648,9 @@ class ParseResults(object): return tokens patt.addParseAction(remove_LABEL) print(patt.parseString("AAB 123 321").dump()) + prints:: + ['AAB', '123', '321'] - LABEL: AAB @@ -549,8 +663,8 @@ class ParseResults(object): args = (args[0], v) else: raise TypeError("pop() got an unexpected keyword argument '%s'" % k) - if (isinstance(args[0], int) or - len(args) == 1 or + if (isinstance(args[0], int) or + len(args) == 1 or args[0] in self): index = args[0] ret = self[index] @@ -563,14 +677,15 @@ class ParseResults(object): def get(self, key, defaultValue=None): """ Returns named result matching the given key, or if there is no - such name, then returns the given C{defaultValue} or C{None} if no - C{defaultValue} is specified. + such name, then returns the given ``defaultValue`` or ``None`` if no + ``defaultValue`` is specified. + + Similar to ``dict.get()``. - Similar to C{dict.get()}. - Example:: + integer = Word(nums) - date_str = integer("year") + '/' + integer("month") + '/' + integer("day") + date_str = integer("year") + '/' + integer("month") + '/' + integer("day") result = date_str.parseString("1999/12/31") print(result.get("year")) # -> '1999' @@ -585,10 +700,11 @@ class ParseResults(object): def insert( self, index, insStr ): """ Inserts new element at location index in the list of parsed tokens. - - Similar to C{list.insert()}. + + Similar to ``list.insert()``. Example:: + print(OneOrMore(Word(nums)).parseString("0 123 321")) # -> ['0', '123', '321'] # use a parse action to insert the parse location in the front of the parsed results @@ -607,8 +723,9 @@ class ParseResults(object): Add single element to end of ParseResults list of elements. Example:: + print(OneOrMore(Word(nums)).parseString("0 123 321")) # -> ['0', '123', '321'] - + # use a parse action to compute the sum of the parsed integers, and add it to the end def append_sum(tokens): tokens.append(sum(map(int, tokens))) @@ -621,8 +738,9 @@ class ParseResults(object): Add sequence of elements to end of ParseResults list of elements. Example:: + patt = OneOrMore(Word(alphas)) - + # use a parse action to append the reverse of the matched strings, to make a palindrome def make_palindrome(tokens): tokens.extend(reversed([t[::-1] for t in tokens])) @@ -646,7 +764,7 @@ class ParseResults(object): return self[name] except KeyError: return "" - + if name in self.__tokdict: if name not in self.__accumNames: return self.__tokdict[name][-1][0] @@ -671,7 +789,7 @@ class ParseResults(object): self[k] = v if isinstance(v[0],ParseResults): v[0].__parent = wkref(self) - + self.__toklist += other.__toklist self.__accumNames.update( other.__accumNames ) return self @@ -683,7 +801,7 @@ class ParseResults(object): else: # this may raise a TypeError - so be it return other + self - + def __repr__( self ): return "(%s, %s)" % ( repr( self.__toklist ), repr( self.__tokdict ) ) @@ -706,11 +824,12 @@ class ParseResults(object): Returns the parse results as a nested list of matching tokens, all converted to strings. Example:: + patt = OneOrMore(Word(alphas)) result = patt.parseString("sldkj lsdkj sldkj") # even though the result prints in string-like form, it is actually a pyparsing ParseResults print(type(result), result) # -> ['sldkj', 'lsdkj', 'sldkj'] - + # Use asList() to create an actual list result_list = result.asList() print(type(result_list), result_list) # -> ['sldkj', 'lsdkj', 'sldkj'] @@ -722,12 +841,13 @@ class ParseResults(object): Returns the named parse results as a nested dictionary. Example:: + integer = Word(nums) date_str = integer("year") + '/' + integer("month") + '/' + integer("day") - + result = date_str.parseString('12/31/1999') print(type(result), repr(result)) # -> (['12', '/', '31', '/', '1999'], {'day': [('1999', 4)], 'year': [('12', 0)], 'month': [('31', 2)]}) - + result_dict = result.asDict() print(type(result_dict), repr(result_dict)) # -> {'day': '1999', 'year': '12', 'month': '31'} @@ -740,7 +860,7 @@ class ParseResults(object): item_fn = self.items else: item_fn = self.iteritems - + def toItem(obj): if isinstance(obj, ParseResults): if obj.haskeys(): @@ -749,15 +869,15 @@ class ParseResults(object): return [toItem(v) for v in obj] else: return obj - + return dict((k,toItem(v)) for k,v in item_fn()) def copy( self ): """ - Returns a new copy of a C{ParseResults} object. + Returns a new copy of a :class:`ParseResults` object. """ ret = ParseResults( self.__toklist ) - ret.__tokdict = self.__tokdict.copy() + ret.__tokdict = dict(self.__tokdict.items()) ret.__parent = self.__parent ret.__accumNames.update( self.__accumNames ) ret.__name = self.__name @@ -833,22 +953,25 @@ class ParseResults(object): def getName(self): r""" - Returns the results name for this token expression. Useful when several + Returns the results name for this token expression. Useful when several different expressions might match at a particular location. Example:: + integer = Word(nums) ssn_expr = Regex(r"\d\d\d-\d\d-\d\d\d\d") house_number_expr = Suppress('#') + Word(nums, alphanums) - user_data = (Group(house_number_expr)("house_number") + user_data = (Group(house_number_expr)("house_number") | Group(ssn_expr)("ssn") | Group(integer)("age")) user_info = OneOrMore(user_data) - + result = user_info.parseString("22 111-22-3333 #221B") for item in result: print(item.getName(), ':', item[0]) + prints:: + age : 22 ssn : 111-22-3333 house_number : 221B @@ -870,17 +993,20 @@ class ParseResults(object): def dump(self, indent='', depth=0, full=True): """ - Diagnostic method for listing out the contents of a C{ParseResults}. - Accepts an optional C{indent} argument so that this string can be embedded - in a nested display of other data. + Diagnostic method for listing out the contents of + a :class:`ParseResults`. Accepts an optional ``indent`` argument so + that this string can be embedded in a nested display of other data. Example:: + integer = Word(nums) date_str = integer("year") + '/' + integer("month") + '/' + integer("day") - + result = date_str.parseString('12/31/1999') print(result.dump()) + prints:: + ['12', '/', '31', '/', '1999'] - day: 1999 - month: 31 @@ -910,16 +1036,18 @@ class ParseResults(object): out.append("\n%s%s[%d]:\n%s%s%s" % (indent,(' '*(depth)),i,indent,(' '*(depth+1)),vv.dump(indent,depth+1) )) else: out.append("\n%s%s[%d]:\n%s%s%s" % (indent,(' '*(depth)),i,indent,(' '*(depth+1)),_ustr(vv))) - + return "".join(out) def pprint(self, *args, **kwargs): """ - Pretty-printer for parsed results as a list, using the C{pprint} module. - Accepts additional positional or keyword args as defined for the - C{pprint.pprint} method. (U{http://docs.python.org/3/library/pprint.html#pprint.pprint}) + Pretty-printer for parsed results as a list, using the + `pprint `_ module. + Accepts additional positional or keyword args as defined for + `pprint.pprint `_ . Example:: + ident = Word(alphas, alphanums) num = Word(nums) func = Forward() @@ -927,7 +1055,9 @@ class ParseResults(object): func <<= ident + Group(Optional(delimitedList(term))) result = func.parseString("fna a,b,(fnb c,d,200),100") result.pprint(width=40) + prints:: + ['fna', ['a', 'b', @@ -970,24 +1100,25 @@ def col (loc,strg): The first column is number 1. Note: the default parsing behavior is to expand tabs in the input string - before starting the parsing process. See L{I{ParserElement.parseString}} for more information - on parsing strings containing C{}s, and suggested methods to maintain a - consistent view of the parsed string, the parse location, and line and column - positions within the parsed string. + before starting the parsing process. See + :class:`ParserElement.parseString` for more + information on parsing strings containing ```` s, and suggested + methods to maintain a consistent view of the parsed string, the parse + location, and line and column positions within the parsed string. """ s = strg return 1 if 0} for more information - on parsing strings containing C{}s, and suggested methods to maintain a - consistent view of the parsed string, the parse location, and line and column - positions within the parsed string. - """ + Note - the default parsing behavior is to expand tabs in the input string + before starting the parsing process. See :class:`ParserElement.parseString` + for more information on parsing strings containing ```` s, and + suggested methods to maintain a consistent view of the parsed string, the + parse location, and line and column positions within the parsed string. + """ return strg.count("\n",0,loc) + 1 def line( loc, strg ): @@ -1041,7 +1172,7 @@ def _trim_arity(func, maxargs=2): return lambda s,l,t: func(t) limit = [0] foundArity = [False] - + # traceback return data structure changed in Py3.5 - normalize back to plain tuples if system_version[:2] >= (3,5): def extract_stack(limit=0): @@ -1056,12 +1187,12 @@ def _trim_arity(func, maxargs=2): else: extract_stack = traceback.extract_stack extract_tb = traceback.extract_tb - - # synthesize what would be returned by traceback.extract_stack at the call to + + # synthesize what would be returned by traceback.extract_stack at the call to # user's parse action 'func', so that we don't incur call penalty at parse time - + LINE_DIFF = 6 - # IF ANY CODE CHANGES, EVEN JUST COMMENTS OR BLANK LINES, BETWEEN THE NEXT LINE AND + # IF ANY CODE CHANGES, EVEN JUST COMMENTS OR BLANK LINES, BETWEEN THE NEXT LINE AND # THE CALL TO FUNC INSIDE WRAPPER, LINE_DIFF MUST BE MODIFIED!!!! this_line = extract_stack(limit=2)[-1] pa_call_line_synth = (this_line[0], this_line[1]+LINE_DIFF) @@ -1092,7 +1223,7 @@ def _trim_arity(func, maxargs=2): # copy func name to wrapper for sensible debug output func_name = "" try: - func_name = getattr(func, '__name__', + func_name = getattr(func, '__name__', getattr(func, '__class__').__name__) except Exception: func_name = str(func) @@ -1111,9 +1242,10 @@ class ParserElement(object): Overrides the default whitespace chars Example:: + # default whitespace chars are space, and newline OneOrMore(Word(alphas)).parseString("abc def\nghi jkl") # -> ['abc', 'def', 'ghi', 'jkl'] - + # change to just treat newline as significant ParserElement.setDefaultWhitespaceChars(" \t") OneOrMore(Word(alphas)).parseString("abc def\nghi jkl") # -> ['abc', 'def'] @@ -1124,18 +1256,19 @@ class ParserElement(object): def inlineLiteralsUsing(cls): """ Set class to be used for inclusion of string literals into a parser. - + Example:: + # default literal class used is Literal integer = Word(nums) - date_str = integer("year") + '/' + integer("month") + '/' + integer("day") + date_str = integer("year") + '/' + integer("month") + '/' + integer("day") date_str.parseString("1999/12/31") # -> ['1999', '/', '12', '/', '31'] # change to Suppress ParserElement.inlineLiteralsUsing(Suppress) - date_str = integer("year") + '/' + integer("month") + '/' + integer("day") + date_str = integer("year") + '/' + integer("month") + '/' + integer("day") date_str.parseString("1999/12/31") # -> ['1999', '12', '31'] """ @@ -1149,7 +1282,7 @@ class ParserElement(object): self.resultsName = None self.saveAsList = savelist self.skipWhitespace = True - self.whiteChars = ParserElement.DEFAULT_WHITE_CHARS + self.whiteChars = set(ParserElement.DEFAULT_WHITE_CHARS) self.copyDefaultWhiteChars = True self.mayReturnEmpty = False # used when checking for left-recursion self.keepTabs = False @@ -1166,18 +1299,24 @@ class ParserElement(object): def copy( self ): """ - Make a copy of this C{ParserElement}. Useful for defining different parse actions - for the same parsing pattern, using copies of the original parse element. - + Make a copy of this :class:`ParserElement`. Useful for defining + different parse actions for the same parsing pattern, using copies of + the original parse element. + Example:: + integer = Word(nums).setParseAction(lambda toks: int(toks[0])) integerK = integer.copy().addParseAction(lambda toks: toks[0]*1024) + Suppress("K") integerM = integer.copy().addParseAction(lambda toks: toks[0]*1024*1024) + Suppress("M") - + print(OneOrMore(integerK | integerM | integer).parseString("5K 100 640K 256M")) + prints:: + [5120, 100, 655360, 268435456] - Equivalent form of C{expr.copy()} is just C{expr()}:: + + Equivalent form of ``expr.copy()`` is just ``expr()``:: + integerM = integer().addParseAction(lambda toks: toks[0]*1024*1024) + Suppress("M") """ cpy = copy.copy( self ) @@ -1190,8 +1329,9 @@ class ParserElement(object): def setName( self, name ): """ Define name for this expression, makes debugging and exception messages clearer. - + Example:: + Word(nums).parseString("ABC") # -> Exception: Expected W:(0123...) (at char 0), (line:1, col:1) Word(nums).setName("integer").parseString("ABC") # -> Exception: Expected integer (at char 0), (line:1, col:1) """ @@ -1205,17 +1345,18 @@ class ParserElement(object): """ Define name for referencing matching tokens as a nested attribute of the returned parse results. - NOTE: this returns a *copy* of the original C{ParserElement} object; + NOTE: this returns a *copy* of the original :class:`ParserElement` object; this is so that the client can define a basic element, such as an integer, and reference it in multiple places with different names. You can also set results names using the abbreviated syntax, - C{expr("name")} in place of C{expr.setResultsName("name")} - - see L{I{__call__}<__call__>}. + ``expr("name")`` in place of ``expr.setResultsName("name")`` + - see :class:`__call__`. Example:: - date_str = (integer.setResultsName("year") + '/' - + integer.setResultsName("month") + '/' + + date_str = (integer.setResultsName("year") + '/' + + integer.setResultsName("month") + '/' + integer.setResultsName("day")) # equivalent form: @@ -1231,7 +1372,7 @@ class ParserElement(object): def setBreak(self,breakFlag = True): """Method to invoke the Python pdb debugger when this element is - about to be parsed. Set C{breakFlag} to True to enable, False to + about to be parsed. Set ``breakFlag`` to True to enable, False to disable. """ if breakFlag: @@ -1250,25 +1391,28 @@ class ParserElement(object): def setParseAction( self, *fns, **kwargs ): """ Define one or more actions to perform when successfully matching parse element definition. - Parse action fn is a callable method with 0-3 arguments, called as C{fn(s,loc,toks)}, - C{fn(loc,toks)}, C{fn(toks)}, or just C{fn()}, where: - - s = the original string being parsed (see note below) - - loc = the location of the matching substring - - toks = a list of the matched tokens, packaged as a C{L{ParseResults}} object + Parse action fn is a callable method with 0-3 arguments, called as ``fn(s,loc,toks)`` , + ``fn(loc,toks)`` , ``fn(toks)`` , or just ``fn()`` , where: + + - s = the original string being parsed (see note below) + - loc = the location of the matching substring + - toks = a list of the matched tokens, packaged as a :class:`ParseResults` object + If the functions in fns modify the tokens, they can return them as the return value from fn, and the modified list of tokens will replace the original. Otherwise, fn does not need to return any value. Optional keyword arguments: - - callDuringTry = (default=C{False}) indicate if parse action should be run during lookaheads and alternate testing + - callDuringTry = (default= ``False`` ) indicate if parse action should be run during lookaheads and alternate testing Note: the default parsing behavior is to expand tabs in the input string - before starting the parsing process. See L{I{parseString}} for more information - on parsing strings containing C{}s, and suggested methods to maintain a - consistent view of the parsed string, the parse location, and line and column - positions within the parsed string. - + before starting the parsing process. See :class:`parseString for more + information on parsing strings containing ```` s, and suggested + methods to maintain a consistent view of the parsed string, the parse + location, and line and column positions within the parsed string. + Example:: + integer = Word(nums) date_str = integer + '/' + integer + '/' + integer @@ -1287,24 +1431,25 @@ class ParserElement(object): def addParseAction( self, *fns, **kwargs ): """ - Add one or more parse actions to expression's list of parse actions. See L{I{setParseAction}}. - - See examples in L{I{copy}}. + Add one or more parse actions to expression's list of parse actions. See :class:`setParseAction`. + + See examples in :class:`copy`. """ self.parseAction += list(map(_trim_arity, list(fns))) self.callDuringTry = self.callDuringTry or kwargs.get("callDuringTry", False) return self def addCondition(self, *fns, **kwargs): - """Add a boolean predicate function to expression's list of parse actions. See - L{I{setParseAction}} for function call signatures. Unlike C{setParseAction}, - functions passed to C{addCondition} need to return boolean success/fail of the condition. + """Add a boolean predicate function to expression's list of parse actions. See + :class:`setParseAction` for function call signatures. Unlike ``setParseAction``, + functions passed to ``addCondition`` need to return boolean success/fail of the condition. Optional keyword arguments: - - message = define a custom message to be used in the raised exception - - fatal = if True, will raise ParseFatalException to stop parsing immediately; otherwise will raise ParseException - + - message = define a custom message to be used in the raised exception + - fatal = if True, will raise ParseFatalException to stop parsing immediately; otherwise will raise ParseException + Example:: + integer = Word(nums).setParseAction(lambda toks: int(toks[0])) year_int = integer.copy() year_int.addCondition(lambda toks: toks[0] >= 2000, message="Only support years 2000 and later") @@ -1315,8 +1460,9 @@ class ParserElement(object): msg = kwargs.get("message", "failed user-defined condition") exc_type = ParseFatalException if kwargs.get("fatal", False) else ParseException for fn in fns: + fn = _trim_arity(fn) def pa(s,l,t): - if not bool(_trim_arity(fn)(s,l,t)): + if not bool(fn(s,l,t)): raise exc_type(s,l,msg) self.parseAction.append(pa) self.callDuringTry = self.callDuringTry or kwargs.get("callDuringTry", False) @@ -1325,12 +1471,12 @@ class ParserElement(object): def setFailAction( self, fn ): """Define action to perform if parsing fails at this expression. Fail acton fn is a callable function that takes the arguments - C{fn(s,loc,expr,err)} where: - - s = string being parsed - - loc = location where expression match was attempted and failed - - expr = the parse expression that failed - - err = the exception thrown - The function returns no value. It may throw C{L{ParseFatalException}} + ``fn(s,loc,expr,err)`` where: + - s = string being parsed + - loc = location where expression match was attempted and failed + - expr = the parse expression that failed + - err = the exception thrown + The function returns no value. It may throw :class:`ParseFatalException` if it is desired to stop parsing immediately.""" self.failAction = fn return self @@ -1412,8 +1558,14 @@ class ParserElement(object): if debugging: try: for fn in self.parseAction: - tokens = fn( instring, tokensStart, retTokens ) - if tokens is not None: + try: + tokens = fn( instring, tokensStart, retTokens ) + except IndexError as parse_action_exc: + exc = ParseException("exception raised in parse action") + exc.__cause__ = parse_action_exc + raise exc + + if tokens is not None and tokens is not retTokens: retTokens = ParseResults( tokens, self.resultsName, asList=self.saveAsList and isinstance(tokens,(ParseResults,list)), @@ -1425,8 +1577,14 @@ class ParserElement(object): raise else: for fn in self.parseAction: - tokens = fn( instring, tokensStart, retTokens ) - if tokens is not None: + try: + tokens = fn( instring, tokensStart, retTokens ) + except IndexError as parse_action_exc: + exc = ParseException("exception raised in parse action") + exc.__cause__ = parse_action_exc + raise exc + + if tokens is not None and tokens is not retTokens: retTokens = ParseResults( tokens, self.resultsName, asList=self.saveAsList and isinstance(tokens,(ParseResults,list)), @@ -1443,7 +1601,7 @@ class ParserElement(object): return self._parse( instring, loc, doActions=False )[0] except ParseFatalException: raise ParseException( instring, loc, self.errmsg, self) - + def canParseNext(self, instring, loc): try: self.tryParse(instring, loc) @@ -1465,7 +1623,7 @@ class ParserElement(object): def clear(self): cache.clear() - + def cache_len(self): return len(cache) @@ -1577,23 +1735,23 @@ class ParserElement(object): often in many complex grammars) can immediately return a cached value, instead of re-executing parsing/validating code. Memoizing is done of both valid results and parsing exceptions. - + Parameters: - - cache_size_limit - (default=C{128}) - if an integer value is provided - will limit the size of the packrat cache; if None is passed, then - the cache size will be unbounded; if 0 is passed, the cache will - be effectively disabled. - + + - cache_size_limit - (default= ``128``) - if an integer value is provided + will limit the size of the packrat cache; if None is passed, then + the cache size will be unbounded; if 0 is passed, the cache will + be effectively disabled. + This speedup may break existing programs that use parse actions that have side-effects. For this reason, packrat parsing is disabled when you first import pyparsing. To activate the packrat feature, your - program must call the class method C{ParserElement.enablePackrat()}. If - your program uses C{psyco} to "compile as you go", you must call - C{enablePackrat} before calling C{psyco.full()}. If you do not do this, - Python will crash. For best results, call C{enablePackrat()} immediately - after importing pyparsing. - + program must call the class method :class:`ParserElement.enablePackrat`. + For best results, call ``enablePackrat()`` immediately after + importing pyparsing. + Example:: + import pyparsing pyparsing.ParserElement.enablePackrat() """ @@ -1612,23 +1770,25 @@ class ParserElement(object): expression has been built. If you want the grammar to require that the entire input string be - successfully parsed, then set C{parseAll} to True (equivalent to ending - the grammar with C{L{StringEnd()}}). + successfully parsed, then set ``parseAll`` to True (equivalent to ending + the grammar with ``StringEnd()``). - Note: C{parseString} implicitly calls C{expandtabs()} on the input string, + Note: ``parseString`` implicitly calls ``expandtabs()`` on the input string, in order to report proper column numbers in parse actions. If the input string contains tabs and - the grammar uses parse actions that use the C{loc} argument to index into the + the grammar uses parse actions that use the ``loc`` argument to index into the string being parsed, you can ensure you have a consistent view of the input string by: - - calling C{parseWithTabs} on your grammar before calling C{parseString} - (see L{I{parseWithTabs}}) - - define your parse action using the full C{(s,loc,toks)} signature, and - reference the input string using the parse action's C{s} argument - - explictly expand the tabs in your input string before calling - C{parseString} - + + - calling ``parseWithTabs`` on your grammar before calling ``parseString`` + (see :class:`parseWithTabs`) + - define your parse action using the full ``(s,loc,toks)`` signature, and + reference the input string using the parse action's ``s`` argument + - explictly expand the tabs in your input string before calling + ``parseString`` + Example:: + Word('a').parseString('aaaaabaaa') # -> ['aaaaa'] Word('a').parseString('aaaaabaaa', parseAll=True) # -> Exception: Expected end of text """ @@ -1659,22 +1819,23 @@ class ParserElement(object): """ Scan the input string for expression matches. Each match will return the matching tokens, start location, and end location. May be called with optional - C{maxMatches} argument, to clip scanning after 'n' matches are found. If - C{overlap} is specified, then overlapping matches will be reported. + ``maxMatches`` argument, to clip scanning after 'n' matches are found. If + ``overlap`` is specified, then overlapping matches will be reported. Note that the start and end locations are reported relative to the string - being parsed. See L{I{parseString}} for more information on parsing + being parsed. See :class:`parseString` for more information on parsing strings with embedded tabs. Example:: + source = "sldjf123lsdjjkf345sldkjf879lkjsfd987" print(source) for tokens,start,end in Word(alphas).scanString(source): print(' '*start + '^'*(end-start)) print(' '*start + tokens[0]) - + prints:: - + sldjf123lsdjjkf345sldkjf879lkjsfd987 ^^^^^ sldjf @@ -1728,19 +1889,22 @@ class ParserElement(object): def transformString( self, instring ): """ - Extension to C{L{scanString}}, to modify matching text with modified tokens that may - be returned from a parse action. To use C{transformString}, define a grammar and + Extension to :class:`scanString`, to modify matching text with modified tokens that may + be returned from a parse action. To use ``transformString``, define a grammar and attach a parse action to it that modifies the returned token list. - Invoking C{transformString()} on a target string will then scan for matches, + Invoking ``transformString()`` on a target string will then scan for matches, and replace the matched text patterns according to the logic in the parse - action. C{transformString()} returns the resulting transformed string. - + action. ``transformString()`` returns the resulting transformed string. + Example:: + wd = Word(alphas) wd.setParseAction(lambda toks: toks[0].title()) - + print(wd.transformString("now is the winter of our discontent made glorious summer by this sun of york.")) - Prints:: + + prints:: + Now Is The Winter Of Our Discontent Made Glorious Summer By This Sun Of York. """ out = [] @@ -1771,19 +1935,22 @@ class ParserElement(object): def searchString( self, instring, maxMatches=_MAX_INT ): """ - Another extension to C{L{scanString}}, simplifying the access to the tokens found + Another extension to :class:`scanString`, simplifying the access to the tokens found to match the given parse expression. May be called with optional - C{maxMatches} argument, to clip searching after 'n' matches are found. - + ``maxMatches`` argument, to clip searching after 'n' matches are found. + Example:: + # a capitalized word starts with an uppercase letter, followed by zero or more lowercase letters cap_word = Word(alphas.upper(), alphas.lower()) - + print(cap_word.searchString("More than Iron, more than Lead, more than Gold I need Electricity")) # the sum() builtin can be used to merge results into a single ParseResults object print(sum(cap_word.searchString("More than Iron, more than Lead, more than Gold I need Electricity"))) + prints:: + [['More'], ['Iron'], ['Lead'], ['Gold'], ['I'], ['Electricity']] ['More', 'Iron', 'Lead', 'Gold', 'I', 'Electricity'] """ @@ -1799,14 +1966,17 @@ class ParserElement(object): def split(self, instring, maxsplit=_MAX_INT, includeSeparators=False): """ Generator method to split a string using the given expression as a separator. - May be called with optional C{maxsplit} argument, to limit the number of splits; - and the optional C{includeSeparators} argument (default=C{False}), if the separating + May be called with optional ``maxsplit`` argument, to limit the number of splits; + and the optional ``includeSeparators`` argument (default= ``False``), if the separating matching text should be included in the split results. - - Example:: + + Example:: + punc = oneOf(list(".,;:/-!?")) print(list(punc.split("This, this?, this sentence, is badly punctuated!"))) + prints:: + ['This', ' this', '', ' this sentence', ' is badly punctuated', ''] """ splits = 0 @@ -1820,14 +1990,17 @@ class ParserElement(object): def __add__(self, other ): """ - Implementation of + operator - returns C{L{And}}. Adding strings to a ParserElement - converts them to L{Literal}s by default. - + Implementation of + operator - returns :class:`And`. Adding strings to a ParserElement + converts them to :class:`Literal`s by default. + Example:: + greet = Word(alphas) + "," + Word(alphas) + "!" hello = "Hello, World!" print (hello, "->", greet.parseString(hello)) - Prints:: + + prints:: + Hello, World! -> ['Hello', ',', 'World', '!'] """ if isinstance( other, basestring ): @@ -1840,7 +2013,7 @@ class ParserElement(object): def __radd__(self, other ): """ - Implementation of + operator when left operand is not a C{L{ParserElement}} + Implementation of + operator when left operand is not a :class:`ParserElement` """ if isinstance( other, basestring ): other = ParserElement._literalStringClass( other ) @@ -1852,7 +2025,7 @@ class ParserElement(object): def __sub__(self, other): """ - Implementation of - operator, returns C{L{And}} with error stop + Implementation of - operator, returns :class:`And` with error stop """ if isinstance( other, basestring ): other = ParserElement._literalStringClass( other ) @@ -1864,7 +2037,7 @@ class ParserElement(object): def __rsub__(self, other ): """ - Implementation of - operator when left operand is not a C{L{ParserElement}} + Implementation of - operator when left operand is not a :class:`ParserElement` """ if isinstance( other, basestring ): other = ParserElement._literalStringClass( other ) @@ -1876,23 +2049,23 @@ class ParserElement(object): def __mul__(self,other): """ - Implementation of * operator, allows use of C{expr * 3} in place of - C{expr + expr + expr}. Expressions may also me multiplied by a 2-integer - tuple, similar to C{{min,max}} multipliers in regular expressions. Tuples - may also include C{None} as in: - - C{expr*(n,None)} or C{expr*(n,)} is equivalent - to C{expr*n + L{ZeroOrMore}(expr)} - (read as "at least n instances of C{expr}") - - C{expr*(None,n)} is equivalent to C{expr*(0,n)} - (read as "0 to n instances of C{expr}") - - C{expr*(None,None)} is equivalent to C{L{ZeroOrMore}(expr)} - - C{expr*(1,None)} is equivalent to C{L{OneOrMore}(expr)} + Implementation of * operator, allows use of ``expr * 3`` in place of + ``expr + expr + expr``. Expressions may also me multiplied by a 2-integer + tuple, similar to ``{min,max}`` multipliers in regular expressions. Tuples + may also include ``None`` as in: + - ``expr*(n,None)`` or ``expr*(n,)`` is equivalent + to ``expr*n + ZeroOrMore(expr)`` + (read as "at least n instances of ``expr``") + - ``expr*(None,n)`` is equivalent to ``expr*(0,n)`` + (read as "0 to n instances of ``expr``") + - ``expr*(None,None)`` is equivalent to ``ZeroOrMore(expr)`` + - ``expr*(1,None)`` is equivalent to ``OneOrMore(expr)`` - Note that C{expr*(None,n)} does not raise an exception if + Note that ``expr*(None,n)`` does not raise an exception if more than n exprs exist in the input stream; that is, - C{expr*(None,n)} does not enforce a maximum number of expr + ``expr*(None,n)`` does not enforce a maximum number of expr occurrences. If this behavior is desired, then write - C{expr*(None,n) + ~expr} + ``expr*(None,n) + ~expr`` """ if isinstance(other,int): minElements, optElements = other,0 @@ -1947,7 +2120,7 @@ class ParserElement(object): def __or__(self, other ): """ - Implementation of | operator - returns C{L{MatchFirst}} + Implementation of | operator - returns :class:`MatchFirst` """ if isinstance( other, basestring ): other = ParserElement._literalStringClass( other ) @@ -1959,7 +2132,7 @@ class ParserElement(object): def __ror__(self, other ): """ - Implementation of | operator when left operand is not a C{L{ParserElement}} + Implementation of | operator when left operand is not a :class:`ParserElement` """ if isinstance( other, basestring ): other = ParserElement._literalStringClass( other ) @@ -1971,7 +2144,7 @@ class ParserElement(object): def __xor__(self, other ): """ - Implementation of ^ operator - returns C{L{Or}} + Implementation of ^ operator - returns :class:`Or` """ if isinstance( other, basestring ): other = ParserElement._literalStringClass( other ) @@ -1983,7 +2156,7 @@ class ParserElement(object): def __rxor__(self, other ): """ - Implementation of ^ operator when left operand is not a C{L{ParserElement}} + Implementation of ^ operator when left operand is not a :class:`ParserElement` """ if isinstance( other, basestring ): other = ParserElement._literalStringClass( other ) @@ -1995,7 +2168,7 @@ class ParserElement(object): def __and__(self, other ): """ - Implementation of & operator - returns C{L{Each}} + Implementation of & operator - returns :class:`Each` """ if isinstance( other, basestring ): other = ParserElement._literalStringClass( other ) @@ -2007,7 +2180,7 @@ class ParserElement(object): def __rand__(self, other ): """ - Implementation of & operator when left operand is not a C{L{ParserElement}} + Implementation of & operator when left operand is not a :class:`ParserElement` """ if isinstance( other, basestring ): other = ParserElement._literalStringClass( other ) @@ -2019,23 +2192,24 @@ class ParserElement(object): def __invert__( self ): """ - Implementation of ~ operator - returns C{L{NotAny}} + Implementation of ~ operator - returns :class:`NotAny` """ return NotAny( self ) def __call__(self, name=None): """ - Shortcut for C{L{setResultsName}}, with C{listAllMatches=False}. - - If C{name} is given with a trailing C{'*'} character, then C{listAllMatches} will be - passed as C{True}. - - If C{name} is omitted, same as calling C{L{copy}}. + Shortcut for :class:`setResultsName`, with ``listAllMatches=False``. + + If ``name`` is given with a trailing ``'*'`` character, then ``listAllMatches`` will be + passed as ``True``. + + If ``name` is omitted, same as calling :class:`copy`. Example:: + # these are equivalent userdata = Word(alphas).setResultsName("name") + Word(nums+"-").setResultsName("socsecno") - userdata = Word(alphas)("name") + Word(nums+"-")("socsecno") + userdata = Word(alphas)("name") + Word(nums+"-")("socsecno") """ if name is not None: return self.setResultsName(name) @@ -2044,7 +2218,7 @@ class ParserElement(object): def suppress( self ): """ - Suppresses the output of this C{ParserElement}; useful to keep punctuation from + Suppresses the output of this :class:`ParserElement`; useful to keep punctuation from cluttering up returned output. """ return Suppress( self ) @@ -2052,7 +2226,7 @@ class ParserElement(object): def leaveWhitespace( self ): """ Disables the skipping of whitespace before matching the characters in the - C{ParserElement}'s defined pattern. This is normally only used internally by + :class:`ParserElement`'s defined pattern. This is normally only used internally by the pyparsing module, but may be needed in some whitespace-sensitive grammars. """ self.skipWhitespace = False @@ -2069,9 +2243,9 @@ class ParserElement(object): def parseWithTabs( self ): """ - Overrides default behavior to expand C{}s to spaces before parsing the input string. - Must be called before C{parseString} when the input grammar contains elements that - match C{} characters. + Overrides default behavior to expand ````s to spaces before parsing the input string. + Must be called before ``parseString`` when the input grammar contains elements that + match ```` characters. """ self.keepTabs = True return self @@ -2081,11 +2255,12 @@ class ParserElement(object): Define expression to be ignored (e.g., comments) while doing pattern matching; may be called repeatedly, to define multiple comment or other ignorable patterns. - + Example:: + patt = OneOrMore(Word(alphas)) patt.parseString('ablaj /* comment */ lskjd') # -> ['ablaj'] - + patt.ignore(cStyleComment) patt.parseString('ablaj /* comment */ lskjd') # -> ['ablaj', 'lskjd'] """ @@ -2112,19 +2287,21 @@ class ParserElement(object): def setDebug( self, flag=True ): """ Enable display of debugging messages while doing pattern matching. - Set C{flag} to True to enable, False to disable. + Set ``flag`` to True to enable, False to disable. Example:: + wd = Word(alphas).setName("alphaword") integer = Word(nums).setName("numword") term = wd | integer - + # turn on debugging for wd wd.setDebug() OneOrMore(term).parseString("abc 123 xyz 890") - + prints:: + Match alphaword at loc 0(1,1) Matched alphaword -> ['abc'] Match alphaword at loc 3(1,4) @@ -2137,12 +2314,12 @@ class ParserElement(object): Exception raised:Expected alphaword (at char 15), (line:1, col:16) The output shown is that produced by the default debug actions - custom debug actions can be - specified using L{setDebugActions}. Prior to attempting - to match the C{wd} expression, the debugging message C{"Match at loc (,)"} - is shown. Then if the parse succeeds, a C{"Matched"} message is shown, or an C{"Exception raised"} - message is shown. Also note the use of L{setName} to assign a human-readable name to the expression, + specified using :class:`setDebugActions`. Prior to attempting + to match the ``wd`` expression, the debugging message ``"Match at loc (,)"`` + is shown. Then if the parse succeeds, a ``"Matched"`` message is shown, or an ``"Exception raised"`` + message is shown. Also note the use of :class:`setName` to assign a human-readable name to the expression, which makes debugging and exception messages easier to understand - for instance, the default - name created for the C{Word} expression without calling C{setName} is C{"W:(ABCD...)"}. + name created for the :class:`Word` expression without calling ``setName`` is ``"W:(ABCD...)"``. """ if flag: self.setDebugActions( _defaultStartDebugAction, _defaultSuccessDebugAction, _defaultExceptionDebugAction ) @@ -2212,14 +2389,15 @@ class ParserElement(object): def matches(self, testString, parseAll=True): """ - Method for quick testing of a parser against a test string. Good for simple + Method for quick testing of a parser against a test string. Good for simple inline microtests of sub expressions while building up larger parser. - + Parameters: - testString - to test against this expression for a match - - parseAll - (default=C{True}) - flag to pass to C{L{parseString}} when running tests - + - parseAll - (default= ``True``) - flag to pass to :class:`parseString` when running tests + Example:: + expr = Word(nums) assert expr.matches("100") """ @@ -2228,28 +2406,32 @@ class ParserElement(object): return True except ParseBaseException: return False - - def runTests(self, tests, parseAll=True, comment='#', fullDump=True, printResults=True, failureTests=False): + + def runTests(self, tests, parseAll=True, comment='#', + fullDump=True, printResults=True, failureTests=False, postParse=None): """ Execute the parse expression on a series of test strings, showing each test, the parsed results or where the parse failed. Quick and easy way to run a parse expression against a list of sample strings. - + Parameters: - tests - a list of separate test strings, or a multiline string of test strings - - parseAll - (default=C{True}) - flag to pass to C{L{parseString}} when running tests - - comment - (default=C{'#'}) - expression for indicating embedded comments in the test + - parseAll - (default= ``True``) - flag to pass to :class:`parseString` when running tests + - comment - (default= ``'#'``) - expression for indicating embedded comments in the test string; pass None to disable comment filtering - - fullDump - (default=C{True}) - dump results as list followed by results names in nested outline; + - fullDump - (default= ``True``) - dump results as list followed by results names in nested outline; if False, only dump nested list - - printResults - (default=C{True}) prints test output to stdout - - failureTests - (default=C{False}) indicates if these tests are expected to fail parsing + - printResults - (default= ``True``) prints test output to stdout + - failureTests - (default= ``False``) indicates if these tests are expected to fail parsing + - postParse - (default= ``None``) optional callback for successful parse results; called as + `fn(test_string, parse_results)` and returns a string to be added to the test output Returns: a (success, results) tuple, where success indicates that all tests succeeded - (or failed if C{failureTests} is True), and the results contain a list of lines of each + (or failed if ``failureTests`` is True), and the results contain a list of lines of each test's output - + Example:: + number_expr = pyparsing_common.number.copy() result = number_expr.runTests(''' @@ -2273,7 +2455,9 @@ class ParserElement(object): 3.14.159 ''', failureTests=True) print("Success" if result[0] else "Failed!") + prints:: + # unsigned integer 100 [100] @@ -2291,7 +2475,7 @@ class ParserElement(object): [1e-12] Success - + # stray character 100Z ^ @@ -2313,7 +2497,7 @@ class ParserElement(object): lines, create a test like this:: expr.runTest(r"this is a test\\n of strings that spans \\n 3 lines") - + (Note that this is a raw string literal, you must include the leading 'r'.) """ if isinstance(tests, basestring): @@ -2332,10 +2516,18 @@ class ParserElement(object): out = ['\n'.join(comments), t] comments = [] try: - t = t.replace(r'\n','\n') + # convert newline marks to actual newlines, and strip leading BOM if present + t = t.replace(r'\n','\n').lstrip('\ufeff') result = self.parseString(t, parseAll=parseAll) out.append(result.dump(full=fullDump)) success = success and not failureTests + if postParse is not None: + try: + pp_value = postParse(t, result) + if pp_value is not None: + out.append(str(pp_value)) + except Exception as e: + out.append("{0} failed: {1}: {2}".format(postParse.__name__, type(e).__name__, e)) except ParseBaseException as pe: fatal = "(FATAL)" if isinstance(pe, ParseFatalException) else "" if '\n' in t: @@ -2357,21 +2549,20 @@ class ParserElement(object): print('\n'.join(out)) allResults.append((t, result)) - + return success, allResults - + class Token(ParserElement): - """ - Abstract C{ParserElement} subclass, for defining atomic matching patterns. + """Abstract :class:`ParserElement` subclass, for defining atomic + matching patterns. """ def __init__( self ): super(Token,self).__init__( savelist=False ) class Empty(Token): - """ - An empty token, will always match. + """An empty token, will always match. """ def __init__( self ): super(Empty,self).__init__() @@ -2381,8 +2572,7 @@ class Empty(Token): class NoMatch(Token): - """ - A token that will never match. + """A token that will never match. """ def __init__( self ): super(NoMatch,self).__init__() @@ -2396,18 +2586,18 @@ class NoMatch(Token): class Literal(Token): - """ - Token to exactly match a specified string. - + """Token to exactly match a specified string. + Example:: + Literal('blah').parseString('blah') # -> ['blah'] Literal('blah').parseString('blahfooblah') # -> ['blah'] Literal('blah').parseString('bla') # -> Exception: Expected "blah" - - For case-insensitive matching, use L{CaselessLiteral}. - + + For case-insensitive matching, use :class:`CaselessLiteral`. + For keyword matching (force word break before and after the matched string), - use L{Keyword} or L{CaselessKeyword}. + use :class:`Keyword` or :class:`CaselessKeyword`. """ def __init__( self, matchString ): super(Literal,self).__init__() @@ -2437,21 +2627,29 @@ _L = Literal ParserElement._literalStringClass = Literal class Keyword(Token): - """ - Token to exactly match a specified string as a keyword, that is, it must be - immediately followed by a non-keyword character. Compare with C{L{Literal}}: - - C{Literal("if")} will match the leading C{'if'} in C{'ifAndOnlyIf'}. - - C{Keyword("if")} will not; it will only match the leading C{'if'} in C{'if x=1'}, or C{'if(y==2)'} - Accepts two optional constructor arguments in addition to the keyword string: - - C{identChars} is a string of characters that would be valid identifier characters, - defaulting to all alphanumerics + "_" and "$" - - C{caseless} allows case-insensitive matching, default is C{False}. - + """Token to exactly match a specified string as a keyword, that is, + it must be immediately followed by a non-keyword character. Compare + with :class:`Literal`: + + - ``Literal("if")`` will match the leading ``'if'`` in + ``'ifAndOnlyIf'``. + - ``Keyword("if")`` will not; it will only match the leading + ``'if'`` in ``'if x=1'``, or ``'if(y==2)'`` + + Accepts two optional constructor arguments in addition to the + keyword string: + + - ``identChars`` is a string of characters that would be valid + identifier characters, defaulting to all alphanumerics + "_" and + "$" + - ``caseless`` allows case-insensitive matching, default is ``False``. + Example:: + Keyword("start").parseString("start") # -> ['start'] Keyword("start").parseString("starting") # -> Exception - For case-insensitive matching, use L{CaselessKeyword}. + For case-insensitive matching, use :class:`CaselessKeyword`. """ DEFAULT_KEYWORD_CHARS = alphanums+"_$" @@ -2502,15 +2700,15 @@ class Keyword(Token): Keyword.DEFAULT_KEYWORD_CHARS = chars class CaselessLiteral(Literal): - """ - Token to match a specified string, ignoring case of letters. + """Token to match a specified string, ignoring case of letters. Note: the matched results will always be in the case of the given match string, NOT the case of the input text. Example:: + OneOrMore(CaselessLiteral("CMD")).parseString("cmd CMD Cmd10") # -> ['CMD', 'CMD', 'CMD'] - - (Contrast with example for L{CaselessKeyword}.) + + (Contrast with example for :class:`CaselessKeyword`.) """ def __init__( self, matchString ): super(CaselessLiteral,self).__init__( matchString.upper() ) @@ -2526,36 +2724,39 @@ class CaselessLiteral(Literal): class CaselessKeyword(Keyword): """ - Caseless version of L{Keyword}. + Caseless version of :class:`Keyword`. Example:: + OneOrMore(CaselessKeyword("CMD")).parseString("cmd CMD Cmd10") # -> ['CMD', 'CMD'] - - (Contrast with example for L{CaselessLiteral}.) + + (Contrast with example for :class:`CaselessLiteral`.) """ def __init__( self, matchString, identChars=None ): super(CaselessKeyword,self).__init__( matchString, identChars, caseless=True ) - def parseImpl( self, instring, loc, doActions=True ): - if ( (instring[ loc:loc+self.matchLen ].upper() == self.caselessmatch) and - (loc >= len(instring)-self.matchLen or instring[loc+self.matchLen].upper() not in self.identChars) ): - return loc+self.matchLen, self.match - raise ParseException(instring, loc, self.errmsg, self) - class CloseMatch(Token): - """ - A variation on L{Literal} which matches "close" matches, that is, - strings with at most 'n' mismatching characters. C{CloseMatch} takes parameters: - - C{match_string} - string to be matched - - C{maxMismatches} - (C{default=1}) maximum number of mismatches allowed to count as a match - - The results from a successful parse will contain the matched text from the input string and the following named results: - - C{mismatches} - a list of the positions within the match_string where mismatches were found - - C{original} - the original match_string used to compare against the input string - - If C{mismatches} is an empty list, then the match was an exact match. - + """A variation on :class:`Literal` which matches "close" matches, + that is, strings with at most 'n' mismatching characters. + :class:`CloseMatch` takes parameters: + + - ``match_string`` - string to be matched + - ``maxMismatches`` - (``default=1``) maximum number of + mismatches allowed to count as a match + + The results from a successful parse will contain the matched text + from the input string and the following named results: + + - ``mismatches`` - a list of the positions within the + match_string where mismatches were found + - ``original`` - the original match_string used to compare + against the input string + + If ``mismatches`` is an empty list, then the match was an exact + match. + Example:: + patt = CloseMatch("ATCATCGAATGGA") patt.parseString("ATCATCGAAXGGA") # -> (['ATCATCGAAXGGA'], {'mismatches': [[9]], 'original': ['ATCATCGAATGGA']}) patt.parseString("ATCAXCGAAXGGA") # -> Exception: Expected 'ATCATCGAATGGA' (with up to 1 mismatches) (at char 0), (line:1, col:1) @@ -2604,49 +2805,55 @@ class CloseMatch(Token): class Word(Token): - """ - Token for matching words composed of allowed character sets. - Defined with string containing all allowed initial characters, - an optional string containing allowed body characters (if omitted, + """Token for matching words composed of allowed character sets. + Defined with string containing all allowed initial characters, an + optional string containing allowed body characters (if omitted, defaults to the initial character set), and an optional minimum, - maximum, and/or exact length. The default value for C{min} is 1 (a - minimum value < 1 is not valid); the default values for C{max} and C{exact} - are 0, meaning no maximum or exact length restriction. An optional - C{excludeChars} parameter can list characters that might be found in - the input C{bodyChars} string; useful to define a word of all printables - except for one or two characters, for instance. - - L{srange} is useful for defining custom character set strings for defining - C{Word} expressions, using range notation from regular expression character sets. - - A common mistake is to use C{Word} to match a specific literal string, as in - C{Word("Address")}. Remember that C{Word} uses the string argument to define - I{sets} of matchable characters. This expression would match "Add", "AAA", - "dAred", or any other word made up of the characters 'A', 'd', 'r', 'e', and 's'. - To match an exact literal string, use L{Literal} or L{Keyword}. + maximum, and/or exact length. The default value for ``min`` is + 1 (a minimum value < 1 is not valid); the default values for + ``max`` and ``exact`` are 0, meaning no maximum or exact + length restriction. An optional ``excludeChars`` parameter can + list characters that might be found in the input ``bodyChars`` + string; useful to define a word of all printables except for one or + two characters, for instance. + + :class:`srange` is useful for defining custom character set strings + for defining ``Word`` expressions, using range notation from + regular expression character sets. + + A common mistake is to use :class:`Word` to match a specific literal + string, as in ``Word("Address")``. Remember that :class:`Word` + uses the string argument to define *sets* of matchable characters. + This expression would match "Add", "AAA", "dAred", or any other word + made up of the characters 'A', 'd', 'r', 'e', and 's'. To match an + exact literal string, use :class:`Literal` or :class:`Keyword`. pyparsing includes helper strings for building Words: - - L{alphas} - - L{nums} - - L{alphanums} - - L{hexnums} - - L{alphas8bit} (alphabetic characters in ASCII range 128-255 - accented, tilded, umlauted, etc.) - - L{punc8bit} (non-alphabetic characters in ASCII range 128-255 - currency, symbols, superscripts, diacriticals, etc.) - - L{printables} (any non-whitespace character) + + - :class:`alphas` + - :class:`nums` + - :class:`alphanums` + - :class:`hexnums` + - :class:`alphas8bit` (alphabetic characters in ASCII range 128-255 + - accented, tilded, umlauted, etc.) + - :class:`punc8bit` (non-alphabetic characters in ASCII range + 128-255 - currency, symbols, superscripts, diacriticals, etc.) + - :class:`printables` (any non-whitespace character) Example:: + # a word composed of digits integer = Word(nums) # equivalent to Word("0123456789") or Word(srange("0-9")) - + # a word with a leading capital, and zero or more lowercase capital_word = Word(alphas.upper(), alphas.lower()) # hostnames are alphanumeric, with leading alpha, and '-' hostname = Word(alphas, alphanums+'-') - + # roman numeral (not a strict parser, accepts invalid mix of characters) roman = Word("IVXLCDM") - + # any string of non-whitespace characters, except for ',' csv_value = Word(printables, excludeChars=",") """ @@ -2762,22 +2969,38 @@ class Word(Token): return self.strRepr +class Char(Word): + """A short-cut class for defining ``Word(characters, exact=1)``, + when defining a match of any single character in a string of + characters. + """ + def __init__(self, charset): + super(Char, self).__init__(charset, exact=1) + self.reString = "[%s]" % _escapeRegexRangeChars(self.initCharsOrig) + self.re = re.compile( self.reString ) + + class Regex(Token): - r""" - Token for matching strings that match a given regular expression. - Defined with string specifying the regular expression in a form recognized by the inbuilt Python re module. - If the given regex contains named groups (defined using C{(?P...)}), these will be preserved as - named parse results. + r"""Token for matching strings that match a given regular + expression. Defined with string specifying the regular expression in + a form recognized by the stdlib Python `re module `_. + If the given regex contains named groups (defined using ``(?P...)``), + these will be preserved as named parse results. Example:: + realnum = Regex(r"[+-]?\d+\.\d*") date = Regex(r'(?P\d{4})-(?P\d\d?)-(?P\d\d?)') - # ref: http://stackoverflow.com/questions/267399/how-do-you-match-only-valid-roman-numerals-with-a-regular-expression - roman = Regex(r"M{0,4}(CM|CD|D?C{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3})") + # ref: https://stackoverflow.com/questions/267399/how-do-you-match-only-valid-roman-numerals-with-a-regular-expression + roman = Regex(r"M{0,4}(CM|CD|D?{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3})") """ compiledREtype = type(re.compile("[A-Z]")) def __init__( self, pattern, flags=0, asGroupList=False, asMatch=False): - """The parameters C{pattern} and C{flags} are passed to the C{re.compile()} function as-is. See the Python C{re} module for an explanation of the acceptable patterns and flags.""" + """The parameters ``pattern`` and ``flags`` are passed + to the ``re.compile()`` function as-is. See the Python + `re module `_ module for an + explanation of the acceptable patterns and flags. + """ super(Regex,self).__init__() if isinstance(pattern, basestring): @@ -2801,7 +3024,7 @@ class Regex(Token): self.pattern = \ self.reString = str(pattern) self.flags = flags - + else: raise ValueError("Regex may only be constructed with a string or a compiled RE object") @@ -2818,16 +3041,16 @@ class Regex(Token): raise ParseException(instring, loc, self.errmsg, self) loc = result.end() - d = result.groupdict() if self.asMatch: ret = result elif self.asGroupList: ret = result.groups() else: ret = ParseResults(result.group()) + d = result.groupdict() if d: - for k in d: - ret[k] = d[k] + for k, v in d.items(): + ret[k] = v return loc,ret def __str__( self ): @@ -2844,17 +3067,23 @@ class Regex(Token): def sub(self, repl): """ Return Regex with an attached parse action to transform the parsed - result as if called using C{re.sub(expr, repl, string)}. + result as if called using `re.sub(expr, repl, string) `_. + + Example:: + + make_html = Regex(r"(\w+):(.*?):").sub(r"<\1>\2") + print(make_html.transformString("h1:main title:")) + # prints "

main title

" """ if self.asGroupList: - warnings.warn("cannot use sub() with Regex(asGroupList=True)", + warnings.warn("cannot use sub() with Regex(asGroupList=True)", SyntaxWarning, stacklevel=2) raise SyntaxError() if self.asMatch and callable(repl): - warnings.warn("cannot use sub() with a callable with Regex(asMatch=True)", + warnings.warn("cannot use sub() with a callable with Regex(asMatch=True)", SyntaxWarning, stacklevel=2) - raise SyntaxError() + raise SyntaxError() if self.asMatch: def pa(tokens): @@ -2867,24 +3096,38 @@ class Regex(Token): class QuotedString(Token): r""" Token for matching strings that are delimited by quoting characters. - + Defined with the following parameters: - - quoteChar - string of one or more characters defining the quote delimiting string - - escChar - character to escape quotes, typically backslash (default=C{None}) - - escQuote - special quote sequence to escape an embedded quote string (such as SQL's "" to escape an embedded ") (default=C{None}) - - multiline - boolean indicating whether quotes can span multiple lines (default=C{False}) - - unquoteResults - boolean indicating whether the matched text should be unquoted (default=C{True}) - - endQuoteChar - string of one or more characters defining the end of the quote delimited string (default=C{None} => same as quoteChar) - - convertWhitespaceEscapes - convert escaped whitespace (C{'\t'}, C{'\n'}, etc.) to actual whitespace (default=C{True}) + + - quoteChar - string of one or more characters defining the + quote delimiting string + - escChar - character to escape quotes, typically backslash + (default= ``None`` ) + - escQuote - special quote sequence to escape an embedded quote + string (such as SQL's ``""`` to escape an embedded ``"``) + (default= ``None`` ) + - multiline - boolean indicating whether quotes can span + multiple lines (default= ``False`` ) + - unquoteResults - boolean indicating whether the matched text + should be unquoted (default= ``True`` ) + - endQuoteChar - string of one or more characters defining the + end of the quote delimited string (default= ``None`` => same as + quoteChar) + - convertWhitespaceEscapes - convert escaped whitespace + (``'\t'``, ``'\n'``, etc.) to actual whitespace + (default= ``True`` ) Example:: + qs = QuotedString('"') print(qs.searchString('lsjdf "This is the quote" sldjf')) complex_qs = QuotedString('{{', endQuoteChar='}}') print(complex_qs.searchString('lsjdf {{This is the "quote"}} sldjf')) sql_qs = QuotedString('"', escQuote='""') print(sql_qs.searchString('lsjdf "This is the quote with ""embedded"" quotes" sldjf')) + prints:: + [['This is the quote']] [['This is the "quote"']] [['This is the quote with "embedded" quotes']] @@ -3002,19 +3245,23 @@ class QuotedString(Token): class CharsNotIn(Token): - """ - Token for matching words composed of characters I{not} in a given set (will - include whitespace in matched characters if not listed in the provided exclusion set - see example). - Defined with string containing all disallowed characters, and an optional - minimum, maximum, and/or exact length. The default value for C{min} is 1 (a - minimum value < 1 is not valid); the default values for C{max} and C{exact} - are 0, meaning no maximum or exact length restriction. + """Token for matching words composed of characters *not* in a given + set (will include whitespace in matched characters if not listed in + the provided exclusion set - see example). Defined with string + containing all disallowed characters, and an optional minimum, + maximum, and/or exact length. The default value for ``min`` is + 1 (a minimum value < 1 is not valid); the default values for + ``max`` and ``exact`` are 0, meaning no maximum or exact + length restriction. Example:: + # define a comma-separated-value as anything that is not a ',' csv_value = CharsNotIn(',') print(delimitedList(csv_value).parseString("dkls,lsdkjf,s12 34,@!#,213")) + prints:: + ['dkls', 'lsdkjf', 's12 34', '@!#', '213'] """ def __init__( self, notChars, min=1, max=0, exact=0 ): @@ -3023,7 +3270,9 @@ class CharsNotIn(Token): self.notChars = notChars if min < 1: - raise ValueError("cannot specify a minimum length < 1; use Optional(CharsNotIn()) if zero-length char group is permitted") + raise ValueError( + "cannot specify a minimum length < 1; use " + + "Optional(CharsNotIn()) if zero-length char group is permitted") self.minLen = min @@ -3073,19 +3322,38 @@ class CharsNotIn(Token): return self.strRepr class White(Token): - """ - Special matching class for matching whitespace. Normally, whitespace is ignored - by pyparsing grammars. This class is included when some whitespace structures - are significant. Define with a string containing the whitespace characters to be - matched; default is C{" \\t\\r\\n"}. Also takes optional C{min}, C{max}, and C{exact} arguments, - as defined for the C{L{Word}} class. + """Special matching class for matching whitespace. Normally, + whitespace is ignored by pyparsing grammars. This class is included + when some whitespace structures are significant. Define with + a string containing the whitespace characters to be matched; default + is ``" \\t\\r\\n"``. Also takes optional ``min``, + ``max``, and ``exact`` arguments, as defined for the + :class:`Word` class. """ whiteStrs = { - " " : "", - "\t": "", - "\n": "", - "\r": "", - "\f": "", + ' ' : '', + '\t': '', + '\n': '', + '\r': '', + '\f': '', + 'u\00A0': '', + 'u\1680': '', + 'u\180E': '', + 'u\2000': '', + 'u\2001': '', + 'u\2002': '', + 'u\2003': '', + 'u\2004': '', + 'u\2005': '', + 'u\2006': '', + 'u\2007': '', + 'u\2008': '', + 'u\2009': '', + 'u\200A': '', + 'u\200B': '', + 'u\202F': '', + 'u\205F': '', + 'u\3000': '', } def __init__(self, ws=" \t\r\n", min=1, max=0, exact=0): super(White,self).__init__() @@ -3131,8 +3399,8 @@ class _PositionToken(Token): self.mayIndexError = False class GoToColumn(_PositionToken): - """ - Token to advance to a specific column of input text; useful for tabular report scraping. + """Token to advance to a specific column of input text; useful for + tabular report scraping. """ def __init__( self, colno ): super(GoToColumn,self).__init__() @@ -3157,11 +3425,11 @@ class GoToColumn(_PositionToken): class LineStart(_PositionToken): - """ - Matches if current position is at the beginning of a line within the parse string - + """Matches if current position is at the beginning of a line within + the parse string + Example:: - + test = '''\ AAA this line AAA and this line @@ -3171,10 +3439,11 @@ class LineStart(_PositionToken): for t in (LineStart() + 'AAA' + restOfLine).searchString(test): print(t) - - Prints:: + + prints:: + ['AAA', ' this line'] - ['AAA', ' and this line'] + ['AAA', ' and this line'] """ def __init__( self ): @@ -3187,8 +3456,8 @@ class LineStart(_PositionToken): raise ParseException(instring, loc, self.errmsg, self) class LineEnd(_PositionToken): - """ - Matches if current position is at the end of a line within the parse string + """Matches if current position is at the end of a line within the + parse string """ def __init__( self ): super(LineEnd,self).__init__() @@ -3207,8 +3476,8 @@ class LineEnd(_PositionToken): raise ParseException(instring, loc, self.errmsg, self) class StringStart(_PositionToken): - """ - Matches if current position is at the beginning of the parse string + """Matches if current position is at the beginning of the parse + string """ def __init__( self ): super(StringStart,self).__init__() @@ -3222,8 +3491,7 @@ class StringStart(_PositionToken): return loc, [] class StringEnd(_PositionToken): - """ - Matches if current position is at the end of the parse string + """Matches if current position is at the end of the parse string """ def __init__( self ): super(StringEnd,self).__init__() @@ -3240,12 +3508,13 @@ class StringEnd(_PositionToken): raise ParseException(instring, loc, self.errmsg, self) class WordStart(_PositionToken): - """ - Matches if the current position is at the beginning of a Word, and - is not preceded by any character in a given set of C{wordChars} - (default=C{printables}). To emulate the C{\b} behavior of regular expressions, - use C{WordStart(alphanums)}. C{WordStart} will also match at the beginning of - the string being parsed, or at the beginning of a line. + """Matches if the current position is at the beginning of a Word, + and is not preceded by any character in a given set of + ``wordChars`` (default= ``printables``). To emulate the + ``\b`` behavior of regular expressions, use + ``WordStart(alphanums)``. ``WordStart`` will also match at + the beginning of the string being parsed, or at the beginning of + a line. """ def __init__(self, wordChars = printables): super(WordStart,self).__init__() @@ -3260,12 +3529,12 @@ class WordStart(_PositionToken): return loc, [] class WordEnd(_PositionToken): - """ - Matches if the current position is at the end of a Word, and - is not followed by any character in a given set of C{wordChars} - (default=C{printables}). To emulate the C{\b} behavior of regular expressions, - use C{WordEnd(alphanums)}. C{WordEnd} will also match at the end of - the string being parsed, or at the end of a line. + """Matches if the current position is at the end of a Word, and is + not followed by any character in a given set of ``wordChars`` + (default= ``printables``). To emulate the ``\b`` behavior of + regular expressions, use ``WordEnd(alphanums)``. ``WordEnd`` + will also match at the end of the string being parsed, or at the end + of a line. """ def __init__(self, wordChars = printables): super(WordEnd,self).__init__() @@ -3283,8 +3552,8 @@ class WordEnd(_PositionToken): class ParseExpression(ParserElement): - """ - Abstract subclass of ParserElement, for combining and post-processing parsed tokens. + """Abstract subclass of ParserElement, for combining and + post-processing parsed tokens. """ def __init__( self, exprs, savelist = False ): super(ParseExpression,self).__init__(savelist) @@ -3315,7 +3584,7 @@ class ParseExpression(ParserElement): return self def leaveWhitespace( self ): - """Extends C{leaveWhitespace} defined in base class, and also invokes C{leaveWhitespace} on + """Extends ``leaveWhitespace`` defined in base class, and also invokes ``leaveWhitespace`` on all contained expressions.""" self.skipWhitespace = False self.exprs = [ e.copy() for e in self.exprs ] @@ -3376,7 +3645,7 @@ class ParseExpression(ParserElement): self.mayIndexError |= other.mayIndexError self.errmsg = "Expected " + _ustr(self) - + return self def setResultsName( self, name, listAllMatches=False ): @@ -3388,7 +3657,7 @@ class ParseExpression(ParserElement): for e in self.exprs: e.validate(tmp) self.checkRecursion( [] ) - + def copy(self): ret = super(ParseExpression,self).copy() ret.exprs = [e.copy() for e in self.exprs] @@ -3396,12 +3665,14 @@ class ParseExpression(ParserElement): class And(ParseExpression): """ - Requires all given C{ParseExpression}s to be found in the given order. + Requires all given :class:`ParseExpression` s to be found in the given order. Expressions may be separated by whitespace. - May be constructed using the C{'+'} operator. - May also be constructed using the C{'-'} operator, which will suppress backtracking. + May be constructed using the ``'+'`` operator. + May also be constructed using the ``'-'`` operator, which will + suppress backtracking. Example:: + integer = Word(nums) name_expr = OneOrMore(Word(alphas)) @@ -3423,6 +3694,11 @@ class And(ParseExpression): self.skipWhitespace = self.exprs[0].skipWhitespace self.callPreparse = True + def streamline(self): + super(And, self).streamline() + self.mayReturnEmpty = all(e.mayReturnEmpty for e in self.exprs) + return self + def parseImpl( self, instring, loc, doActions=True ): # pass False as last arg to _parse for first element, since we already # pre-parsed the string as part of our And pre-parsing @@ -3471,17 +3747,20 @@ class And(ParseExpression): class Or(ParseExpression): - """ - Requires that at least one C{ParseExpression} is found. - If two expressions match, the expression that matches the longest string will be used. - May be constructed using the C{'^'} operator. + """Requires that at least one :class:`ParseExpression` is found. If + two expressions match, the expression that matches the longest + string will be used. May be constructed using the ``'^'`` + operator. Example:: + # construct Or using '^' operator - + number = Word(nums) ^ Combine(Word(nums) + '.' + Word(nums)) print(number.searchString("123 3.1416 789")) + prints:: + [['123'], ['3.1416'], ['789']] """ def __init__( self, exprs, savelist = False ): @@ -3491,6 +3770,11 @@ class Or(ParseExpression): else: self.mayReturnEmpty = True + def streamline(self): + super(Or, self).streamline() + self.saveAsList = any(e.saveAsList for e in self.exprs) + return self + def parseImpl( self, instring, loc, doActions=True ): maxExcLoc = -1 maxException = None @@ -3550,14 +3834,14 @@ class Or(ParseExpression): class MatchFirst(ParseExpression): - """ - Requires that at least one C{ParseExpression} is found. - If two expressions match, the first one listed is the one that will match. - May be constructed using the C{'|'} operator. + """Requires that at least one :class:`ParseExpression` is found. If + two expressions match, the first one listed is the one that will + match. May be constructed using the ``'|'`` operator. Example:: + # construct MatchFirst using '|' operator - + # watch the order of expressions to match number = Word(nums) | Combine(Word(nums) + '.' + Word(nums)) print(number.searchString("123 3.1416 789")) # Fail! -> [['123'], ['3'], ['1416'], ['789']] @@ -3570,9 +3854,15 @@ class MatchFirst(ParseExpression): super(MatchFirst,self).__init__(exprs, savelist) if self.exprs: self.mayReturnEmpty = any(e.mayReturnEmpty for e in self.exprs) + # self.saveAsList = any(e.saveAsList for e in self.exprs) else: self.mayReturnEmpty = True + def streamline(self): + super(MatchFirst, self).streamline() + self.saveAsList = any(e.saveAsList for e in self.exprs) + return self + def parseImpl( self, instring, loc, doActions=True ): maxExcLoc = -1 maxException = None @@ -3618,12 +3908,13 @@ class MatchFirst(ParseExpression): class Each(ParseExpression): - """ - Requires all given C{ParseExpression}s to be found, but in any order. - Expressions may be separated by whitespace. - May be constructed using the C{'&'} operator. + """Requires all given :class:`ParseExpression` s to be found, but in + any order. Expressions may be separated by whitespace. + + May be constructed using the ``'&'`` operator. Example:: + color = oneOf("RED ORANGE YELLOW GREEN BLUE PURPLE BLACK WHITE BROWN") shape_type = oneOf("SQUARE CIRCLE TRIANGLE STAR HEXAGON OCTAGON") integer = Word(nums) @@ -3632,7 +3923,7 @@ class Each(ParseExpression): color_attr = "color:" + color("color") size_attr = "size:" + integer("size") - # use Each (using operator '&') to accept attributes in any order + # use Each (using operator '&') to accept attributes in any order # (shape and posn are required, color and size are optional) shape_spec = shape_attr & posn_attr & Optional(color_attr) & Optional(size_attr) @@ -3642,7 +3933,9 @@ class Each(ParseExpression): color:GREEN size:20 shape:TRIANGLE posn:20,40 ''' ) + prints:: + shape: SQUARE color: BLACK posn: 100, 120 ['shape:', 'SQUARE', 'color:', 'BLACK', 'posn:', ['100', ',', '120']] - color: BLACK @@ -3676,6 +3969,12 @@ class Each(ParseExpression): self.mayReturnEmpty = all(e.mayReturnEmpty for e in self.exprs) self.skipWhitespace = True self.initExprGroups = True + self.saveAsList = True + + def streamline(self): + super(Each, self).streamline() + self.mayReturnEmpty = all(e.mayReturnEmpty for e in self.exprs) + return self def parseImpl( self, instring, loc, doActions=True ): if self.initExprGroups: @@ -3742,8 +4041,8 @@ class Each(ParseExpression): class ParseElementEnhance(ParserElement): - """ - Abstract subclass of C{ParserElement}, for combining and post-processing parsed tokens. + """Abstract subclass of :class:`ParserElement`, for combining and + post-processing parsed tokens. """ def __init__( self, expr, savelist=False ): super(ParseElementEnhance,self).__init__(savelist) @@ -3819,20 +4118,25 @@ class ParseElementEnhance(ParserElement): class FollowedBy(ParseElementEnhance): - """ - Lookahead matching of the given parse expression. C{FollowedBy} - does I{not} advance the parsing position within the input string, it only - verifies that the specified parse expression matches at the current - position. C{FollowedBy} always returns a null token list. + """Lookahead matching of the given parse expression. + ``FollowedBy`` does *not* advance the parsing position within + the input string, it only verifies that the specified parse + expression matches at the current position. ``FollowedBy`` + always returns a null token list. If any results names are defined + in the lookahead expression, those *will* be returned for access by + name. Example:: + # use FollowedBy to match a label only if it is followed by a ':' data_word = Word(alphas) label = data_word + FollowedBy(':') attr_expr = Group(label + Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join)) - + OneOrMore(attr_expr).parseString("shape: SQUARE color: BLACK posn: upper left").pprint() + prints:: + [['shape', 'SQUARE'], ['color', 'BLACK'], ['posn', 'upper left']] """ def __init__( self, expr ): @@ -3840,20 +4144,108 @@ class FollowedBy(ParseElementEnhance): self.mayReturnEmpty = True def parseImpl( self, instring, loc, doActions=True ): - self.expr.tryParse( instring, loc ) - return loc, [] + _, ret = self.expr._parse(instring, loc, doActions=doActions) + del ret[:] + return loc, ret + + +class PrecededBy(ParseElementEnhance): + """Lookbehind matching of the given parse expression. + ``PrecededBy`` does not advance the parsing position within the + input string, it only verifies that the specified parse expression + matches prior to the current position. ``PrecededBy`` always + returns a null token list, but if a results name is defined on the + given expression, it is returned. + + Parameters: + + - expr - expression that must match prior to the current parse + location + - retreat - (default= ``None``) - (int) maximum number of characters + to lookbehind prior to the current parse location + + If the lookbehind expression is a string, Literal, Keyword, or + a Word or CharsNotIn with a specified exact or maximum length, then + the retreat parameter is not required. Otherwise, retreat must be + specified to give a maximum number of characters to look back from + the current parse position for a lookbehind match. + + Example:: + + # VB-style variable names with type prefixes + int_var = PrecededBy("#") + pyparsing_common.identifier + str_var = PrecededBy("$") + pyparsing_common.identifier + + """ + def __init__(self, expr, retreat=None): + super(PrecededBy, self).__init__(expr) + self.expr = self.expr().leaveWhitespace() + self.mayReturnEmpty = True + self.mayIndexError = False + self.exact = False + if isinstance(expr, str): + retreat = len(expr) + self.exact = True + elif isinstance(expr, (Literal, Keyword)): + retreat = expr.matchLen + self.exact = True + elif isinstance(expr, (Word, CharsNotIn)) and expr.maxLen != _MAX_INT: + retreat = expr.maxLen + self.exact = True + elif isinstance(expr, _PositionToken): + retreat = 0 + self.exact = True + self.retreat = retreat + self.errmsg = "not preceded by " + str(expr) + self.skipWhitespace = False + + def parseImpl(self, instring, loc=0, doActions=True): + if self.exact: + if loc < self.retreat: + raise ParseException(instring, loc, self.errmsg) + start = loc - self.retreat + _, ret = self.expr._parse(instring, start) + else: + # retreat specified a maximum lookbehind window, iterate + test_expr = self.expr + StringEnd() + instring_slice = instring[:loc] + last_expr = ParseException(instring, loc, self.errmsg) + for offset in range(1, min(loc, self.retreat+1)): + try: + _, ret = test_expr._parse(instring_slice, loc-offset) + except ParseBaseException as pbe: + last_expr = pbe + else: + break + else: + raise last_expr + # return empty list of tokens, but preserve any defined results names + del ret[:] + return loc, ret class NotAny(ParseElementEnhance): - """ - Lookahead to disallow matching with the given parse expression. C{NotAny} - does I{not} advance the parsing position within the input string, it only - verifies that the specified parse expression does I{not} match at the current - position. Also, C{NotAny} does I{not} skip over leading whitespace. C{NotAny} - always returns a null token list. May be constructed using the '~' operator. + """Lookahead to disallow matching with the given parse expression. + ``NotAny`` does *not* advance the parsing position within the + input string, it only verifies that the specified parse expression + does *not* match at the current position. Also, ``NotAny`` does + *not* skip over leading whitespace. ``NotAny`` always returns + a null token list. May be constructed using the '~' operator. Example:: - + + AND, OR, NOT = map(CaselessKeyword, "AND OR NOT".split()) + + # take care not to mistake keywords for identifiers + ident = ~(AND | OR | NOT) + Word(alphas) + boolean_term = Optional(NOT) + ident + + # very crude boolean expression - to support parenthesis groups and + # operation hierarchy, use infixNotation + boolean_expr = boolean_term + ZeroOrMore((AND | OR) + boolean_term) + + # integers that are followed by "." are actually floats + integer = Word(nums) + ~Char(".") """ def __init__( self, expr ): super(NotAny,self).__init__(expr) @@ -3891,7 +4283,7 @@ class _MultipleMatch(ParseElementEnhance): check_ender = self.not_ender is not None if check_ender: try_not_ender = self.not_ender.tryParse - + # must be at least one (but first see if we are the stopOn sentinel; # if so, fail) if check_ender: @@ -3913,18 +4305,18 @@ class _MultipleMatch(ParseElementEnhance): pass return loc, tokens - + class OneOrMore(_MultipleMatch): - """ - Repetition of one or more of the given expression. - + """Repetition of one or more of the given expression. + Parameters: - expr - expression that must match one or more times - - stopOn - (default=C{None}) - expression for a terminating sentinel - (only required if the sentinel would ordinarily match the repetition - expression) + - stopOn - (default= ``None``) - expression for a terminating sentinel + (only required if the sentinel would ordinarily match the repetition + expression) Example:: + data_word = Word(alphas) label = data_word + FollowedBy(':') attr_expr = Group(label + Suppress(':') + OneOrMore(data_word).setParseAction(' '.join)) @@ -3935,7 +4327,7 @@ class OneOrMore(_MultipleMatch): # use stopOn attribute for OneOrMore to avoid reading label string as part of the data attr_expr = Group(label + Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join)) OneOrMore(attr_expr).parseString(text).pprint() # Better -> [['shape', 'SQUARE'], ['posn', 'upper left'], ['color', 'BLACK']] - + # could also be written as (attr_expr * (1,)).parseString(text).pprint() """ @@ -3950,21 +4342,20 @@ class OneOrMore(_MultipleMatch): return self.strRepr class ZeroOrMore(_MultipleMatch): - """ - Optional repetition of zero or more of the given expression. - + """Optional repetition of zero or more of the given expression. + Parameters: - expr - expression that must match zero or more times - - stopOn - (default=C{None}) - expression for a terminating sentinel - (only required if the sentinel would ordinarily match the repetition - expression) + - stopOn - (default= ``None``) - expression for a terminating sentinel + (only required if the sentinel would ordinarily match the repetition + expression) - Example: similar to L{OneOrMore} + Example: similar to :class:`OneOrMore` """ def __init__( self, expr, stopOn=None): super(ZeroOrMore,self).__init__(expr, stopOn=stopOn) self.mayReturnEmpty = True - + def parseImpl( self, instring, loc, doActions=True ): try: return super(ZeroOrMore, self).parseImpl(instring, loc, doActions) @@ -3989,27 +4380,29 @@ class _NullToken(object): _optionalNotMatched = _NullToken() class Optional(ParseElementEnhance): - """ - Optional matching of the given expression. + """Optional matching of the given expression. Parameters: - expr - expression that must match zero or more times - default (optional) - value to be returned if the optional expression is not found. Example:: + # US postal code can be a 5-digit zip, plus optional 4-digit qualifier zip = Combine(Word(nums, exact=5) + Optional('-' + Word(nums, exact=4))) zip.runTests(''' # traditional ZIP code 12345 - + # ZIP+4 form 12101-0001 - + # invalid ZIP 98765- ''') + prints:: + # traditional ZIP code 12345 ['12345'] @@ -4053,20 +4446,21 @@ class Optional(ParseElementEnhance): return self.strRepr class SkipTo(ParseElementEnhance): - """ - Token for skipping over all undefined text until the matched expression is found. + """Token for skipping over all undefined text until the matched + expression is found. Parameters: - expr - target expression marking the end of the data to be skipped - - include - (default=C{False}) if True, the target expression is also parsed + - include - (default= ``False``) if True, the target expression is also parsed (the skipped text and target expression are returned as a 2-element list). - - ignore - (default=C{None}) used to define grammars (typically quoted strings and + - ignore - (default= ``None``) used to define grammars (typically quoted strings and comments) that might contain false matches to the target expression - - failOn - (default=C{None}) define expressions that are not allowed to be - included in the skipped test; if found before the target expression is found, + - failOn - (default= ``None``) define expressions that are not allowed to be + included in the skipped test; if found before the target expression is found, the SkipTo is not a match Example:: + report = ''' Outstanding Issues Report - 1 Jan 2000 @@ -4083,14 +4477,16 @@ class SkipTo(ParseElementEnhance): # - parse action will call token.strip() for each matched token, i.e., the description body string_data = SkipTo(SEP, ignore=quotedString) string_data.setParseAction(tokenMap(str.strip)) - ticket_expr = (integer("issue_num") + SEP - + string_data("sev") + SEP - + string_data("desc") + SEP + ticket_expr = (integer("issue_num") + SEP + + string_data("sev") + SEP + + string_data("desc") + SEP + integer("days_open")) - + for tkt in ticket_expr.searchString(report): print tkt.dump() + prints:: + ['101', 'Critical', 'Intermittent system crash', '6'] - days_open: 6 - desc: Intermittent system crash @@ -4127,14 +4523,14 @@ class SkipTo(ParseElementEnhance): expr_parse = self.expr._parse self_failOn_canParseNext = self.failOn.canParseNext if self.failOn is not None else None self_ignoreExpr_tryParse = self.ignoreExpr.tryParse if self.ignoreExpr is not None else None - + tmploc = loc while tmploc <= instrlen: if self_failOn_canParseNext is not None: # break if failOn expression matches if self_failOn_canParseNext(instring, tmploc): break - + if self_ignoreExpr_tryParse is not None: # advance past ignore expressions while 1: @@ -4142,7 +4538,7 @@ class SkipTo(ParseElementEnhance): tmploc = self_ignoreExpr_tryParse(instring, tmploc) except ParseBaseException: break - + try: expr_parse(instring, tmploc, doActions=False, callPreParse=False) except (ParseException, IndexError): @@ -4160,7 +4556,7 @@ class SkipTo(ParseElementEnhance): loc = tmploc skiptext = instring[startloc:loc] skipresult = ParseResults(skiptext) - + if self.includeMatch: loc, mat = expr_parse(instring,loc,doActions,callPreParse=False) skipresult += mat @@ -4168,23 +4564,31 @@ class SkipTo(ParseElementEnhance): return loc, skipresult class Forward(ParseElementEnhance): - """ - Forward declaration of an expression to be defined later - + """Forward declaration of an expression to be defined later - used for recursive grammars, such as algebraic infix notation. - When the expression is known, it is assigned to the C{Forward} variable using the '<<' operator. + When the expression is known, it is assigned to the ``Forward`` + variable using the '<<' operator. + + Note: take care when assigning to ``Forward`` not to overlook + precedence of operators. - Note: take care when assigning to C{Forward} not to overlook precedence of operators. Specifically, '|' has a lower precedence than '<<', so that:: + fwdExpr << a | b | c + will actually be evaluated as:: + (fwdExpr << a) | b | c + thereby leaving b and c out as parseable alternatives. It is recommended that you - explicitly group the values inserted into the C{Forward}:: + explicitly group the values inserted into the ``Forward``:: + fwdExpr << (a | b | c) + Converting to use the '<<=' operator instead will avoid this problem. - See L{ParseResults.pprint} for an example of a recursive parser created using - C{Forward}. + See :class:`ParseResults.pprint` for an example of a recursive + parser created using ``Forward``. """ def __init__( self, other=None ): super(Forward,self).__init__( other, savelist=False ) @@ -4201,10 +4605,10 @@ class Forward(ParseElementEnhance): self.saveAsList = self.expr.saveAsList self.ignoreExprs.extend(self.expr.ignoreExprs) return self - + def __ilshift__(self, other): return self << other - + def leaveWhitespace( self ): self.skipWhitespace = False return self @@ -4254,19 +4658,20 @@ class _ForwardNoRecurse(Forward): class TokenConverter(ParseElementEnhance): """ - Abstract subclass of C{ParseExpression}, for converting parsed results. + Abstract subclass of :class:`ParseExpression`, for converting parsed results. """ def __init__( self, expr, savelist=False ): super(TokenConverter,self).__init__( expr )#, savelist ) self.saveAsList = False class Combine(TokenConverter): - """ - Converter to concatenate all matching tokens to a single string. - By default, the matching patterns must also be contiguous in the input string; - this can be disabled by specifying C{'adjacent=False'} in the constructor. + """Converter to concatenate all matching tokens to a single string. + By default, the matching patterns must also be contiguous in the + input string; this can be disabled by specifying + ``'adjacent=False'`` in the constructor. Example:: + real = Word(nums) + '.' + Word(nums) print(real.parseString('3.1416')) # -> ['3', '.', '1416'] # will also erroneously match the following @@ -4305,10 +4710,11 @@ class Combine(TokenConverter): return retToks class Group(TokenConverter): - """ - Converter to return the matched tokens as a list - useful for returning tokens of C{L{ZeroOrMore}} and C{L{OneOrMore}} expressions. + """Converter to return the matched tokens as a list - useful for + returning tokens of :class:`ZeroOrMore` and :class:`OneOrMore` expressions. Example:: + ident = Word(alphas) num = Word(nums) term = ident | num @@ -4320,38 +4726,40 @@ class Group(TokenConverter): """ def __init__( self, expr ): super(Group,self).__init__( expr ) - self.saveAsList = True + self.saveAsList = expr.saveAsList def postParse( self, instring, loc, tokenlist ): return [ tokenlist ] class Dict(TokenConverter): - """ - Converter to return a repetitive expression as a list, but also as a dictionary. - Each element can also be referenced using the first token in the expression as its key. - Useful for tabular report scraping when the first column can be used as a item key. + """Converter to return a repetitive expression as a list, but also + as a dictionary. Each element can also be referenced using the first + token in the expression as its key. Useful for tabular report + scraping when the first column can be used as a item key. Example:: + data_word = Word(alphas) label = data_word + FollowedBy(':') attr_expr = Group(label + Suppress(':') + OneOrMore(data_word).setParseAction(' '.join)) text = "shape: SQUARE posn: upper left color: light blue texture: burlap" attr_expr = (label + Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join)) - + # print attributes as plain groups print(OneOrMore(attr_expr).parseString(text).dump()) - + # instead of OneOrMore(expr), parse using Dict(OneOrMore(Group(expr))) - Dict will auto-assign names result = Dict(OneOrMore(Group(attr_expr))).parseString(text) print(result.dump()) - - # access named fields as dict entries, or output as dict - print(result['shape']) - print(result.asDict()) - prints:: - ['shape', 'SQUARE', 'posn', 'upper left', 'color', 'light blue', 'texture', 'burlap'] + # access named fields as dict entries, or output as dict + print(result['shape']) + print(result.asDict()) + + prints:: + + ['shape', 'SQUARE', 'posn', 'upper left', 'color', 'light blue', 'texture', 'burlap'] [['shape', 'SQUARE'], ['posn', 'upper left'], ['color', 'light blue'], ['texture', 'burlap']] - color: light blue - posn: upper left @@ -4359,7 +4767,8 @@ class Dict(TokenConverter): - texture: burlap SQUARE {'color': 'light blue', 'posn': 'upper left', 'texture': 'burlap', 'shape': 'SQUARE'} - See more examples at L{ParseResults} of accessing fields by results name. + + See more examples at :class:`ParseResults` of accessing fields by results name. """ def __init__( self, expr ): super(Dict,self).__init__( expr ) @@ -4391,10 +4800,10 @@ class Dict(TokenConverter): class Suppress(TokenConverter): - """ - Converter for ignoring the results of a parsed expression. + """Converter for ignoring the results of a parsed expression. Example:: + source = "a, b, c,d" wd = Word(alphas) wd_list1 = wd + ZeroOrMore(',' + wd) @@ -4404,10 +4813,13 @@ class Suppress(TokenConverter): # way afterward - use Suppress to keep them out of the parsed output wd_list2 = wd + ZeroOrMore(Suppress(',') + wd) print(wd_list2.parseString(source)) + prints:: + ['a', ',', 'b', ',', 'c', ',', 'd'] ['a', 'b', 'c', 'd'] - (See also L{delimitedList}.) + + (See also :class:`delimitedList`.) """ def postParse( self, instring, loc, tokenlist ): return [] @@ -4417,8 +4829,7 @@ class Suppress(TokenConverter): class OnlyOnce(object): - """ - Wrapper for parse actions, to ensure they are only called once. + """Wrapper for parse actions, to ensure they are only called once. """ def __init__(self, methodCall): self.callable = _trim_arity(methodCall) @@ -4433,13 +4844,15 @@ class OnlyOnce(object): self.called = False def traceParseAction(f): - """ - Decorator for debugging parse actions. - - When the parse action is called, this decorator will print C{">> entering I{method-name}(line:I{current_source_line}, I{parse_location}, I{matched_tokens})".} - When the parse action completes, the decorator will print C{"<<"} followed by the returned value, or any exception that the parse action raised. + """Decorator for debugging parse actions. + + When the parse action is called, this decorator will print + ``">> entering method-name(line:, , )"``. + When the parse action completes, the decorator will print + ``"<<"`` followed by the returned value, or any exception that the parse action raised. Example:: + wd = Word(alphas) @traceParseAction @@ -4448,7 +4861,9 @@ def traceParseAction(f): wds = OneOrMore(wd).setParseAction(remove_duplicate_chars) print(wds.parseString("slkdjs sld sldd sdlf sdljf")) + prints:: + >>entering remove_duplicate_chars(line: 'slkdjs sld sldd sdlf sdljf', 0, (['slkdjs', 'sld', 'sldd', 'sdlf', 'sdljf'], {})) < ['aa', 'bb', 'cc'] delimitedList(Word(hexnums), delim=':', combine=True).parseString("AA:BB:CC:DD:EE") # -> ['AA:BB:CC:DD:EE'] """ @@ -4496,16 +4913,21 @@ def delimitedList( expr, delim=",", combine=False ): return ( expr + ZeroOrMore( Suppress( delim ) + expr ) ).setName(dlName) def countedArray( expr, intExpr=None ): - """ - Helper to define a counted list of expressions. + """Helper to define a counted list of expressions. + This helper defines a pattern of the form:: + integer expr expr expr... + where the leading integer tells how many expr expressions follow. - The matched tokens returns the array of expr tokens as a list - the leading count token is suppressed. - - If C{intExpr} is specified, it should be a pyparsing expression that produces an integer value. + The matched tokens returns the array of expr tokens as a list - the + leading count token is suppressed. + + If ``intExpr`` is specified, it should be a pyparsing expression + that produces an integer value. Example:: + countedArray(Word(alphas)).parseString('2 ab cd ef') # -> ['ab', 'cd'] # in this parser, the leading integer value is given in binary, @@ -4536,17 +4958,19 @@ def _flatten(L): return ret def matchPreviousLiteral(expr): - """ - Helper to define an expression that is indirectly defined from - the tokens matched in a previous expression, that is, it looks - for a 'repeat' of a previous expression. For example:: + """Helper to define an expression that is indirectly defined from + the tokens matched in a previous expression, that is, it looks for + a 'repeat' of a previous expression. For example:: + first = Word(nums) second = matchPreviousLiteral(first) matchExpr = first + ":" + second - will match C{"1:1"}, but not C{"1:2"}. Because this matches a - previous literal, will also match the leading C{"1:1"} in C{"1:10"}. - If this is not desired, use C{matchPreviousExpr}. - Do I{not} use with packrat parsing enabled. + + will match ``"1:1"``, but not ``"1:2"``. Because this + matches a previous literal, will also match the leading + ``"1:1"`` in ``"1:10"``. If this is not desired, use + :class:`matchPreviousExpr`. Do *not* use with packrat parsing + enabled. """ rep = Forward() def copyTokenToRepeater(s,l,t): @@ -4564,18 +4988,19 @@ def matchPreviousLiteral(expr): return rep def matchPreviousExpr(expr): - """ - Helper to define an expression that is indirectly defined from - the tokens matched in a previous expression, that is, it looks - for a 'repeat' of a previous expression. For example:: + """Helper to define an expression that is indirectly defined from + the tokens matched in a previous expression, that is, it looks for + a 'repeat' of a previous expression. For example:: + first = Word(nums) second = matchPreviousExpr(first) matchExpr = first + ":" + second - will match C{"1:1"}, but not C{"1:2"}. Because this matches by - expressions, will I{not} match the leading C{"1:1"} in C{"1:10"}; - the expressions are evaluated first, and then compared, so - C{"1"} is compared with C{"10"}. - Do I{not} use with packrat parsing enabled. + + will match ``"1:1"``, but not ``"1:2"``. Because this + matches by expressions, will *not* match the leading ``"1:1"`` + in ``"1:10"``; the expressions are evaluated first, and then + compared, so ``"1"`` is compared with ``"10"``. Do *not* use + with packrat parsing enabled. """ rep = Forward() e2 = expr.copy() @@ -4600,26 +5025,33 @@ def _escapeRegexRangeChars(s): return _ustr(s) def oneOf( strs, caseless=False, useRegex=True ): - """ - Helper to quickly define a set of alternative Literals, and makes sure to do - longest-first testing when there is a conflict, regardless of the input order, - but returns a C{L{MatchFirst}} for best performance. + """Helper to quickly define a set of alternative Literals, and makes + sure to do longest-first testing when there is a conflict, + regardless of the input order, but returns + a :class:`MatchFirst` for best performance. Parameters: - - strs - a string of space-delimited literals, or a collection of string literals - - caseless - (default=C{False}) - treat all literals as caseless - - useRegex - (default=C{True}) - as an optimization, will generate a Regex - object; otherwise, will generate a C{MatchFirst} object (if C{caseless=True}, or - if creating a C{Regex} raises an exception) + + - strs - a string of space-delimited literals, or a collection of + string literals + - caseless - (default= ``False``) - treat all literals as + caseless + - useRegex - (default= ``True``) - as an optimization, will + generate a Regex object; otherwise, will generate + a :class:`MatchFirst` object (if ``caseless=True``, or if + creating a :class:`Regex` raises an exception) Example:: + comp_oper = oneOf("< = > <= >= !=") var = Word(alphas) number = Word(nums) term = var | number comparison_expr = term + comp_oper + term print(comparison_expr.searchString("B = 12 AA=23 B<=AA AA>12")) + prints:: + [['B', '=', '12'], ['AA', '=', '23'], ['B', '<=', 'AA'], ['AA', '>', '12']] """ if caseless: @@ -4673,19 +5105,21 @@ def oneOf( strs, caseless=False, useRegex=True ): return MatchFirst(parseElementClass(sym) for sym in symbols).setName(' | '.join(symbols)) def dictOf( key, value ): - """ - Helper to easily and clearly define a dictionary by specifying the respective patterns - for the key and value. Takes care of defining the C{L{Dict}}, C{L{ZeroOrMore}}, and C{L{Group}} tokens - in the proper order. The key pattern can include delimiting markers or punctuation, - as long as they are suppressed, thereby leaving the significant key text. The value - pattern can include named results, so that the C{Dict} results can include named token - fields. + """Helper to easily and clearly define a dictionary by specifying + the respective patterns for the key and value. Takes care of + defining the :class:`Dict`, :class:`ZeroOrMore`, and + :class:`Group` tokens in the proper order. The key pattern + can include delimiting markers or punctuation, as long as they are + suppressed, thereby leaving the significant key text. The value + pattern can include named results, so that the :class:`Dict` results + can include named token fields. Example:: + text = "shape: SQUARE posn: upper left color: light blue texture: burlap" attr_expr = (label + Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join)) print(OneOrMore(attr_expr).parseString(text).dump()) - + attr_label = label attr_value = Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join) @@ -4695,7 +5129,9 @@ def dictOf( key, value ): print(result['shape']) print(result.shape) # object attribute access works too print(result.asDict()) + prints:: + [['shape', 'SQUARE'], ['posn', 'upper left'], ['color', 'light blue'], ['texture', 'burlap']] - color: light blue - posn: upper left @@ -4705,29 +5141,34 @@ def dictOf( key, value ): SQUARE {'color': 'light blue', 'shape': 'SQUARE', 'posn': 'upper left', 'texture': 'burlap'} """ - return Dict( ZeroOrMore( Group ( key + value ) ) ) + return Dict(OneOrMore(Group(key + value))) def originalTextFor(expr, asString=True): - """ - Helper to return the original, untokenized text for a given expression. Useful to - restore the parsed fields of an HTML start tag into the raw tag text itself, or to - revert separate tokens with intervening whitespace back to the original matching - input text. By default, returns astring containing the original parsed text. - - If the optional C{asString} argument is passed as C{False}, then the return value is a - C{L{ParseResults}} containing any results names that were originally matched, and a - single token containing the original matched text from the input string. So if - the expression passed to C{L{originalTextFor}} contains expressions with defined - results names, you must set C{asString} to C{False} if you want to preserve those - results name values. + """Helper to return the original, untokenized text for a given + expression. Useful to restore the parsed fields of an HTML start + tag into the raw tag text itself, or to revert separate tokens with + intervening whitespace back to the original matching input text. By + default, returns astring containing the original parsed text. + + If the optional ``asString`` argument is passed as + ``False``, then the return value is + a :class:`ParseResults` containing any results names that + were originally matched, and a single token containing the original + matched text from the input string. So if the expression passed to + :class:`originalTextFor` contains expressions with defined + results names, you must set ``asString`` to ``False`` if you + want to preserve those results name values. Example:: + src = "this is test bold text normal text " for tag in ("b","i"): opener,closer = makeHTMLTags(tag) patt = originalTextFor(opener + SkipTo(closer) + closer) print(patt.searchString(src)[0]) + prints:: + [' bold text '] ['text'] """ @@ -4744,29 +5185,33 @@ def originalTextFor(expr, asString=True): matchExpr.ignoreExprs = expr.ignoreExprs return matchExpr -def ungroup(expr): - """ - Helper to undo pyparsing's default grouping of And expressions, even - if all but one are non-empty. +def ungroup(expr): + """Helper to undo pyparsing's default grouping of And expressions, + even if all but one are non-empty. """ return TokenConverter(expr).setParseAction(lambda t:t[0]) def locatedExpr(expr): - """ - Helper to decorate a returned token with its starting and ending locations in the input string. + """Helper to decorate a returned token with its starting and ending + locations in the input string. + This helper adds the following results names: + - locn_start = location where matched expression begins - locn_end = location where matched expression ends - value = the actual parsed results - Be careful if the input text contains C{} characters, you may want to call - C{L{ParserElement.parseWithTabs}} + Be careful if the input text contains ```` characters, you + may want to call :class:`ParserElement.parseWithTabs` Example:: + wd = Word(alphas) for match in locatedExpr(wd).searchString("ljsdf123lksdjjf123lkkjj1222"): print(match) + prints:: + [[0, 'ljsdf', 5]] [[8, 'lksdjjf', 15]] [[18, 'lkkjj', 23]] @@ -4790,22 +5235,30 @@ _charRange = Group(_singleChar + Suppress("-") + _singleChar) _reBracketExpr = Literal("[") + Optional("^").setResultsName("negate") + Group( OneOrMore( _charRange | _singleChar ) ).setResultsName("body") + "]" def srange(s): - r""" - Helper to easily define string ranges for use in Word construction. Borrows - syntax from regexp '[]' string range definitions:: + r"""Helper to easily define string ranges for use in Word + construction. Borrows syntax from regexp '[]' string range + definitions:: + srange("[0-9]") -> "0123456789" srange("[a-z]") -> "abcdefghijklmnopqrstuvwxyz" srange("[a-z$_]") -> "abcdefghijklmnopqrstuvwxyz$_" - The input string must be enclosed in []'s, and the returned string is the expanded - character set joined into a single string. - The values enclosed in the []'s may be: + + The input string must be enclosed in []'s, and the returned string + is the expanded character set joined into a single string. The + values enclosed in the []'s may be: + - a single character - - an escaped character with a leading backslash (such as C{\-} or C{\]}) - - an escaped hex character with a leading C{'\x'} (C{\x21}, which is a C{'!'} character) - (C{\0x##} is also supported for backwards compatibility) - - an escaped octal character with a leading C{'\0'} (C{\041}, which is a C{'!'} character) - - a range of any of the above, separated by a dash (C{'a-z'}, etc.) - - any combination of the above (C{'aeiouy'}, C{'a-zA-Z0-9_$'}, etc.) + - an escaped character with a leading backslash (such as ``\-`` + or ``\]``) + - an escaped hex character with a leading ``'\x'`` + (``\x21``, which is a ``'!'`` character) (``\0x##`` + is also supported for backwards compatibility) + - an escaped octal character with a leading ``'\0'`` + (``\041``, which is a ``'!'`` character) + - a range of any of the above, separated by a dash (``'a-z'``, + etc.) + - any combination of the above (``'aeiouy'``, + ``'a-zA-Z0-9_$'``, etc.) """ _expanded = lambda p: p if not isinstance(p,ParseResults) else ''.join(unichr(c) for c in range(ord(p[0]),ord(p[1])+1)) try: @@ -4814,9 +5267,8 @@ def srange(s): return "" def matchOnlyAtCol(n): - """ - Helper method for defining parse actions that require matching at a specific - column in the input text. + """Helper method for defining parse actions that require matching at + a specific column in the input text. """ def verifyCol(strg,locn,toks): if col(locn,strg) != n: @@ -4824,24 +5276,26 @@ def matchOnlyAtCol(n): return verifyCol def replaceWith(replStr): - """ - Helper method for common parse actions that simply return a literal value. Especially - useful when used with C{L{transformString}()}. + """Helper method for common parse actions that simply return + a literal value. Especially useful when used with + :class:`transformString` (). Example:: + num = Word(nums).setParseAction(lambda toks: int(toks[0])) na = oneOf("N/A NA").setParseAction(replaceWith(math.nan)) term = na | num - + OneOrMore(term).parseString("324 234 N/A 234") # -> [324, 234, nan, 234] """ return lambda s,l,t: [replStr] def removeQuotes(s,l,t): - """ - Helper parse action for removing quotation marks from parsed quoted strings. + """Helper parse action for removing quotation marks from parsed + quoted strings. Example:: + # by default, quotation marks are included in parsed results quotedString.parseString("'Now is the Winter of our Discontent'") # -> ["'Now is the Winter of our Discontent'"] @@ -4852,18 +5306,20 @@ def removeQuotes(s,l,t): return t[0][1:-1] def tokenMap(func, *args): - """ - Helper to define a parse action by mapping a function to all elements of a ParseResults list.If any additional - args are passed, they are forwarded to the given function as additional arguments after - the token, as in C{hex_integer = Word(hexnums).setParseAction(tokenMap(int, 16))}, which will convert the - parsed data to an integer using base 16. + """Helper to define a parse action by mapping a function to all + elements of a ParseResults list. If any additional args are passed, + they are forwarded to the given function as additional arguments + after the token, as in + ``hex_integer = Word(hexnums).setParseAction(tokenMap(int, 16))``, + which will convert the parsed data to an integer using base 16. + + Example (compare the last to example in :class:`ParserElement.transformString`:: - Example (compare the last to example in L{ParserElement.transformString}:: hex_ints = OneOrMore(Word(hexnums)).setParseAction(tokenMap(int, 16)) hex_ints.runTests(''' 00 11 22 aa FF 0a 0d 1a ''') - + upperword = Word(alphas).setParseAction(tokenMap(str.upper)) OneOrMore(upperword).runTests(''' my kingdom for a horse @@ -4873,7 +5329,9 @@ def tokenMap(func, *args): OneOrMore(wd).setParseAction(' '.join).runTests(''' now is the winter of our discontent made glorious summer by this sun of york ''') + prints:: + 00 11 22 aa FF 0a 0d 1a [0, 17, 34, 170, 255, 10, 13, 26] @@ -4887,7 +5345,7 @@ def tokenMap(func, *args): return [func(tokn, *args) for tokn in t] try: - func_name = getattr(func, '__name__', + func_name = getattr(func, '__name__', getattr(func, '__class__').__name__) except Exception: func_name = str(func) @@ -4896,11 +5354,13 @@ def tokenMap(func, *args): return pa upcaseTokens = tokenMap(lambda t: _ustr(t).upper()) -"""(Deprecated) Helper parse action to convert tokens to upper case. Deprecated in favor of L{pyparsing_common.upcaseTokens}""" +"""(Deprecated) Helper parse action to convert tokens to upper case. +Deprecated in favor of :class:`pyparsing_common.upcaseTokens`""" downcaseTokens = tokenMap(lambda t: _ustr(t).lower()) -"""(Deprecated) Helper parse action to convert tokens to lower case. Deprecated in favor of L{pyparsing_common.downcaseTokens}""" - +"""(Deprecated) Helper parse action to convert tokens to lower case. +Deprecated in favor of :class:`pyparsing_common.downcaseTokens`""" + def _makeTags(tagStr, xml): """Internal helper to construct opening and closing tag expressions, given a tag name""" if isinstance(tagStr,basestring): @@ -4931,55 +5391,63 @@ def _makeTags(tagStr, xml): return openTag, closeTag def makeHTMLTags(tagStr): - """ - Helper to construct opening and closing tag expressions for HTML, given a tag name. Matches - tags in either upper or lower case, attributes with namespaces and with quoted or unquoted values. + """Helper to construct opening and closing tag expressions for HTML, + given a tag name. Matches tags in either upper or lower case, + attributes with namespaces and with quoted or unquoted values. Example:: - text = 'More info at the pyparsing wiki page' - # makeHTMLTags returns pyparsing expressions for the opening and closing tags as a 2-tuple + + text = 'More info at the pyparsing wiki page' + # makeHTMLTags returns pyparsing expressions for the opening and + # closing tags as a 2-tuple a,a_end = makeHTMLTags("A") link_expr = a + SkipTo(a_end)("link_text") + a_end - + for link in link_expr.searchString(text): - # attributes in the tag (like "href" shown here) are also accessible as named results + # attributes in the tag (like "href" shown here) are + # also accessible as named results print(link.link_text, '->', link.href) + prints:: - pyparsing -> http://pyparsing.wikispaces.com + + pyparsing -> https://github.com/pyparsing/pyparsing/wiki """ return _makeTags( tagStr, False ) def makeXMLTags(tagStr): - """ - Helper to construct opening and closing tag expressions for XML, given a tag name. Matches - tags only in the given upper/lower case. + """Helper to construct opening and closing tag expressions for XML, + given a tag name. Matches tags only in the given upper/lower case. - Example: similar to L{makeHTMLTags} + Example: similar to :class:`makeHTMLTags` """ return _makeTags( tagStr, True ) def withAttribute(*args,**attrDict): - """ - Helper to create a validating parse action to be used with start tags created - with C{L{makeXMLTags}} or C{L{makeHTMLTags}}. Use C{withAttribute} to qualify a starting tag - with a required attribute value, to avoid false matches on common tags such as - C{} or C{
}. + """Helper to create a validating parse action to be used with start + tags created with :class:`makeXMLTags` or + :class:`makeHTMLTags`. Use ``withAttribute`` to qualify + a starting tag with a required attribute value, to avoid false + matches on common tags such as ```` or ``
``. - Call C{withAttribute} with a series of attribute names and values. Specify the list - of filter attributes names and values as: - - keyword arguments, as in C{(align="right")}, or - - as an explicit dict with C{**} operator, when an attribute name is also a Python - reserved word, as in C{**{"class":"Customer", "align":"right"}} - - a list of name-value tuples, as in ( ("ns1:class", "Customer"), ("ns2:align","right") ) - For attribute names with a namespace prefix, you must use the second form. Attribute - names are matched insensitive to upper/lower case. - - If just testing for C{class} (with or without a namespace), use C{L{withClass}}. + Call ``withAttribute`` with a series of attribute names and + values. Specify the list of filter attributes names and values as: - To verify that the attribute exists, but without specifying a value, pass - C{withAttribute.ANY_VALUE} as the value. + - keyword arguments, as in ``(align="right")``, or + - as an explicit dict with ``**`` operator, when an attribute + name is also a Python reserved word, as in ``**{"class":"Customer", "align":"right"}`` + - a list of name-value tuples, as in ``(("ns1:class", "Customer"), ("ns2:align","right"))`` + + For attribute names with a namespace prefix, you must use the second + form. Attribute names are matched insensitive to upper/lower case. + + If just testing for ``class`` (with or without a namespace), use + :class:`withClass`. + + To verify that the attribute exists, but without specifying a value, + pass ``withAttribute.ANY_VALUE`` as the value. Example:: + html = '''
Some text @@ -4987,7 +5455,7 @@ def withAttribute(*args,**attrDict):
1,3 2,3 1,1
this has no type
- + ''' div,div_end = makeHTMLTags("div") @@ -4996,13 +5464,15 @@ def withAttribute(*args,**attrDict): grid_expr = div_grid + SkipTo(div | div_end)("body") for grid_header in grid_expr.searchString(html): print(grid_header.body) - + # construct a match with any div tag having a type attribute, regardless of the value div_any_type = div().setParseAction(withAttribute(type=withAttribute.ANY_VALUE)) div_expr = div_any_type + SkipTo(div | div_end)("body") for div_header in div_expr.searchString(html): print(div_header.body) + prints:: + 1 4 0 1 0 1 4 0 1 0 @@ -5024,11 +5494,12 @@ def withAttribute(*args,**attrDict): withAttribute.ANY_VALUE = object() def withClass(classname, namespace=''): - """ - Simplified version of C{L{withAttribute}} when matching on a div class - made - difficult because C{class} is a reserved word in Python. + """Simplified version of :class:`withAttribute` when + matching on a div class - made difficult because ``class`` is + a reserved word in Python. Example:: + html = '''
Some text @@ -5036,84 +5507,96 @@ def withClass(classname, namespace=''):
1,3 2,3 1,1
this <div> has no class
- + ''' div,div_end = makeHTMLTags("div") div_grid = div().setParseAction(withClass("grid")) - + grid_expr = div_grid + SkipTo(div | div_end)("body") for grid_header in grid_expr.searchString(html): print(grid_header.body) - + div_any_type = div().setParseAction(withClass(withAttribute.ANY_VALUE)) div_expr = div_any_type + SkipTo(div | div_end)("body") for div_header in div_expr.searchString(html): print(div_header.body) + prints:: + 1 4 0 1 0 1 4 0 1 0 1,3 2,3 1,1 """ classattr = "%s:class" % namespace if namespace else "class" - return withAttribute(**{classattr : classname}) + return withAttribute(**{classattr : classname}) -opAssoc = _Constants() +opAssoc = SimpleNamespace() opAssoc.LEFT = object() opAssoc.RIGHT = object() def infixNotation( baseExpr, opList, lpar=Suppress('('), rpar=Suppress(')') ): - """ - Helper method for constructing grammars of expressions made up of - operators working in a precedence hierarchy. Operators may be unary or - binary, left- or right-associative. Parse actions can also be attached - to operator expressions. The generated parser will also recognize the use - of parentheses to override operator precedences (see example below). - - Note: if you define a deep operator list, you may see performance issues - when using infixNotation. See L{ParserElement.enablePackrat} for a - mechanism to potentially improve your parser performance. + """Helper method for constructing grammars of expressions made up of + operators working in a precedence hierarchy. Operators may be unary + or binary, left- or right-associative. Parse actions can also be + attached to operator expressions. The generated parser will also + recognize the use of parentheses to override operator precedences + (see example below). + + Note: if you define a deep operator list, you may see performance + issues when using infixNotation. See + :class:`ParserElement.enablePackrat` for a mechanism to potentially + improve your parser performance. Parameters: - - baseExpr - expression representing the most basic element for the nested - - opList - list of tuples, one for each operator precedence level in the - expression grammar; each tuple is of the form - (opExpr, numTerms, rightLeftAssoc, parseAction), where: - - opExpr is the pyparsing expression for the operator; - may also be a string, which will be converted to a Literal; - if numTerms is 3, opExpr is a tuple of two expressions, for the - two operators separating the 3 terms - - numTerms is the number of terms for this operator (must - be 1, 2, or 3) - - rightLeftAssoc is the indicator whether the operator is - right or left associative, using the pyparsing-defined - constants C{opAssoc.RIGHT} and C{opAssoc.LEFT}. + - baseExpr - expression representing the most basic element for the + nested + - opList - list of tuples, one for each operator precedence level + in the expression grammar; each tuple is of the form ``(opExpr, + numTerms, rightLeftAssoc, parseAction)``, where: + + - opExpr is the pyparsing expression for the operator; may also + be a string, which will be converted to a Literal; if numTerms + is 3, opExpr is a tuple of two expressions, for the two + operators separating the 3 terms + - numTerms is the number of terms for this operator (must be 1, + 2, or 3) + - rightLeftAssoc is the indicator whether the operator is right + or left associative, using the pyparsing-defined constants + ``opAssoc.RIGHT`` and ``opAssoc.LEFT``. - parseAction is the parse action to be associated with - expressions matching this operator expression (the - parse action tuple member may be omitted); if the parse action - is passed a tuple or list of functions, this is equivalent to - calling C{setParseAction(*fn)} (L{ParserElement.setParseAction}) - - lpar - expression for matching left-parentheses (default=C{Suppress('(')}) - - rpar - expression for matching right-parentheses (default=C{Suppress(')')}) + expressions matching this operator expression (the parse action + tuple member may be omitted); if the parse action is passed + a tuple or list of functions, this is equivalent to calling + ``setParseAction(*fn)`` + (:class:`ParserElement.setParseAction`) + - lpar - expression for matching left-parentheses + (default= ``Suppress('(')``) + - rpar - expression for matching right-parentheses + (default= ``Suppress(')')``) Example:: - # simple example of four-function arithmetic with ints and variable names + + # simple example of four-function arithmetic with ints and + # variable names integer = pyparsing_common.signed_integer - varname = pyparsing_common.identifier - + varname = pyparsing_common.identifier + arith_expr = infixNotation(integer | varname, [ ('-', 1, opAssoc.RIGHT), (oneOf('* /'), 2, opAssoc.LEFT), (oneOf('+ -'), 2, opAssoc.LEFT), ]) - + arith_expr.runTests(''' 5+3*6 (5+3)*6 -2--11 ''', fullDump=False) + prints:: + 5+3*6 [[5, '+', [3, '*', 6]]] @@ -5123,6 +5606,12 @@ def infixNotation( baseExpr, opList, lpar=Suppress('('), rpar=Suppress(')') ): -2--11 [[['-', 2], '-', ['-', 11]]] """ + # captive version of FollowedBy that does not do parse actions or capture results names + class _FB(FollowedBy): + def parseImpl(self, instring, loc, doActions=True): + self.expr.tryParse(instring, loc) + return loc, [] + ret = Forward() lastExpr = baseExpr | ( lpar + ret + rpar ) for i,operDef in enumerate(opList): @@ -5130,19 +5619,20 @@ def infixNotation( baseExpr, opList, lpar=Suppress('('), rpar=Suppress(')') ): termName = "%s term" % opExpr if arity < 3 else "%s%s term" % opExpr if arity == 3: if opExpr is None or len(opExpr) != 2: - raise ValueError("if numterms=3, opExpr must be a tuple or list of two expressions") + raise ValueError( + "if numterms=3, opExpr must be a tuple or list of two expressions") opExpr1, opExpr2 = opExpr thisExpr = Forward().setName(termName) if rightLeftAssoc == opAssoc.LEFT: if arity == 1: - matchExpr = FollowedBy(lastExpr + opExpr) + Group( lastExpr + OneOrMore( opExpr ) ) + matchExpr = _FB(lastExpr + opExpr) + Group( lastExpr + OneOrMore( opExpr ) ) elif arity == 2: if opExpr is not None: - matchExpr = FollowedBy(lastExpr + opExpr + lastExpr) + Group( lastExpr + OneOrMore( opExpr + lastExpr ) ) + matchExpr = _FB(lastExpr + opExpr + lastExpr) + Group( lastExpr + OneOrMore( opExpr + lastExpr ) ) else: - matchExpr = FollowedBy(lastExpr+lastExpr) + Group( lastExpr + OneOrMore(lastExpr) ) + matchExpr = _FB(lastExpr+lastExpr) + Group( lastExpr + OneOrMore(lastExpr) ) elif arity == 3: - matchExpr = FollowedBy(lastExpr + opExpr1 + lastExpr + opExpr2 + lastExpr) + \ + matchExpr = _FB(lastExpr + opExpr1 + lastExpr + opExpr2 + lastExpr) + \ Group( lastExpr + opExpr1 + lastExpr + opExpr2 + lastExpr ) else: raise ValueError("operator must be unary (1), binary (2), or ternary (3)") @@ -5151,14 +5641,14 @@ def infixNotation( baseExpr, opList, lpar=Suppress('('), rpar=Suppress(')') ): # try to avoid LR with this extra test if not isinstance(opExpr, Optional): opExpr = Optional(opExpr) - matchExpr = FollowedBy(opExpr.expr + thisExpr) + Group( opExpr + thisExpr ) + matchExpr = _FB(opExpr.expr + thisExpr) + Group( opExpr + thisExpr ) elif arity == 2: if opExpr is not None: - matchExpr = FollowedBy(lastExpr + opExpr + thisExpr) + Group( lastExpr + OneOrMore( opExpr + thisExpr ) ) + matchExpr = _FB(lastExpr + opExpr + thisExpr) + Group( lastExpr + OneOrMore( opExpr + thisExpr ) ) else: - matchExpr = FollowedBy(lastExpr + thisExpr) + Group( lastExpr + OneOrMore( thisExpr ) ) + matchExpr = _FB(lastExpr + thisExpr) + Group( lastExpr + OneOrMore( thisExpr ) ) elif arity == 3: - matchExpr = FollowedBy(lastExpr + opExpr1 + thisExpr + opExpr2 + thisExpr) + \ + matchExpr = _FB(lastExpr + opExpr1 + thisExpr + opExpr2 + thisExpr) + \ Group( lastExpr + opExpr1 + thisExpr + opExpr2 + thisExpr ) else: raise ValueError("operator must be unary (1), binary (2), or ternary (3)") @@ -5175,7 +5665,8 @@ def infixNotation( baseExpr, opList, lpar=Suppress('('), rpar=Suppress(')') ): return ret operatorPrecedence = infixNotation -"""(Deprecated) Former name of C{L{infixNotation}}, will be dropped in a future release.""" +"""(Deprecated) Former name of :class:`infixNotation`, will be +dropped in a future release.""" dblQuotedString = Combine(Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*')+'"').setName("string enclosed in double quotes") sglQuotedString = Combine(Regex(r"'(?:[^'\n\r\\]|(?:'')|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*")+"'").setName("string enclosed in single quotes") @@ -5184,28 +5675,33 @@ quotedString = Combine(Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\(?:[^x]|x[0-9a-fA-F]+) unicodeString = Combine(_L('u') + quotedString.copy()).setName("unicode string literal") def nestedExpr(opener="(", closer=")", content=None, ignoreExpr=quotedString.copy()): - """ - Helper method for defining nested lists enclosed in opening and closing - delimiters ("(" and ")" are the default). + """Helper method for defining nested lists enclosed in opening and + closing delimiters ("(" and ")" are the default). Parameters: - - opener - opening character for a nested list (default=C{"("}); can also be a pyparsing expression - - closer - closing character for a nested list (default=C{")"}); can also be a pyparsing expression - - content - expression for items within the nested lists (default=C{None}) - - ignoreExpr - expression for ignoring opening and closing delimiters (default=C{quotedString}) + - opener - opening character for a nested list + (default= ``"("``); can also be a pyparsing expression + - closer - closing character for a nested list + (default= ``")"``); can also be a pyparsing expression + - content - expression for items within the nested lists + (default= ``None``) + - ignoreExpr - expression for ignoring opening and closing + delimiters (default= :class:`quotedString`) - If an expression is not provided for the content argument, the nested - expression will capture all whitespace-delimited content between delimiters - as a list of separate values. + If an expression is not provided for the content argument, the + nested expression will capture all whitespace-delimited content + between delimiters as a list of separate values. - Use the C{ignoreExpr} argument to define expressions that may contain - opening or closing characters that should not be treated as opening - or closing characters for nesting, such as quotedString or a comment - expression. Specify multiple expressions using an C{L{Or}} or C{L{MatchFirst}}. - The default is L{quotedString}, but if no expressions are to be ignored, - then pass C{None} for this argument. + Use the ``ignoreExpr`` argument to define expressions that may + contain opening or closing characters that should not be treated as + opening or closing characters for nesting, such as quotedString or + a comment expression. Specify multiple expressions using an + :class:`Or` or :class:`MatchFirst`. The default is + :class:`quotedString`, but if no expressions are to be ignored, then + pass ``None`` for this argument. Example:: + data_type = oneOf("void int short long char float double") decl_data_type = Combine(data_type + Optional(Word('*'))) ident = Word(alphas+'_', alphanums+'_') @@ -5215,29 +5711,31 @@ def nestedExpr(opener="(", closer=")", content=None, ignoreExpr=quotedString.cop code_body = nestedExpr('{', '}', ignoreExpr=(quotedString | cStyleComment)) - c_function = (decl_data_type("type") + c_function = (decl_data_type("type") + ident("name") - + LPAR + Optional(delimitedList(arg), [])("args") + RPAR + + LPAR + Optional(delimitedList(arg), [])("args") + RPAR + code_body("body")) c_function.ignore(cStyleComment) - + source_code = ''' - int is_odd(int x) { - return (x%2); + int is_odd(int x) { + return (x%2); } - - int dec_to_hex(char hchar) { - if (hchar >= '0' && hchar <= '9') { - return (ord(hchar)-ord('0')); - } else { + + int dec_to_hex(char hchar) { + if (hchar >= '0' && hchar <= '9') { + return (ord(hchar)-ord('0')); + } else { return (10+ord(hchar)-ord('A')); - } + } } ''' for func in c_function.searchString(source_code): print("%(name)s (%(type)s) args: %(args)s" % func) + prints:: + is_odd (int) args: [['int', 'x']] dec_to_hex (int) args: [['char', 'hchar']] """ @@ -5255,7 +5753,7 @@ def nestedExpr(opener="(", closer=")", content=None, ignoreExpr=quotedString.cop ).setParseAction(lambda t:t[0].strip())) else: if ignoreExpr is not None: - content = (Combine(OneOrMore(~ignoreExpr + + content = (Combine(OneOrMore(~ignoreExpr + ~Literal(opener) + ~Literal(closer) + CharsNotIn(ParserElement.DEFAULT_WHITE_CHARS,exact=1)) ).setParseAction(lambda t:t[0].strip())) @@ -5274,23 +5772,24 @@ def nestedExpr(opener="(", closer=")", content=None, ignoreExpr=quotedString.cop return ret def indentedBlock(blockStatementExpr, indentStack, indent=True): - """ - Helper method for defining space-delimited indentation blocks, such as - those used to define block statements in Python source code. + """Helper method for defining space-delimited indentation blocks, + such as those used to define block statements in Python source code. Parameters: - - blockStatementExpr - expression defining syntax of statement that - is repeated within the indented block - - indentStack - list created by caller to manage indentation stack - (multiple statementWithIndentedBlock expressions within a single grammar - should share a common indentStack) - - indent - boolean indicating whether block must be indented beyond the - the current level; set to False for block of left-most statements - (default=C{True}) - A valid block must contain at least one C{blockStatement}. + - blockStatementExpr - expression defining syntax of statement that + is repeated within the indented block + - indentStack - list created by caller to manage indentation stack + (multiple statementWithIndentedBlock expressions within a single + grammar should share a common indentStack) + - indent - boolean indicating whether block must be indented beyond + the the current level; set to False for block of left-most + statements (default= ``True``) + + A valid block must contain at least one ``blockStatement``. Example:: + data = ''' def A(z): A1 @@ -5331,7 +5830,9 @@ def indentedBlock(blockStatementExpr, indentStack, indent=True): parseTree = module_body.parseString(data) parseTree.pprint() + prints:: + [['def', 'A', ['(', 'z', ')'], @@ -5349,7 +5850,7 @@ def indentedBlock(blockStatementExpr, indentStack, indent=True): 'spam', ['(', 'x', 'y', ')'], ':', - [[['def', 'eggs', ['(', 'z', ')'], ':', [['pass']]]]]]] + [[['def', 'eggs', ['(', 'z', ')'], ':', [['pass']]]]]]] """ def checkPeerIndent(s,l,t): if l >= len(s): return @@ -5399,51 +5900,61 @@ def replaceHTMLEntity(t): # it's easy to get these comment structures wrong - they're very common, so may as well make them available cStyleComment = Combine(Regex(r"/\*(?:[^*]|\*(?!/))*") + '*/').setName("C style comment") -"Comment of the form C{/* ... */}" +"Comment of the form ``/* ... */``" htmlComment = Regex(r"").setName("HTML comment") -"Comment of the form C{}" +"Comment of the form ````" restOfLine = Regex(r".*").leaveWhitespace().setName("rest of line") dblSlashComment = Regex(r"//(?:\\\n|[^\n])*").setName("// comment") -"Comment of the form C{// ... (to end of line)}" +"Comment of the form ``// ... (to end of line)``" cppStyleComment = Combine(Regex(r"/\*(?:[^*]|\*(?!/))*") + '*/'| dblSlashComment).setName("C++ style comment") -"Comment of either form C{L{cStyleComment}} or C{L{dblSlashComment}}" +"Comment of either form :class:`cStyleComment` or :class:`dblSlashComment`" javaStyleComment = cppStyleComment -"Same as C{L{cppStyleComment}}" +"Same as :class:`cppStyleComment`" pythonStyleComment = Regex(r"#.*").setName("Python style comment") -"Comment of the form C{# ... (to end of line)}" +"Comment of the form ``# ... (to end of line)``" _commasepitem = Combine(OneOrMore(Word(printables, excludeChars=',') + Optional( Word(" \t") + ~Literal(",") + ~LineEnd() ) ) ).streamline().setName("commaItem") commaSeparatedList = delimitedList( Optional( quotedString.copy() | _commasepitem, default="") ).setName("commaSeparatedList") -"""(Deprecated) Predefined expression of 1 or more printable words or quoted strings, separated by commas. - This expression is deprecated in favor of L{pyparsing_common.comma_separated_list}.""" +"""(Deprecated) Predefined expression of 1 or more printable words or +quoted strings, separated by commas. + +This expression is deprecated in favor of :class:`pyparsing_common.comma_separated_list`. +""" # some other useful expressions - using lower-case class name since we are really using this as a namespace class pyparsing_common: - """ - Here are some common low-level expressions that may be useful in jump-starting parser development: - - numeric forms (L{integers}, L{reals}, L{scientific notation}) - - common L{programming identifiers} - - network addresses (L{MAC}, L{IPv4}, L{IPv6}) - - ISO8601 L{dates} and L{datetime} - - L{UUID} - - L{comma-separated list} + """Here are some common low-level expressions that may be useful in + jump-starting parser development: + + - numeric forms (:class:`integers`, :class:`reals`, + :class:`scientific notation`) + - common :class:`programming identifiers` + - network addresses (:class:`MAC`, + :class:`IPv4`, :class:`IPv6`) + - ISO8601 :class:`dates` and + :class:`datetime` + - :class:`UUID` + - :class:`comma-separated list` + Parse actions: - - C{L{convertToInteger}} - - C{L{convertToFloat}} - - C{L{convertToDate}} - - C{L{convertToDatetime}} - - C{L{stripHTMLTags}} - - C{L{upcaseTokens}} - - C{L{downcaseTokens}} + + - :class:`convertToInteger` + - :class:`convertToFloat` + - :class:`convertToDate` + - :class:`convertToDatetime` + - :class:`stripHTMLTags` + - :class:`upcaseTokens` + - :class:`downcaseTokens` Example:: + pyparsing_common.number.runTests(''' # any int or real number, returned as the appropriate type 100 @@ -5490,7 +6001,9 @@ class pyparsing_common: # uuid 12345678-1234-5678-1234-567812345678 ''') + prints:: + # any int or real number, returned as the appropriate type 100 [100] @@ -5592,7 +6105,8 @@ class pyparsing_common: """expression that parses a floating point number and returns a float""" sci_real = Regex(r'[+-]?\d+([eE][+-]?\d+|\.\d*([eE][+-]?\d+)?)').setName("real number with scientific notation").setParseAction(convertToFloat) - """expression that parses a floating point number with optional scientific notation and returns a float""" + """expression that parses a floating point number with optional + scientific notation and returns a float""" # streamlining this expression makes the docs nicer-looking number = (sci_real | real | signed_integer).streamline() @@ -5600,12 +6114,12 @@ class pyparsing_common: fnumber = Regex(r'[+-]?\d+\.?\d*([eE][+-]?\d+)?').setName("fnumber").setParseAction(convertToFloat) """any int or real number, returned as float""" - + identifier = Word(alphas+'_', alphanums+'_').setName("identifier") """typical code identifier (leading alpha or '_', followed by 0 or more alphas, nums, or '_')""" - + ipv4_address = Regex(r'(25[0-5]|2[0-4][0-9]|1?[0-9]{1,2})(\.(25[0-5]|2[0-4][0-9]|1?[0-9]{1,2})){3}').setName("IPv4 address") - "IPv4 address (C{0.0.0.0 - 255.255.255.255})" + "IPv4 address (``0.0.0.0 - 255.255.255.255``)" _ipv6_part = Regex(r'[0-9a-fA-F]{1,4}').setName("hex_integer") _full_ipv6_address = (_ipv6_part + (':' + _ipv6_part)*7).setName("full IPv6 address") @@ -5614,7 +6128,7 @@ class pyparsing_common: _mixed_ipv6_address = ("::ffff:" + ipv4_address).setName("mixed IPv6 address") ipv6_address = Combine((_full_ipv6_address | _mixed_ipv6_address | _short_ipv6_address).setName("IPv6 address")).setName("IPv6 address") "IPv6 address (long, short, or mixed form)" - + mac_address = Regex(r'[0-9a-fA-F]{2}([:.-])[0-9a-fA-F]{2}(?:\1[0-9a-fA-F]{2}){4}').setName("MAC address") "MAC address xx:xx:xx:xx:xx (may also have '-' or '.' delimiters)" @@ -5624,13 +6138,16 @@ class pyparsing_common: Helper to create a parse action for converting parsed date string to Python datetime.date Params - - - fmt - format to be passed to datetime.strptime (default=C{"%Y-%m-%d"}) + - fmt - format to be passed to datetime.strptime (default= ``"%Y-%m-%d"``) Example:: + date_expr = pyparsing_common.iso8601_date.copy() date_expr.setParseAction(pyparsing_common.convertToDate()) print(date_expr.parseString("1999-12-31")) + prints:: + [datetime.date(1999, 12, 31)] """ def cvt_fn(s,l,t): @@ -5642,17 +6159,20 @@ class pyparsing_common: @staticmethod def convertToDatetime(fmt="%Y-%m-%dT%H:%M:%S.%f"): - """ - Helper to create a parse action for converting parsed datetime string to Python datetime.datetime + """Helper to create a parse action for converting parsed + datetime string to Python datetime.datetime Params - - - fmt - format to be passed to datetime.strptime (default=C{"%Y-%m-%dT%H:%M:%S.%f"}) + - fmt - format to be passed to datetime.strptime (default= ``"%Y-%m-%dT%H:%M:%S.%f"``) Example:: + dt_expr = pyparsing_common.iso8601_datetime.copy() dt_expr.setParseAction(pyparsing_common.convertToDatetime()) print(dt_expr.parseString("1999-12-31T23:59:59.999")) + prints:: + [datetime.datetime(1999, 12, 31, 23, 59, 59, 999000)] """ def cvt_fn(s,l,t): @@ -5663,31 +6183,34 @@ class pyparsing_common: return cvt_fn iso8601_date = Regex(r'(?P\d{4})(?:-(?P\d\d)(?:-(?P\d\d))?)?').setName("ISO8601 date") - "ISO8601 date (C{yyyy-mm-dd})" + "ISO8601 date (``yyyy-mm-dd``)" iso8601_datetime = Regex(r'(?P\d{4})-(?P\d\d)-(?P\d\d)[T ](?P\d\d):(?P\d\d)(:(?P\d\d(\.\d*)?)?)?(?PZ|[+-]\d\d:?\d\d)?').setName("ISO8601 datetime") - "ISO8601 datetime (C{yyyy-mm-ddThh:mm:ss.s(Z|+-00:00)}) - trailing seconds, milliseconds, and timezone optional; accepts separating C{'T'} or C{' '}" + "ISO8601 datetime (``yyyy-mm-ddThh:mm:ss.s(Z|+-00:00)``) - trailing seconds, milliseconds, and timezone optional; accepts separating ``'T'`` or ``' '``" uuid = Regex(r'[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}').setName("UUID") - "UUID (C{xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx})" + "UUID (``xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx``)" _html_stripper = anyOpenTag.suppress() | anyCloseTag.suppress() @staticmethod def stripHTMLTags(s, l, tokens): - """ - Parse action to remove HTML tags from web page HTML source + """Parse action to remove HTML tags from web page HTML source Example:: - # strip HTML links from normal text - text = 'More info at the pyparsing wiki page' + + # strip HTML links from normal text + text = 'More info at the pyparsing wiki page' td,td_end = makeHTMLTags("TD") table_text = td + SkipTo(td_end).setParseAction(pyparsing_common.stripHTMLTags)("body") + td_end - - print(table_text.parseString(text).body) # -> 'More info at the pyparsing wiki page' + print(table_text.parseString(text).body) + + Prints:: + + More info at the pyparsing wiki page """ return pyparsing_common._html_stripper.transformString(tokens[0]) - _commasepitem = Combine(OneOrMore(~Literal(",") + ~LineEnd() + Word(printables, excludeChars=',') + _commasepitem = Combine(OneOrMore(~Literal(",") + ~LineEnd() + Word(printables, excludeChars=',') + Optional( White(" \t") ) ) ).streamline().setName("commaItem") comma_separated_list = delimitedList( Optional( quotedString.copy() | _commasepitem, default="") ).setName("comma separated list") """Predefined expression of 1 or more printable words or quoted strings, separated by commas.""" @@ -5699,6 +6222,164 @@ class pyparsing_common: """Parse action to convert tokens to lower case.""" +class _lazyclassproperty(object): + def __init__(self, fn): + self.fn = fn + self.__doc__ = fn.__doc__ + self.__name__ = fn.__name__ + + def __get__(self, obj, cls): + if cls is None: + cls = type(obj) + if not hasattr(cls, '_intern') or any(cls._intern is getattr(superclass, '_intern', []) for superclass in cls.__mro__[1:]): + cls._intern = {} + attrname = self.fn.__name__ + if attrname not in cls._intern: + cls._intern[attrname] = self.fn(cls) + return cls._intern[attrname] + + +class unicode_set(object): + """ + A set of Unicode characters, for language-specific strings for + ``alphas``, ``nums``, ``alphanums``, and ``printables``. + A unicode_set is defined by a list of ranges in the Unicode character + set, in a class attribute ``_ranges``, such as:: + + _ranges = [(0x0020, 0x007e), (0x00a0, 0x00ff),] + + A unicode set can also be defined using multiple inheritance of other unicode sets:: + + class CJK(Chinese, Japanese, Korean): + pass + """ + _ranges = [] + + @classmethod + def _get_chars_for_ranges(cls): + ret = [] + for cc in cls.__mro__: + if cc is unicode_set: + break + for rr in cc._ranges: + ret.extend(range(rr[0], rr[-1]+1)) + return [unichr(c) for c in sorted(set(ret))] + + @_lazyclassproperty + def printables(cls): + "all non-whitespace characters in this range" + return u''.join(filterfalse(unicode.isspace, cls._get_chars_for_ranges())) + + @_lazyclassproperty + def alphas(cls): + "all alphabetic characters in this range" + return u''.join(filter(unicode.isalpha, cls._get_chars_for_ranges())) + + @_lazyclassproperty + def nums(cls): + "all numeric digit characters in this range" + return u''.join(filter(unicode.isdigit, cls._get_chars_for_ranges())) + + @_lazyclassproperty + def alphanums(cls): + "all alphanumeric characters in this range" + return cls.alphas + cls.nums + + +class pyparsing_unicode(unicode_set): + """ + A namespace class for defining common language unicode_sets. + """ + _ranges = [(32, sys.maxunicode)] + + class Latin1(unicode_set): + "Unicode set for Latin-1 Unicode Character Range" + _ranges = [(0x0020, 0x007e), (0x00a0, 0x00ff),] + + class LatinA(unicode_set): + "Unicode set for Latin-A Unicode Character Range" + _ranges = [(0x0100, 0x017f),] + + class LatinB(unicode_set): + "Unicode set for Latin-B Unicode Character Range" + _ranges = [(0x0180, 0x024f),] + + class Greek(unicode_set): + "Unicode set for Greek Unicode Character Ranges" + _ranges = [ + (0x0370, 0x03ff), (0x1f00, 0x1f15), (0x1f18, 0x1f1d), (0x1f20, 0x1f45), (0x1f48, 0x1f4d), + (0x1f50, 0x1f57), (0x1f59,), (0x1f5b,), (0x1f5d,), (0x1f5f, 0x1f7d), (0x1f80, 0x1fb4), (0x1fb6, 0x1fc4), + (0x1fc6, 0x1fd3), (0x1fd6, 0x1fdb), (0x1fdd, 0x1fef), (0x1ff2, 0x1ff4), (0x1ff6, 0x1ffe), + ] + + class Cyrillic(unicode_set): + "Unicode set for Cyrillic Unicode Character Range" + _ranges = [(0x0400, 0x04ff)] + + class Chinese(unicode_set): + "Unicode set for Chinese Unicode Character Range" + _ranges = [(0x4e00, 0x9fff), (0x3000, 0x303f), ] + + class Japanese(unicode_set): + "Unicode set for Japanese Unicode Character Range, combining Kanji, Hiragana, and Katakana ranges" + _ranges = [ ] + + class Kanji(unicode_set): + "Unicode set for Kanji Unicode Character Range" + _ranges = [(0x4E00, 0x9Fbf), (0x3000, 0x303f), ] + + class Hiragana(unicode_set): + "Unicode set for Hiragana Unicode Character Range" + _ranges = [(0x3040, 0x309f), ] + + class Katakana(unicode_set): + "Unicode set for Katakana Unicode Character Range" + _ranges = [(0x30a0, 0x30ff), ] + + class Korean(unicode_set): + "Unicode set for Korean Unicode Character Range" + _ranges = [(0xac00, 0xd7af), (0x1100, 0x11ff), (0x3130, 0x318f), (0xa960, 0xa97f), (0xd7b0, 0xd7ff), (0x3000, 0x303f), ] + + class CJK(Chinese, Japanese, Korean): + "Unicode set for combined Chinese, Japanese, and Korean (CJK) Unicode Character Range" + pass + + class Thai(unicode_set): + "Unicode set for Thai Unicode Character Range" + _ranges = [(0x0e01, 0x0e3a), (0x0e3f, 0x0e5b), ] + + class Arabic(unicode_set): + "Unicode set for Arabic Unicode Character Range" + _ranges = [(0x0600, 0x061b), (0x061e, 0x06ff), (0x0700, 0x077f), ] + + class Hebrew(unicode_set): + "Unicode set for Hebrew Unicode Character Range" + _ranges = [(0x0590, 0x05ff), ] + + class Devanagari(unicode_set): + "Unicode set for Devanagari Unicode Character Range" + _ranges = [(0x0900, 0x097f), (0xa8e0, 0xa8ff)] + +pyparsing_unicode.Japanese._ranges = (pyparsing_unicode.Japanese.Kanji._ranges + + pyparsing_unicode.Japanese.Hiragana._ranges + + pyparsing_unicode.Japanese.Katakana._ranges) + +# define ranges in language character sets +if PY_3: + setattr(pyparsing_unicode, "العربية", pyparsing_unicode.Arabic) + setattr(pyparsing_unicode, "中文", pyparsing_unicode.Chinese) + setattr(pyparsing_unicode, "кириллица", pyparsing_unicode.Cyrillic) + setattr(pyparsing_unicode, "Ελληνικά", pyparsing_unicode.Greek) + setattr(pyparsing_unicode, "עִברִית", pyparsing_unicode.Hebrew) + setattr(pyparsing_unicode, "日本語", pyparsing_unicode.Japanese) + setattr(pyparsing_unicode.Japanese, "漢字", pyparsing_unicode.Japanese.Kanji) + setattr(pyparsing_unicode.Japanese, "カタカナ", pyparsing_unicode.Japanese.Katakana) + setattr(pyparsing_unicode.Japanese, "ひらがな", pyparsing_unicode.Japanese.Hiragana) + setattr(pyparsing_unicode, "한국어", pyparsing_unicode.Korean) + setattr(pyparsing_unicode, "ไทย", pyparsing_unicode.Thai) + setattr(pyparsing_unicode, "देवनागरी", pyparsing_unicode.Devanagari) + + if __name__ == "__main__": selectToken = CaselessLiteral("select") @@ -5712,7 +6393,7 @@ if __name__ == "__main__": tableName = delimitedList(ident, ".", combine=True).setParseAction(upcaseTokens) tableNameList = Group(delimitedList(tableName)).setName("tables") - + simpleSQL = selectToken("command") + columnSpec("columns") + fromToken + tableNameList("tables") # demo runTests method, including embedded comments in test string diff --git a/pipenv/vendor/pythonfinder/__init__.py b/pipenv/vendor/pythonfinder/__init__.py index 8e12a198..428599bd 100644 --- a/pipenv/vendor/pythonfinder/__init__.py +++ b/pipenv/vendor/pythonfinder/__init__.py @@ -1,16 +1,19 @@ -from __future__ import print_function, absolute_import - -__version__ = '1.1.10' +from __future__ import absolute_import, print_function # Add NullHandler to "pythonfinder" logger, because Python2's default root # logger has no handler and warnings like this would be reported: # # > No handlers could be found for logger "pythonfinder.models.pyenv" import logging + +from .exceptions import InvalidPythonVersion +from .models import SystemPath, WindowsFinder +from .pythonfinder import Finder + +__version__ = "1.2.2.dev0" + + logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) __all__ = ["Finder", "WindowsFinder", "SystemPath", "InvalidPythonVersion"] -from .pythonfinder import Finder -from .models import SystemPath, WindowsFinder -from .exceptions import InvalidPythonVersion diff --git a/pipenv/vendor/pythonfinder/__main__.py b/pipenv/vendor/pythonfinder/__main__.py index c804d573..3083e72d 100644 --- a/pipenv/vendor/pythonfinder/__main__.py +++ b/pipenv/vendor/pythonfinder/__main__.py @@ -1,12 +1,17 @@ +#!env python +# -*- coding=utf-8 -*- + from __future__ import absolute_import import os import sys +from pythonfinder.cli import cli + + PYTHONFINDER_MAIN = os.path.dirname(os.path.abspath(__file__)) PYTHONFINDER_PACKAGE = os.path.dirname(PYTHONFINDER_MAIN) -from pythonfinder import cli as cli if __name__ == "__main__": sys.exit(cli()) diff --git a/pipenv/vendor/blindspin/LICENSE b/pipenv/vendor/pythonfinder/_vendor/pep514tools/LICENSE similarity index 95% rename from pipenv/vendor/blindspin/LICENSE rename to pipenv/vendor/pythonfinder/_vendor/pep514tools/LICENSE index 00bf847d..c7ac395f 100644 --- a/pipenv/vendor/blindspin/LICENSE +++ b/pipenv/vendor/pythonfinder/_vendor/pep514tools/LICENSE @@ -1,6 +1,6 @@ -The MIT License (MIT) +MIT License -Copyright 2018 Kenneth Reitz +Copyright (c) 2016 Steve Dower Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pipenv/vendor/pythonfinder/_vendor/pep514tools/_registry.py b/pipenv/vendor/pythonfinder/_vendor/pep514tools/_registry.py index 2611f601..da72ecb5 100644 --- a/pipenv/vendor/pythonfinder/_vendor/pep514tools/_registry.py +++ b/pipenv/vendor/pythonfinder/_vendor/pep514tools/_registry.py @@ -172,7 +172,7 @@ class RegistryAccessor(object): items = info.items() else: raise TypeError('info must be a dictionary') - + self._set_all_values(self._root, self.subkey, items, errors) if len(errors) == 1: raise ValueError(errors[0]) diff --git a/pipenv/vendor/pythonfinder/_vendor/pep514tools/environment.py b/pipenv/vendor/pythonfinder/_vendor/pep514tools/environment.py index 2c09ccbc..e201d0b5 100644 --- a/pipenv/vendor/pythonfinder/_vendor/pep514tools/environment.py +++ b/pipenv/vendor/pythonfinder/_vendor/pep514tools/environment.py @@ -15,7 +15,8 @@ import sys # These tags are treated specially when the Company is 'PythonCore' _PYTHONCORE_COMPATIBILITY_TAGS = { '2.0', '2.1', '2.2', '2.3', '2.4', '2.5', '2.6', '2.7', - '3.0', '3.1', '3.2', '3.3', '3.4' + '3.0', '3.1', '3.2', '3.3', '3.4', '3.5', '3.6', '3.7', + '3.8' } _IS_64BIT_OS = None diff --git a/pipenv/vendor/pythonfinder/_vendor/vendor.txt b/pipenv/vendor/pythonfinder/_vendor/vendor.txt index 88752498..e635a2a8 100644 --- a/pipenv/vendor/pythonfinder/_vendor/vendor.txt +++ b/pipenv/vendor/pythonfinder/_vendor/vendor.txt @@ -1 +1 @@ --e git+https://github.com/zooba/pep514tools.git@320e48745660b696e2dcaee888fc2e516b435e48#egg=pep514tools +git+https://github.com/zooba/pep514tools.git@master#egg=pep514tools diff --git a/pipenv/vendor/pythonfinder/cli.py b/pipenv/vendor/pythonfinder/cli.py index 221cb2fd..eb2e603a 100644 --- a/pipenv/vendor/pythonfinder/cli.py +++ b/pipenv/vendor/pythonfinder/cli.py @@ -1,9 +1,11 @@ -#!/usr/bin/env python # -*- coding=utf-8 -*- -from __future__ import print_function, absolute_import +from __future__ import absolute_import, print_function, unicode_literals + +import sys + import click import crayons -import sys + from . import __version__ from .pythonfinder import Finder @@ -11,16 +13,22 @@ from .pythonfinder import Finder @click.command() @click.option("--find", default=False, nargs=1, help="Find a specific python version.") @click.option("--which", default=False, nargs=1, help="Run the which command.") -@click.option( - "--findall", is_flag=True, default=False, help="Find all python versions." -) +@click.option("--findall", is_flag=True, default=False, help="Find all python versions.") @click.option( "--version", is_flag=True, default=False, help="Display PythonFinder version." ) -@click.option("--ignore-unsupported/--no-unsupported", is_flag=True, default=True, envvar="PYTHONFINDER_IGNORE_UNSUPPORTED", help="Ignore unsupported python versions.") -@click.version_option(prog_name='pyfinder', version=__version__) +@click.option( + "--ignore-unsupported/--no-unsupported", + is_flag=True, + default=True, + envvar="PYTHONFINDER_IGNORE_UNSUPPORTED", + help="Ignore unsupported python versions.", +) +@click.version_option(prog_name="pyfinder", version=__version__) @click.pass_context -def cli(ctx, find=False, which=False, findall=False, version=False, ignore_unsupported=True): +def cli( + ctx, find=False, which=False, findall=False, version=False, ignore_unsupported=True +): if version: click.echo( "{0} version {1}".format( diff --git a/pipenv/vendor/pythonfinder/environment.py b/pipenv/vendor/pythonfinder/environment.py index e8878403..6b22fb08 100644 --- a/pipenv/vendor/pythonfinder/environment.py +++ b/pipenv/vendor/pythonfinder/environment.py @@ -1,5 +1,6 @@ # -*- coding=utf-8 -*- -from __future__ import print_function, absolute_import +from __future__ import absolute_import, print_function + import os import platform import sys @@ -34,3 +35,20 @@ else: IGNORE_UNSUPPORTED = bool(os.environ.get("PYTHONFINDER_IGNORE_UNSUPPORTED", False)) MYPY_RUNNING = os.environ.get("MYPY_RUNNING", is_type_checking()) +SUBPROCESS_TIMEOUT = os.environ.get("PYTHONFINDER_SUBPROCESS_TIMEOUT", 5) +"""The default subprocess timeout for determining python versions + +Set to **5** by default. +""" + + +def get_shim_paths(): + shim_paths = [] + if ASDF_INSTALLED: + shim_paths.append(os.path.join(ASDF_DATA_DIR, "shims")) + if PYENV_INSTALLED: + shim_paths.append(os.path.join(PYENV_ROOT, "shims")) + return [os.path.normpath(os.path.normcase(p)) for p in shim_paths] + + +SHIM_PATHS = get_shim_paths() diff --git a/pipenv/vendor/pythonfinder/exceptions.py b/pipenv/vendor/pythonfinder/exceptions.py index df381daf..adfac16b 100644 --- a/pipenv/vendor/pythonfinder/exceptions.py +++ b/pipenv/vendor/pythonfinder/exceptions.py @@ -1,5 +1,5 @@ # -*- coding=utf-8 -*- -from __future__ import print_function, absolute_import +from __future__ import absolute_import, print_function class InvalidPythonVersion(Exception): diff --git a/pipenv/vendor/pythonfinder/models/mixins.py b/pipenv/vendor/pythonfinder/models/mixins.py index 7d406548..a3637e12 100644 --- a/pipenv/vendor/pythonfinder/models/mixins.py +++ b/pipenv/vendor/pythonfinder/models/mixins.py @@ -2,16 +2,83 @@ from __future__ import absolute_import, unicode_literals import abc -import attr import operator +from collections import defaultdict + +import attr import six +from cached_property import cached_property +from vistir.compat import fs_str -from ..utils import ensure_path, KNOWN_EXTS, unnest +from ..environment import MYPY_RUNNING +from ..exceptions import InvalidPythonVersion +from ..utils import ( + KNOWN_EXTS, + Sequence, + expand_paths, + looks_like_python, + path_is_known_executable, +) + +if MYPY_RUNNING: + from .path import PathEntry + from .python import PythonVersion + from typing import ( + Optional, + Union, + Any, + Dict, + Iterator, + List, + DefaultDict, + Generator, + Tuple, + TypeVar, + Type, + ) + from vistir.compat import Path + + BaseFinderType = TypeVar("BaseFinderType") -@attr.s +@attr.s(slots=True) class BasePath(object): + path = attr.ib(default=None) # type: Path + _children = attr.ib( + default=attr.Factory(dict), cmp=False + ) # type: Dict[str, PathEntry] + only_python = attr.ib(default=False) # type: bool + name = attr.ib(type=str) + _py_version = attr.ib(default=None, cmp=False) # type: Optional[PythonVersion] + _pythons = attr.ib( + default=attr.Factory(defaultdict), cmp=False + ) # type: DefaultDict[str, PathEntry] + _is_dir = attr.ib(default=None, cmp=False) # type: Optional[bool] + _is_executable = attr.ib(default=None, cmp=False) # type: Optional[bool] + _is_python = attr.ib(default=None, cmp=False) # type: Optional[bool] + + def __str__(self): + # type: () -> str + return fs_str("{0}".format(self.path.as_posix())) + + def __lt__(self, other): + # type: ("BasePath") -> bool + return self.path.as_posix() < other.path.as_posix() + + def __lte__(self, other): + # type: ("BasePath") -> bool + return self.path.as_posix() <= other.path.as_posix() + + def __gt__(self, other): + # type: ("BasePath") -> bool + return self.path.as_posix() > other.path.as_posix() + + def __gte__(self, other): + # type: ("BasePath") -> bool + return self.path.as_posix() >= other.path.as_posix() + def which(self, name): + # type: (str) -> Optional[PathEntry] """Search in this path for an executable. :param executable: The name of an executable to search for. @@ -24,26 +91,226 @@ class BasePath(object): for ext in KNOWN_EXTS ] children = self.children - found = next( - ( - children[(self.path / child).as_posix()] - for child in valid_names - if (self.path / child).as_posix() in children - ), - None, - ) + found = None + if self.path is not None: + found = next( + ( + children[(self.path / child).as_posix()] + for child in valid_names + if (self.path / child).as_posix() in children + ), + None, + ) return found + def __del__(self): + for key in ["_is_dir", "_is_python", "_is_executable", "_py_version"]: + if getattr(self, key, None): + try: + delattr(self, key) + except Exception: + print("failed deleting key: {0}".format(key)) + self._children = {} + for key in list(self._pythons.keys()): + del self._pythons[key] + self._pythons = None + self._py_version = None + self.path = None + + @property + def children(self): + # type: () -> Dict[str, PathEntry] + if not self.is_dir: + return {} + return self._children + + @property + def as_python(self): + # type: () -> PythonVersion + py_version = None + if self.py_version: + return self.py_version + if not self.is_dir and self.is_python: + try: + from .python import PythonVersion + + py_version = PythonVersion.from_path( # type: ignore + path=self, name=self.name + ) + except (ValueError, InvalidPythonVersion): + pass + if py_version is None: + pass + self.py_version = py_version + return py_version # type: ignore + + @name.default + def get_name(self): + # type: () -> Optional[str] + if self.path: + return self.path.name + return None + + @property + def is_dir(self): + # type: () -> bool + if self._is_dir is None: + if not self.path: + ret_val = False + try: + ret_val = self.path.is_dir() + except OSError: + ret_val = False + self._is_dir = ret_val + return self._is_dir + + @is_dir.setter + def is_dir(self, val): + # type: (bool) -> None + self._is_dir = val + + @is_dir.deleter + def is_dir(self): + # type: () -> None + self._is_dir = None + + # @cached_property + @property + def is_executable(self): + # type: () -> bool + if self._is_executable is None: + if not self.path: + self._is_executable = False + else: + self._is_executable = path_is_known_executable(self.path) + return self._is_executable + + @is_executable.setter + def is_executable(self, val): + # type: (bool) -> None + self._is_executable = val + + @is_executable.deleter + def is_executable(self): + # type: () -> None + self._is_executable = None + + # @cached_property + @property + def is_python(self): + # type: () -> bool + if self._is_python is None: + if not self.path: + self._is_python = False + else: + self._is_python = self.is_executable and ( + looks_like_python(self.path.name) + ) + return self._is_python + + @is_python.setter + def is_python(self, val): + # type: (bool) -> None + self._is_python = val + + @is_python.deleter + def is_python(self): + # type: () -> None + self._is_python = None + + def get_py_version(self): + # type: () -> Optional[PythonVersion] + from ..environment import IGNORE_UNSUPPORTED + + if self.is_dir: + return None + if self.is_python: + py_version = None + from .python import PythonVersion + + try: + py_version = PythonVersion.from_path( # type: ignore + path=self, name=self.name + ) + except (InvalidPythonVersion, ValueError): + py_version = None + except Exception: + if not IGNORE_UNSUPPORTED: + raise + return py_version + return None + + # @cached_property + @property + def py_version(self): + # type: () -> Optional[PythonVersion] + if not self._py_version: + py_version = self.get_py_version() + self._py_version = py_version + else: + py_version = self._py_version + return py_version + + @py_version.setter + def py_version(self, val): + # type: (Optional[PythonVersion]) -> None + self._py_version = val + + @py_version.deleter + def py_version(self): + # type: () -> None + self._py_version = None + + def _iter_pythons(self): + # type: () -> Iterator + if self.is_dir: + for entry in self.children.values(): + if entry is None: + continue + elif entry.is_dir: + for python in entry._iter_pythons(): + yield python + elif entry.is_python and entry.as_python is not None: + yield entry + elif self.is_python and self.as_python is not None: + yield self # type: ignore + + @property + def pythons(self): + # type: () -> DefaultDict[Union[str, Path], PathEntry] + if not self._pythons: + from .path import PathEntry + + self._pythons = defaultdict(PathEntry) + for python in self._iter_pythons(): + python_path = python.path.as_posix() # type: ignore + self._pythons[python_path] = python + return self._pythons + + def __iter__(self): + # type: () -> Iterator + for entry in self.children.values(): + yield entry + + def __next__(self): + # type: () -> Generator + return next(iter(self)) + + def next(self): + # type: () -> Generator + return self.__next__() + def find_all_python_versions( self, - major=None, - minor=None, - patch=None, - pre=None, - dev=None, - arch=None, - name=None, + major=None, # type: Optional[Union[str, int]] + minor=None, # type: Optional[int] + patch=None, # type: Optional[int] + pre=None, # type: Optional[bool] + dev=None, # type: Optional[bool] + arch=None, # type: Optional[str] + name=None, # type: Optional[str] ): + # type: (...) -> List[PathEntry] """Search for a specific python version on the path. Return all copies :param major: Major python version to search for. @@ -58,35 +325,29 @@ class BasePath(object): :rtype: List[:class:`~pythonfinder.models.PathEntry`] """ - call_method = ( - "find_all_python_versions" if self.is_dir else "find_python_version" - ) + call_method = "find_all_python_versions" if self.is_dir else "find_python_version" sub_finder = operator.methodcaller( - call_method, - major=major, - minor=minor, - patch=patch, - pre=pre, - dev=dev, - arch=arch, - name=name, + call_method, major, minor, patch, pre, dev, arch, name ) if not self.is_dir: return sub_finder(self) - path_filter = filter(None, (sub_finder(p) for p in self.children.values())) + unnested = [sub_finder(path) for path in expand_paths(self)] version_sort = operator.attrgetter("as_python.version_sort") - return [c for c in sorted(path_filter, key=version_sort, reverse=True)] + unnested = [p for p in unnested if p is not None and p.as_python is not None] + paths = sorted(unnested, key=version_sort, reverse=True) + return list(paths) def find_python_version( self, - major=None, - minor=None, - patch=None, - pre=None, - dev=None, - arch=None, - name=None, + major=None, # type: Optional[Union[str, int]] + minor=None, # type: Optional[int] + patch=None, # type: Optional[int] + pre=None, # type: Optional[bool] + dev=None, # type: Optional[bool] + arch=None, # type: Optional[str] + name=None, # type: Optional[str] ): + # type: (...) -> Optional[PathEntry] """Search or self for the specified Python version and return the first match. :param major: Major version number. @@ -101,55 +362,60 @@ class BasePath(object): """ version_matcher = operator.methodcaller( - "matches", - major=major, - minor=minor, - patch=patch, - pre=pre, - dev=dev, - arch=arch, - name=name, + "matches", major, minor, patch, pre, dev, arch, python_name=name ) - is_py = operator.attrgetter("is_python") - py_version = operator.attrgetter("as_python") if not self.is_dir: if self.is_python and self.as_python and version_matcher(self.py_version): - return attr.evolve(self) - return - finder = ( - (child, child.as_python) - for child in unnest(self.pythons.values()) - if child.as_python - ) - py_filter = filter( - None, filter(lambda child: version_matcher(child[1]), finder) - ) - version_sort = operator.attrgetter("version_sort") - return next( - ( - c[0] - for c in sorted( - py_filter, key=lambda child: child[1].version_sort, reverse=True - ) - ), - None, - ) + return self # type: ignore + + matching_pythons = [ + [entry, entry.as_python.version_sort] + for entry in self._iter_pythons() + if ( + entry is not None + and entry.as_python is not None + and version_matcher(entry.py_version) + ) + ] + results = sorted(matching_pythons, key=operator.itemgetter(1, 0), reverse=True) + return next(iter(r[0] for r in results if r is not None), None) @six.add_metaclass(abc.ABCMeta) class BaseFinder(object): + def __init__(self): + #: Maps executable paths to PathEntries + from .path import PathEntry + + self._pythons = defaultdict(PathEntry) # type: DefaultDict[str, PathEntry] + self._versions = defaultdict(PathEntry) # type: Dict[Tuple, PathEntry] + def get_versions(self): + # type: () -> DefaultDict[Tuple, PathEntry] """Return the available versions from the finder""" raise NotImplementedError @classmethod - def create(cls): + def create(cls, *args, **kwargs): + # type: (Any, Any) -> BaseFinderType raise NotImplementedError @property def version_paths(self): - return self.versions.values() + # type: () -> Any + return self._versions.values() @property def expanded_paths(self): + # type: () -> Any return (p.paths.values() for p in self.version_paths) + + @property + def pythons(self): + # type: () -> DefaultDict[str, PathEntry] + return self._pythons + + @pythons.setter + def pythons(self, value): + # type: (DefaultDict[str, PathEntry]) -> None + self._pythons = value diff --git a/pipenv/vendor/pythonfinder/models/path.py b/pipenv/vendor/pythonfinder/models/path.py index 42608cde..80d5ac59 100644 --- a/pipenv/vendor/pythonfinder/models/path.py +++ b/pipenv/vendor/pythonfinder/models/path.py @@ -5,62 +5,159 @@ import copy import operator import os import sys - from collections import defaultdict from itertools import chain import attr import six - from cached_property import cached_property - from vistir.compat import Path, fs_str +from vistir.misc import dedup -from .mixins import BasePath -from ..environment import PYENV_INSTALLED, PYENV_ROOT, ASDF_INSTALLED, ASDF_DATA_DIR +from ..environment import ( + ASDF_DATA_DIR, + ASDF_INSTALLED, + MYPY_RUNNING, + PYENV_INSTALLED, + PYENV_ROOT, + SHIM_PATHS, + get_shim_paths, +) from ..exceptions import InvalidPythonVersion from ..utils import ( + Iterable, + Sequence, ensure_path, + expand_paths, filter_pythons, + is_in_path, looks_like_python, - optional_instance_of, - path_is_known_executable, - unnest, normalize_path, + optional_instance_of, + parse_asdf_version_order, parse_pyenv_version_order, - parse_asdf_version_order + path_is_known_executable, + split_version_and_name, + unnest, ) +from .mixins import BaseFinder, BasePath from .python import PythonVersion +if MYPY_RUNNING: + from typing import ( + Optional, + Dict, + DefaultDict, + Iterator, + List, + Union, + Tuple, + Generator, + Callable, + Type, + Any, + TypeVar, + ) + from .python import PythonFinder + from .windows import WindowsFinder -ASDF_SHIM_PATH = normalize_path(os.path.join(ASDF_DATA_DIR, "shims")) -PYENV_SHIM_PATH = normalize_path(os.path.join(PYENV_ROOT, "shims")) -SHIM_PATHS = [ASDF_SHIM_PATH, PYENV_SHIM_PATH] + FinderType = TypeVar("FinderType", BaseFinder, PythonFinder, WindowsFinder) + ChildType = Union[PythonFinder, "PathEntry"] + PathType = Union[PythonFinder, "PathEntry"] @attr.s class SystemPath(object): global_search = attr.ib(default=True) - paths = attr.ib(default=attr.Factory(defaultdict)) - _executables = attr.ib(default=attr.Factory(list)) - _python_executables = attr.ib(default=attr.Factory(list)) - path_order = attr.ib(default=attr.Factory(list)) - python_version_dict = attr.ib(default=attr.Factory(defaultdict)) - only_python = attr.ib(default=False) - pyenv_finder = attr.ib(default=None, validator=optional_instance_of("PyenvPath")) - asdf_finder = attr.ib(default=None) - system = attr.ib(default=False) - _version_dict = attr.ib(default=attr.Factory(defaultdict)) - ignore_unsupported = attr.ib(default=False) + paths = attr.ib( + default=attr.Factory(defaultdict) + ) # type: DefaultDict[str, Union[PythonFinder, PathEntry]] + _executables = attr.ib(default=attr.Factory(list)) # type: List[PathEntry] + _python_executables = attr.ib( + default=attr.Factory(dict) + ) # type: Dict[str, PathEntry] + path_order = attr.ib(default=attr.Factory(list)) # type: List[str] + python_version_dict = attr.ib() # type: DefaultDict[Tuple, List[PythonVersion]] + only_python = attr.ib(default=False, type=bool) + pyenv_finder = attr.ib(default=None) # type: Optional[PythonFinder] + asdf_finder = attr.ib(default=None) # type: Optional[PythonFinder] + windows_finder = attr.ib(default=None) # type: Optional[WindowsFinder] + system = attr.ib(default=False, type=bool) + _version_dict = attr.ib( + default=attr.Factory(defaultdict) + ) # type: DefaultDict[Tuple, List[PathEntry]] + ignore_unsupported = attr.ib(default=False, type=bool) - __finders = attr.ib(default=attr.Factory(dict)) + __finders = attr.ib( + default=attr.Factory(dict) + ) # type: Dict[str, Union[WindowsFinder, PythonFinder]] def _register_finder(self, finder_name, finder): + # type: (str, Union[WindowsFinder, PythonFinder]) -> "SystemPath" if finder_name not in self.__finders: self.__finders[finder_name] = finder + return self + + def clear_caches(self): + for key in ["executables", "python_executables", "version_dict", "path_entries"]: + if key in self.__dict__: + del self.__dict__[key] + for finder in list(self.__finders.keys()): + del self.__finders[finder] + self.__finders = {} + return attr.evolve( + self, + executables=[], + python_executables={}, + python_version_dict=defaultdict(list), + version_dict=defaultdict(list), + pyenv_finder=None, + windows_finder=None, + asdf_finder=None, + path_order=[], + paths=defaultdict(PathEntry), + ) + + def __del__(self): + for key in ["executables", "python_executables", "version_dict", "path_entries"]: + try: + del self.__dict__[key] + except KeyError: + pass + for finder in list(self.__finders.keys()): + del self.__finders[finder] + self.__finders = {} + self._python_executables = {} + self._executables = [] + self.python_version_dict = defaultdict(list) + self.version_dict = defaultdict(list) + self.path_order = [] + self.pyenv_finder = None + self.asdf_finder = None + self.paths = defaultdict(PathEntry) + self.__finders = {} + + @property + def finders(self): + # type: () -> List[str] + return [k for k in self.__finders.keys()] + + @staticmethod + def check_for_pyenv(): + return PYENV_INSTALLED or os.path.exists(normalize_path(PYENV_ROOT)) + + @staticmethod + def check_for_asdf(): + return ASDF_INSTALLED or os.path.exists(normalize_path(ASDF_DATA_DIR)) + + @python_version_dict.default + def create_python_version_dict(self): + # type: () -> DefaultDict[Tuple, List[PythonVersion]] + return defaultdict(list) @cached_property def executables(self): + # type: () -> List[PathEntry] self.executables = [ p for p in chain(*(child.children.values() for child in self.paths.values())) @@ -70,6 +167,7 @@ class SystemPath(object): @cached_property def python_executables(self): + # type: () -> Dict[str, PathEntry] python_executables = {} for child in self.paths.values(): if child.pythons: @@ -82,142 +180,254 @@ class SystemPath(object): @cached_property def version_dict(self): - self._version_dict = defaultdict(list) + # type: () -> DefaultDict[Tuple, List[PathEntry]] + self._version_dict = defaultdict( + list + ) # type: DefaultDict[Tuple, List[PathEntry]] for finder_name, finder in self.__finders.items(): for version, entry in finder.versions.items(): if finder_name == "windows": if entry not in self._version_dict[version]: self._version_dict[version].append(entry) continue - if type(entry).__name__ == "VersionPath": - for path in entry.paths.values(): - if path not in self._version_dict[version] and path.is_python: - self._version_dict[version].append(path) - continue - continue - elif entry not in self._version_dict[version] and entry.is_python: + if entry not in self._version_dict[version] and entry.is_python: self._version_dict[version].append(entry) for p, entry in self.python_executables.items(): version = entry.as_python if not version: continue - version = version.version_tuple + if not isinstance(version, tuple): + version = version.version_tuple if version and entry not in self._version_dict[version]: self._version_dict[version].append(entry) return self._version_dict - def __attrs_post_init__(self): - #: slice in pyenv + def _run_setup(self): + # type: () -> "SystemPath" if not self.__class__ == SystemPath: - return - if os.name == "nt": - self._setup_windows() - if PYENV_INSTALLED: - self._setup_pyenv() - if ASDF_INSTALLED: - self._setup_asdf() + return self + new_instance = self + path_order = new_instance.path_order[:] + path_entries = self.paths.copy() + if self.global_search and "PATH" in os.environ: + path_order = path_order + os.environ["PATH"].split(os.pathsep) + path_order = list(dedup(path_order)) + path_instances = [ + ensure_path(p.strip('"')) + for p in path_order + if not any( + is_in_path(normalize_path(str(p)), normalize_path(shim)) + for shim in SHIM_PATHS + ) + ] + path_entries.update( + { + p.as_posix(): PathEntry.create( + path=p.absolute(), is_root=True, only_python=self.only_python + ) + for p in path_instances + } + ) + new_instance = attr.evolve( + new_instance, + path_order=[p.as_posix() for p in path_instances], + paths=path_entries, + ) + if os.name == "nt" and "windows" not in self.finders: + new_instance = new_instance._setup_windows() + #: slice in pyenv + if self.check_for_pyenv() and "pyenv" not in self.finders: + new_instance = new_instance._setup_pyenv() + #: slice in asdf + if self.check_for_asdf() and "asdf" not in self.finders: + new_instance = new_instance._setup_asdf() venv = os.environ.get("VIRTUAL_ENV") if os.name == "nt": bin_dir = "Scripts" else: bin_dir = "bin" - if venv and (self.system or self.global_search): + if venv and (new_instance.system or new_instance.global_search): p = ensure_path(venv) - self.path_order = [(p / bin_dir).as_posix()] + self.path_order - self.paths[p] = PathEntry.create(path=p, is_root=True, only_python=False) - if self.system: + path_order = [(p / bin_dir).as_posix()] + new_instance.path_order + new_instance = attr.evolve(new_instance, path_order=path_order) + paths = new_instance.paths.copy() + paths[p] = new_instance.get_path(p.joinpath(bin_dir)) + new_instance = attr.evolve(new_instance, paths=paths) + if new_instance.system: syspath = Path(sys.executable) syspath_bin = syspath.parent if syspath_bin.name != bin_dir and syspath_bin.joinpath(bin_dir).exists(): syspath_bin = syspath_bin / bin_dir - self.path_order = [syspath_bin.as_posix()] + self.path_order - self.paths[syspath_bin] = PathEntry.create( + path_order = [syspath_bin.as_posix()] + new_instance.path_order + paths = new_instance.paths.copy() + paths[syspath_bin] = PathEntry.create( path=syspath_bin, is_root=True, only_python=False ) + new_instance = attr.evolve(new_instance, path_order=path_order, paths=paths) + return new_instance def _get_last_instance(self, path): + # type: (str) -> int reversed_paths = reversed(self.path_order) paths = [normalize_path(p) for p in reversed_paths] normalized_target = normalize_path(path) - last_instance = next( - iter(p for p in paths if normalized_target in p), None - ) - try: - path_index = self.path_order.index(last_instance) - except ValueError: - return + last_instance = next(iter(p for p in paths if normalized_target in p), None) + if last_instance is None: + raise ValueError("No instance found on path for target: {0!s}".format(path)) + path_index = self.path_order.index(last_instance) return path_index def _slice_in_paths(self, start_idx, paths): - before_path = self.path_order[: start_idx + 1] - after_path = self.path_order[start_idx + 2 :] - self.path_order = ( - before_path + [p.as_posix() for p in paths] + after_path - ) + # type: (int, List[Path]) -> "SystemPath" + before_path = [] # type: List[str] + after_path = [] # type: List[str] + if start_idx == 0: + after_path = self.path_order[:] + elif start_idx == -1: + before_path = self.path_order[:] + else: + before_path = self.path_order[: start_idx + 1] + after_path = self.path_order[start_idx + 2 :] + path_order = before_path + [p.as_posix() for p in paths] + after_path + if path_order == self.path_order: + return self + return attr.evolve(self, path_order=path_order) def _remove_path(self, path): + # type: (str) -> "SystemPath" path_copy = [p for p in reversed(self.path_order[:])] new_order = [] target = normalize_path(path) - path_map = { - normalize_path(pth): pth - for pth in self.paths.keys() - } + path_map = {normalize_path(pth): pth for pth in self.paths.keys()} + new_paths = self.paths.copy() if target in path_map: - del self.paths[path_map.get(target)] + del new_paths[path_map[target]] for current_path in path_copy: normalized = normalize_path(current_path) if normalized != target: new_order.append(normalized) new_order = [p for p in reversed(new_order)] - self.path_order = new_order + return attr.evolve(self, path_order=new_order, paths=new_paths) def _setup_asdf(self): + # type: () -> "SystemPath" + if "asdf" in self.finders and self.asdf_finder is not None: + return self from .python import PythonFinder - self.asdf_finder = PythonFinder.create( - root=ASDF_DATA_DIR, ignore_unsupported=True, - sort_function=parse_asdf_version_order, version_glob_path="installs/python/*") - asdf_index = self._get_last_instance(ASDF_DATA_DIR) - if not asdf_index: + + os_path = os.environ["PATH"].split(os.pathsep) + asdf_finder = PythonFinder.create( + root=ASDF_DATA_DIR, + ignore_unsupported=True, + sort_function=parse_asdf_version_order, + version_glob_path="installs/python/*", + ) + asdf_index = None + try: + asdf_index = self._get_last_instance(ASDF_DATA_DIR) + except ValueError: + asdf_index = 0 if is_in_path(next(iter(os_path), ""), ASDF_DATA_DIR) else -1 + if asdf_index is None: # we are in a virtualenv without global pyenv on the path, so we should # not write pyenv to the path here - return - root_paths = [p for p in self.asdf_finder.roots] - self._slice_in_paths(asdf_index, root_paths) - self.paths.update(self.asdf_finder.roots) - self._remove_path(normalize_path(os.path.join(ASDF_DATA_DIR, "shims"))) - self._register_finder("asdf", self.asdf_finder) + return self + root_paths = [p for p in asdf_finder.roots] + new_instance = self._slice_in_paths(asdf_index, [asdf_finder.root]) + paths = self.paths.copy() + paths[asdf_finder.root] = asdf_finder + paths.update(asdf_finder.roots) + return ( + attr.evolve(new_instance, paths=paths, asdf_finder=asdf_finder) + ._remove_path(normalize_path(os.path.join(ASDF_DATA_DIR, "shims"))) + ._register_finder("asdf", asdf_finder) + ) + + def reload_finder(self, finder_name): + # type: (str) -> "SystemPath" + if finder_name is None: + raise TypeError("Must pass a string as the name of the target finder") + finder_attr = "{0}_finder".format(finder_name) + setup_attr = "_setup_{0}".format(finder_name) + try: + current_finder = getattr(self, finder_attr) # type: Any + except AttributeError: + raise ValueError("Must pass a valid finder to reload.") + try: + setup_fn = getattr(self, setup_attr) + except AttributeError: + raise ValueError("Finder has no valid setup function: %s" % finder_name) + if current_finder is None: + # TODO: This is called 'reload', should we load a new finder for the first + # time here? lets just skip that for now to avoid unallowed finders + pass + if (finder_name == "pyenv" and not PYENV_INSTALLED) or ( + finder_name == "asdf" and not ASDF_INSTALLED + ): + # Don't allow loading of finders that aren't explicitly 'installed' as it were + return self + setattr(self, finder_attr, None) + if finder_name in self.__finders: + del self.__finders[finder_name] + return setup_fn() def _setup_pyenv(self): + # type: () -> "SystemPath" + if "pyenv" in self.finders and self.pyenv_finder is not None: + return self from .python import PythonFinder - self.pyenv_finder = PythonFinder.create( - root=PYENV_ROOT, sort_function=parse_pyenv_version_order, - version_glob_path="versions/*", ignore_unsupported=self.ignore_unsupported + os_path = os.environ["PATH"].split(os.pathsep) + + pyenv_finder = PythonFinder.create( + root=PYENV_ROOT, + sort_function=parse_pyenv_version_order, + version_glob_path="versions/*", + ignore_unsupported=self.ignore_unsupported, ) - pyenv_index = self._get_last_instance(PYENV_ROOT) - if not pyenv_index: + pyenv_index = None + try: + pyenv_index = self._get_last_instance(PYENV_ROOT) + except ValueError: + pyenv_index = 0 if is_in_path(next(iter(os_path), ""), PYENV_ROOT) else -1 + if pyenv_index is None: # we are in a virtualenv without global pyenv on the path, so we should # not write pyenv to the path here - return - root_paths = [p for p in self.pyenv_finder.roots] - self._slice_in_paths(pyenv_index, root_paths) + return self - self.paths.update(self.pyenv_finder.roots) - self._remove_path(os.path.join(PYENV_ROOT, "shims")) - self._register_finder("pyenv", self.pyenv_finder) + root_paths = [p for p in pyenv_finder.roots] + new_instance = self._slice_in_paths(pyenv_index, [pyenv_finder.root]) + paths = new_instance.paths.copy() + paths[pyenv_finder.root] = pyenv_finder + paths.update(pyenv_finder.roots) + return ( + attr.evolve(new_instance, paths=paths, pyenv_finder=pyenv_finder) + ._remove_path(os.path.join(PYENV_ROOT, "shims")) + ._register_finder("pyenv", pyenv_finder) + ) def _setup_windows(self): + # type: () -> "SystemPath" + if "windows" in self.finders and self.windows_finder is not None: + return self from .windows import WindowsFinder - self.windows_finder = WindowsFinder.create() - root_paths = (p for p in self.windows_finder.paths if p.is_root) + windows_finder = WindowsFinder.create() + root_paths = (p for p in windows_finder.paths if p.is_root) path_addition = [p.path.as_posix() for p in root_paths] - self.path_order = self.path_order[:] + path_addition - self.paths.update({p.path: p for p in root_paths}) - self._register_finder("windows", self.windows_finder) + new_path_order = self.path_order[:] + path_addition + new_paths = self.paths.copy() + new_paths.update({p.path: p for p in root_paths}) + return attr.evolve( + self, + windows_finder=windows_finder, + path_order=new_path_order, + paths=new_paths, + )._register_finder("windows", windows_finder) def get_path(self, path): + # type: (Union[str, Path]) -> PathType + if path is None: + raise TypeError("A path must be provided in order to generate a path entry.") path = ensure_path(path) _path = self.paths.get(path) if not _path: @@ -227,69 +437,90 @@ class SystemPath(object): path=path.absolute(), is_root=True, only_python=self.only_python ) self.paths[path.as_posix()] = _path + if not _path: + raise ValueError("Path not found or generated: {0!r}".format(path)) return _path def _get_paths(self): - return (self.get_path(k) for k in self.path_order) + # type: () -> Generator[Union[PathType, WindowsFinder], None, None] + for path in self.path_order: + try: + entry = self.get_path(path) + except ValueError: + continue + else: + yield entry @cached_property def path_entries(self): - paths = self._get_paths() + # type: () -> List[Union[PathType, WindowsFinder]] + paths = list(self._get_paths()) return paths def find_all(self, executable): - """Search the path for an executable. Return all copies. + # type: (str) -> List[Union[PathEntry, FinderType]] + """ + Search the path for an executable. Return all copies. :param executable: Name of the executable :type executable: str :returns: List[PathEntry] """ - sub_which = operator.methodcaller("which", name=executable) + + sub_which = operator.methodcaller("which", executable) filtered = (sub_which(self.get_path(k)) for k in self.path_order) return list(filtered) def which(self, executable): - """Search for an executable on the path. + # type: (str) -> Union[PathEntry, None] + """ + Search for an executable on the path. :param executable: Name of the executable to be located. :type executable: str :returns: :class:`~pythonfinder.models.PathEntry` object. """ - sub_which = operator.methodcaller("which", name=executable) + + sub_which = operator.methodcaller("which", executable) filtered = (sub_which(self.get_path(k)) for k in self.path_order) return next(iter(f for f in filtered if f is not None), None) def _filter_paths(self, finder): - return ( - pth for pth in unnest(finder(p) for p in self.path_entries if p is not None) - if pth is not None - ) + # type: (Callable) -> Iterator + for path in self._get_paths(): + if path is None: + continue + python_versions = finder(path) + if python_versions is not None: + for python in python_versions: + if python is not None: + yield python def _get_all_pythons(self, finder): - paths = {p.path.as_posix(): p for p in self._filter_paths(finder)} - paths.update(self.python_executables) - return (p for p in paths.values() if p is not None) + # type: (Callable) -> Iterator + for python in self._filter_paths(finder): + if python is not None and python.is_python: + yield python def get_pythons(self, finder): + # type: (Callable) -> Iterator sort_key = operator.attrgetter("as_python.version_sort") - return ( - k for k in sorted( - (p for p in self._filter_paths(finder) if p.is_python), - key=sort_key, - reverse=True - ) if k is not None - ) + pythons = [entry for entry in self._get_all_pythons(finder)] + for python in sorted(pythons, key=sort_key, reverse=True): + if python is not None: + yield python def find_all_python_versions( self, - major=None, - minor=None, - patch=None, - pre=None, - dev=None, - arch=None, - name=None, + major=None, # type: Optional[Union[str, int]] + minor=None, # type: Optional[int] + patch=None, # type: Optional[int] + pre=None, # type: Optional[bool] + dev=None, # type: Optional[bool] + arch=None, # type: Optional[str] + name=None, # type: Optional[str] ): + # type (...) -> List[PathEntry] """Search for a specific python version on the path. Return all copies :param major: Major python version to search for. @@ -305,21 +536,12 @@ class SystemPath(object): """ sub_finder = operator.methodcaller( - "find_all_python_versions", - major=major, - minor=minor, - patch=patch, - pre=pre, - dev=dev, - arch=arch, - name=name, + "find_all_python_versions", major, minor, patch, pre, dev, arch, name ) alternate_sub_finder = None if major and not (minor or patch or pre or dev or arch or name): alternate_sub_finder = operator.methodcaller( - "find_all_python_versions", - major=None, - name=major + "find_all_python_versions", None, None, None, None, None, None, major ) if os.name == "nt" and self.windows_finder: windows_finder_version = sub_finder(self.windows_finder) @@ -332,14 +554,16 @@ class SystemPath(object): def find_python_version( self, - major=None, - minor=None, - patch=None, - pre=None, - dev=None, - arch=None, - name=None, + major=None, # type: Optional[Union[str, int]] + minor=None, # type: Optional[Union[str, int]] + patch=None, # type: Optional[Union[str, int]] + pre=None, # type: Optional[bool] + dev=None, # type: Optional[bool] + arch=None, # type: Optional[str] + name=None, # type: Optional[str] + sort_by_path=False, # type: bool ): + # type: (...) -> PathEntry """Search for a specific python version on the path. :param major: Major python version to search for. @@ -350,39 +574,19 @@ class SystemPath(object): :param bool dev: Search for devreleases (default None) - prioritize releases if None :param str arch: Architecture to include, e.g. '64bit', defaults to None :param str name: The name of a python version, e.g. ``anaconda3-5.3.0`` + :param bool sort_by_path: Whether to sort by path -- default sort is by version(default: False) :return: A :class:`~pythonfinder.models.PathEntry` instance matching the version requested. :rtype: :class:`~pythonfinder.models.PathEntry` """ - if isinstance(major, six.string_types) and not minor and not patch: - # Only proceed if this is in the format "x.y.z" or similar - if major.count(".") > 0 and major[0].isdigit(): - version = major.split(".", 2) - if len(version) > 3: - major, minor, patch, rest = version - elif len(version) == 3: - major, minor, patch = version - else: - major, minor = version - else: - name = "{0!s}".format(major) - major = None + major, minor, patch, name = split_version_and_name(major, minor, patch, name) sub_finder = operator.methodcaller( - "find_python_version", - major, - minor=minor, - patch=patch, - pre=pre, - dev=dev, - arch=arch, - name=name, + "find_python_version", major, minor, patch, pre, dev, arch, name ) alternate_sub_finder = None - if major and not (minor or patch or pre or dev or arch or name): + if name and not (minor or patch or pre or dev or arch or major): alternate_sub_finder = operator.methodcaller( - "find_all_python_versions", - major=None, - name=major + "find_all_python_versions", None, None, None, None, None, None, name ) if major and minor and patch: _tuple_pre = pre if pre is not None else False @@ -393,6 +597,18 @@ class SystemPath(object): windows_finder_version = sub_finder(self.windows_finder) if windows_finder_version: return windows_finder_version + if sort_by_path: + paths = [self.get_path(k) for k in self.path_order] + for path in paths: + found_version = sub_finder(path) + if found_version: + return found_version + if alternate_sub_finder: + for path in paths: + found_version = alternate_sub_finder(path) + if found_version: + return found_version + ver = next(iter(self.get_pythons(sub_finder)), None) if not ver and alternate_sub_finder is not None: ver = next(iter(self.get_pythons(alternate_sub_finder)), None) @@ -406,12 +622,13 @@ class SystemPath(object): @classmethod def create( cls, - path=None, - system=False, - only_python=False, - global_search=True, - ignore_unsupported=True, + path=None, # type: str + system=False, # type: bool + only_python=False, # type: bool + global_search=True, # type: bool + ignore_unsupported=True, # type: bool ): + # type: (...) -> SystemPath """Create a new :class:`pythonfinder.models.SystemPath` instance. :param path: Search path to prepend when searching, defaults to None @@ -423,14 +640,30 @@ class SystemPath(object): :rtype: :class:`pythonfinder.models.SystemPath` """ - path_entries = defaultdict(PathEntry) - paths = [] + path_entries = defaultdict( + PathEntry + ) # type: DefaultDict[str, Union[PythonFinder, PathEntry]] + paths = [] # type: List[str] if ignore_unsupported: os.environ["PYTHONFINDER_IGNORE_UNSUPPORTED"] = fs_str("1") if global_search: - paths = os.environ.get("PATH").split(os.pathsep) + if "PATH" in os.environ: + paths = os.environ["PATH"].split(os.pathsep) + path_order = [] if path: + path_order = [path] + path_instance = ensure_path(path) + path_entries.update( + { + path_instance.as_posix(): PathEntry.create( + path=path_instance.absolute(), + is_root=True, + only_python=only_python, + ) + } + ) paths = [path] + paths + paths = [p for p in paths if not any(is_in_path(p, shim) for shim in SHIM_PATHS)] _path_objects = [ensure_path(p.strip('"')) for p in paths] paths = [p.as_posix() for p in _path_objects] path_entries.update( @@ -439,33 +672,43 @@ class SystemPath(object): path=p.absolute(), is_root=True, only_python=only_python ) for p in _path_objects - if not any(shim in normalize_path(str(p)) for shim in SHIM_PATHS) } ) - return cls( + instance = cls( paths=path_entries, - path_order=paths, + path_order=path_order, only_python=only_python, system=system, global_search=global_search, ignore_unsupported=ignore_unsupported, ) + instance = instance._run_setup() + return instance @attr.s(slots=True) class PathEntry(BasePath): - path = attr.ib(default=None, validator=optional_instance_of(Path)) - _children = attr.ib(default=attr.Factory(dict)) - is_root = attr.ib(default=True) - only_python = attr.ib(default=False) - name = attr.ib() - py_version = attr.ib() - _pythons = attr.ib(default=attr.Factory(defaultdict)) + is_root = attr.ib(default=True, type=bool, cmp=False) - def __str__(self): - return fs_str("{0}".format(self.path.as_posix())) + def __lt__(self, other): + return self.path.as_posix() < other.path.as_posix() + + def __lte__(self, other): + return self.path.as_posix() <= other.path.as_posix() + + def __gt__(self, other): + return self.path.as_posix() > other.path.as_posix() + + def __gte__(self, other): + return self.path.as_posix() >= other.path.as_posix() + + def __del__(self): + if getattr(self, "_children"): + del self._children + BasePath.__del__(self) def _filter_children(self): + # type: () -> Iterator[Path] if self.only_python: children = filter_pythons(self.path) else: @@ -473,85 +716,56 @@ class PathEntry(BasePath): return children def _gen_children(self): + # type: () -> Iterator + shim_paths = get_shim_paths() pass_name = self.name != self.path.name pass_args = {"is_root": False, "only_python": self.only_python} if pass_name: - pass_args["name"] = self.name + if self.name is not None and isinstance(self.name, six.string_types): + pass_args["name"] = self.name # type: ignore + elif self.path is not None and isinstance(self.path.name, six.string_types): + pass_args["name"] = self.path.name # type: ignore if not self.is_dir: - yield (self.path.as_posix(), copy.deepcopy(self)) + yield (self.path.as_posix(), self) elif self.is_root: for child in self._filter_children(): - if any(shim in normalize_path(str(child)) for shim in SHIM_PATHS): + if any(is_in_path(str(child), shim) for shim in shim_paths): continue if self.only_python: try: - entry = PathEntry.create(path=child, **pass_args) + entry = PathEntry.create(path=child, **pass_args) # type: ignore except (InvalidPythonVersion, ValueError): continue else: - entry = PathEntry.create(path=child, **pass_args) + entry = PathEntry.create(path=child, **pass_args) # type: ignore yield (child.as_posix(), entry) return - @cached_property + # @cached_property + @property def children(self): - if not self._children: - children = {} + # type: () -> Dict[str, PathEntry] + children = getattr(self, "_children", {}) # type: Dict[str, PathEntry] + if not children: for child_key, child_val in self._gen_children(): children[child_key] = child_val - self._children = children + self.children = children return self._children - @name.default - def get_name(self): - return self.path.name + @children.setter + def children(self, val): + # type: (Dict[str, PathEntry]) -> None + self._children = val - @py_version.default - def get_py_version(self): - from ..environment import IGNORE_UNSUPPORTED - if self.is_dir: - return None - if self.is_python: - try: - py_version = PythonVersion.from_path(path=self, name=self.name) - except (InvalidPythonVersion, ValueError): - py_version = None - except Exception: - if not IGNORE_UNSUPPORTED: - raise - return py_version - return - - @property - def pythons(self): - if not self._pythons: - if self.is_dir: - for path, entry in self.children.items(): - _path = ensure_path(entry.path) - if entry.is_python: - self._pythons[_path.as_posix()] = entry - else: - if self.is_python: - _path = ensure_path(self.path) - self._pythons[_path.as_posix()] = self - return self._pythons - - @cached_property - def as_python(self): - py_version = None - if self.py_version: - return self.py_version - if not self.is_dir and self.is_python: - try: - from .python import PythonVersion - py_version = PythonVersion.from_path(path=attr.evolve(self), name=self.name) - except (ValueError, InvalidPythonVersion): - py_version = None - return py_version + @children.deleter + def children(self): + # type: () -> None + del self._children @classmethod def create(cls, path, is_root=False, only_python=False, pythons=None, name=None): + # type: (Union[str, Path], bool, bool, Dict[str, PythonVersion], Optional[str]) -> PathEntry """Helper method for creating new :class:`pythonfinder.models.PathEntry` instances. :param str path: Path to the specified location. @@ -568,53 +782,35 @@ class PathEntry(BasePath): if not name: guessed_name = True name = target.name - creation_args = {"path": target, "is_root": is_root, "only_python": only_python, "name": name} + creation_args = { + "path": target, + "is_root": is_root, + "only_python": only_python, + "name": name, + } if pythons: creation_args["pythons"] = pythons _new = cls(**creation_args) if pythons and only_python: children = {} - child_creation_args = { - "is_root": False, - "only_python": only_python - } + child_creation_args = {"is_root": False, "only_python": only_python} if not guessed_name: - child_creation_args["name"] = name + child_creation_args["name"] = _new.name # type: ignore for pth, python in pythons.items(): if any(shim in normalize_path(str(pth)) for shim in SHIM_PATHS): continue pth = ensure_path(pth) - children[pth.as_posix()] = PathEntry( - py_version=python, - path=pth, - **child_creation_args + children[pth.as_posix()] = PathEntry( # type: ignore + py_version=python, path=pth, **child_creation_args ) _new._children = children return _new - @cached_property - def is_dir(self): - try: - ret_val = self.path.is_dir() - except OSError: - ret_val = False - return ret_val - - @cached_property - def is_executable(self): - return path_is_known_executable(self.path) - - @cached_property - def is_python(self): - return self.is_executable and ( - looks_like_python(self.path.name) - ) - @attr.s class VersionPath(SystemPath): - base = attr.ib(default=None, validator=optional_instance_of(Path)) - name = attr.ib(default=None) + base = attr.ib(default=None, validator=optional_instance_of(Path)) # type: Path + name = attr.ib(default=None) # type: str @classmethod def create(cls, path, only_python=True, pythons=None, name=None): @@ -622,6 +818,7 @@ class VersionPath(SystemPath): Generates the version listings for it""" from .path import PathEntry + path = ensure_path(path) path_entries = defaultdict(PathEntry) bin_ = "{base}/bin" diff --git a/pipenv/vendor/pythonfinder/models/python.py b/pipenv/vendor/pythonfinder/models/python.py index 1e5caa16..427eb694 100644 --- a/pipenv/vendor/pythonfinder/models/python.py +++ b/pipenv/vendor/pythonfinder/models/python.py @@ -2,168 +2,260 @@ from __future__ import absolute_import, print_function import copy -import platform -import operator import logging - +import operator +import platform +import sys from collections import defaultdict import attr +import six +from packaging.version import Version +from vistir.compat import Path, lru_cache -from packaging.version import Version, LegacyVersion -from packaging.version import parse as parse_version -from vistir.compat import Path - -from ..environment import SYSTEM_ARCH, PYENV_ROOT, ASDF_DATA_DIR -from .mixins import BaseFinder, BasePath +from ..environment import ASDF_DATA_DIR, MYPY_RUNNING, PYENV_ROOT, SYSTEM_ARCH +from ..exceptions import InvalidPythonVersion from ..utils import ( + RE_MATCHER, _filter_none, ensure_path, get_python_version, - optional_instance_of, - unnest, + guess_company, is_in_path, - parse_pyenv_version_order, + looks_like_python, + optional_instance_of, parse_asdf_version_order, + parse_pyenv_version_order, parse_python_version, + path_is_pythoncore, + unnest, ) +from .mixins import BaseFinder, BasePath + +if MYPY_RUNNING: + from typing import ( + DefaultDict, + Optional, + Callable, + Generator, + Any, + Union, + Tuple, + List, + Dict, + Type, + TypeVar, + Iterator, + ) + from .path import PathEntry + from .._vendor.pep514tools.environment import Environment + logger = logging.getLogger(__name__) @attr.s(slots=True) class PythonFinder(BaseFinder, BasePath): - root = attr.ib(default=None, validator=optional_instance_of(Path)) - #: ignore_unsupported should come before versions, because its value is used - #: in versions's default initializer. - ignore_unsupported = attr.ib(default=True) - #: The function to use to sort version order when returning an ordered verion set - sort_function = attr.ib(default=None) - paths = attr.ib(default=attr.Factory(list)) - roots = attr.ib(default=attr.Factory(defaultdict)) + root = attr.ib(default=None, validator=optional_instance_of(Path), type=Path) + # should come before versions, because its value is used in versions's default initializer. + #: Whether to ignore any paths which raise exceptions and are not actually python + ignore_unsupported = attr.ib(default=True, type=bool) #: Glob path for python versions off of the root directory - version_glob_path = attr.ib(default="versions/*") - versions = attr.ib() - pythons = attr.ib() + version_glob_path = attr.ib(default="versions/*", type=str) + #: The function to use to sort version order when returning an ordered verion set + sort_function = attr.ib(default=None) # type: Callable + #: The root locations used for discovery + roots = attr.ib(default=attr.Factory(defaultdict), type=defaultdict) + #: List of paths discovered during search + paths = attr.ib(type=list) + #: shim directory + shim_dir = attr.ib(default="shims", type=str) + #: Versions discovered in the specified paths + _versions = attr.ib(default=attr.Factory(defaultdict), type=defaultdict) + _pythons = attr.ib(default=attr.Factory(defaultdict), type=defaultdict) + + def __del__(self): + # type: () -> None + self._versions = defaultdict() + self._pythons = defaultdict() + self.roots = defaultdict() + self.paths = [] @property def expanded_paths(self): + # type: () -> Generator return ( - path for path in unnest(p for p in self.versions.values()) - if path is not None + path for path in unnest(p for p in self.versions.values()) if path is not None ) @property def is_pyenv(self): + # type: () -> bool return is_in_path(str(self.root), PYENV_ROOT) @property def is_asdf(self): + # type: () -> bool return is_in_path(str(self.root), ASDF_DATA_DIR) def get_version_order(self): + # type: () -> List[Path] version_paths = [ - p for p in self.root.glob(self.version_glob_path) + p + for p in self.root.glob(self.version_glob_path) if not (p.parent.name == "envs" or p.name == "envs") ] versions = {v.name: v for v in version_paths} + version_order = [] # type: List[Path] if self.is_pyenv: - version_order = [versions[v] for v in parse_pyenv_version_order() if v in versions] + version_order = [ + versions[v] for v in parse_pyenv_version_order() if v in versions + ] elif self.is_asdf: - version_order = [versions[v] for v in parse_asdf_version_order() if v in versions] + version_order = [ + versions[v] for v in parse_asdf_version_order() if v in versions + ] for version in version_order: - version_paths.remove(version) + if version in version_paths: + version_paths.remove(version) if version_order: version_order += version_paths else: version_order = version_paths return version_order + def get_bin_dir(self, base): + # type: (Union[Path, str]) -> Path + if isinstance(base, six.string_types): + base = Path(base) + return base / "bin" + @classmethod - def version_from_bin_dir(cls, base_dir, name=None): - from .path import PathEntry + def version_from_bin_dir(cls, entry): + # type: (PathEntry) -> Optional[PathEntry] py_version = None - version_path = PathEntry.create( - path=base_dir.absolute().as_posix(), - only_python=True, - name=base_dir.parent.name, - ) - py_version = next(iter(version_path.find_all_python_versions()), None) + py_version = next(iter(entry.find_all_python_versions()), None) return py_version - @versions.default - def get_versions(self): + def _iter_version_bases(self): + # type: () -> Iterator[Tuple[Path, PathEntry]] from .path import PathEntry - versions = defaultdict() - bin_ = "{base}/bin" + for p in self.get_version_order(): - bin_dir = Path(bin_.format(base=p.as_posix())) - version_path = None - if bin_dir.exists(): - version_path = PathEntry.create( - path=bin_dir.absolute().as_posix(), - only_python=False, - name=p.name, - is_root=True, + bin_dir = self.get_bin_dir(p) + if bin_dir.exists() and bin_dir.is_dir(): + entry = PathEntry.create( + path=bin_dir.absolute(), only_python=False, name=p.name, is_root=True ) + self.roots[p] = entry + yield (p, entry) + + def _iter_versions(self): + # type: () -> Iterator[Tuple[Path, PathEntry, Tuple]] + for base_path, entry in self._iter_version_bases(): version = None + version_entry = None try: - version = PythonVersion.parse(p.name) - except ValueError: - entry = next(iter(version_path.find_all_python_versions()), None) - if not entry: - if self.ignore_unsupported: - continue - raise - else: - version = entry.py_version.as_dict() + version = PythonVersion.parse(entry.name) + except (ValueError, InvalidPythonVersion): + version_entry = next(iter(entry.find_all_python_versions()), None) + if version is None: + if not self.ignore_unsupported: + raise + continue + if version_entry is not None: + version = version_entry.py_version.as_dict() except Exception: if not self.ignore_unsupported: raise logger.warning( - "Unsupported Python version %r, ignoring...", p.name, exc_info=True + "Unsupported Python version %r, ignoring...", + base_path.name, + exc_info=True, ) continue - if not version: - continue - version_tuple = ( - version.get("major"), - version.get("minor"), - version.get("patch"), - version.get("is_prerelease"), - version.get("is_devrelease"), - version.get("is_debug"), - ) - self.roots[p] = version_path - versions[version_tuple] = version_path - self.paths.append(version_path) - return versions + if version is not None: + version_tuple = ( + version.get("major"), + version.get("minor"), + version.get("patch"), + version.get("is_prerelease"), + version.get("is_devrelease"), + version.get("is_debug"), + ) + yield (base_path, entry, version_tuple) + + @property + def versions(self): + # type: () -> DefaultDict[Tuple, PathEntry] + if not self._versions: + for base_path, entry, version_tuple in self._iter_versions(): + self._versions[version_tuple] = entry + return self._versions + + def _iter_pythons(self): + # type: () -> Iterator + for path, entry, version_tuple in self._iter_versions(): + if path.as_posix() in self._pythons: + yield self._pythons[path.as_posix()] + elif version_tuple not in self.versions: + for python in entry.find_all_python_versions(): + yield python + else: + yield self.versions[version_tuple] + + @paths.default + def get_paths(self): + # type: () -> List[PathEntry] + _paths = [base for _, base in self._iter_version_bases()] + return _paths + + @property + def pythons(self): + # type: () -> DefaultDict[str, PathEntry] + if not self._pythons: + from .path import PathEntry + + self._pythons = defaultdict(PathEntry) # type: DefaultDict[str, PathEntry] + for python in self._iter_pythons(): + python_path = python.path.as_posix() # type: ignore + self._pythons[python_path] = python + return self._pythons + + @pythons.setter + def pythons(self, value): + # type: (DefaultDict[str, PathEntry]) -> None + self._pythons = value - @pythons.default def get_pythons(self): - pythons = defaultdict() - for p in self.paths: - pythons.update(p.pythons) - return pythons + # type: () -> DefaultDict[str, PathEntry] + return self.pythons @classmethod - def create(cls, root, sort_function=None, version_glob_path=None, ignore_unsupported=True): + def create(cls, root, sort_function, version_glob_path=None, ignore_unsupported=True): + # type: (str, Callable, Optional[str], bool) -> PythonFinder root = ensure_path(root) if not version_glob_path: version_glob_path = "versions/*" - return cls(root=root, ignore_unsupported=ignore_unsupported, - sort_function=sort_function, version_glob_path=version_glob_path) + return cls( + root=root, + path=root, + ignore_unsupported=ignore_unsupported, # type: ignore + sort_function=sort_function, + version_glob_path=version_glob_path, + ) def find_all_python_versions( self, - major=None, - minor=None, - patch=None, - pre=None, - dev=None, - arch=None, - name=None, + major=None, # type: Optional[Union[str, int]] + minor=None, # type: Optional[int] + patch=None, # type: Optional[int] + pre=None, # type: Optional[bool] + dev=None, # type: Optional[bool] + arch=None, # type: Optional[str] + name=None, # type: Optional[str] ): + # type: (...) -> List[PathEntry] """Search for a specific python version on the path. Return all copies :param major: Major python version to search for. @@ -178,36 +270,37 @@ class PythonFinder(BaseFinder, BasePath): :rtype: List[:class:`~pythonfinder.models.PathEntry`] """ - version_matcher = operator.methodcaller( - "matches", - major=major, - minor=minor, - patch=patch, - pre=pre, - dev=dev, - arch=arch, - name=name, + call_method = "find_all_python_versions" if self.is_dir else "find_python_version" + sub_finder = operator.methodcaller( + call_method, major, minor, patch, pre, dev, arch, name ) - py = operator.attrgetter("as_python") - pythons = ( - py_ver for py_ver in (py(p) for p in self.pythons.values() if p is not None) - if py_ver is not None - ) - # pythons = filter(None, [p.as_python for p in self.pythons.values()]) - matching_versions = filter(lambda py: version_matcher(py), pythons) - version_sort = operator.attrgetter("version_sort") - return sorted(matching_versions, key=version_sort, reverse=True) + if not any([major, minor, patch, name]): + pythons = [ + next(iter(py for py in base.find_all_python_versions()), None) + for _, base in self._iter_version_bases() + ] + else: + pythons = [sub_finder(path) for path in self.paths] + pythons = [p for p in pythons if p and p.is_python and p.as_python is not None] + version_sort = operator.attrgetter("as_python.version_sort") + paths = [ + p + for p in sorted(list(pythons), key=version_sort, reverse=True) + if p is not None + ] + return paths def find_python_version( self, - major=None, - minor=None, - patch=None, - pre=None, - dev=None, - arch=None, - name=None, + major=None, # type: Optional[Union[str, int]] + minor=None, # type: Optional[int] + patch=None, # type: Optional[int] + pre=None, # type: Optional[bool] + dev=None, # type: Optional[bool] + arch=None, # type: Optional[str] + name=None, # type: Optional[str] ): + # type: (...) -> Optional[PathEntry] """Search or self for the specified Python version and return the first match. :param major: Major version number. @@ -221,46 +314,85 @@ class PythonFinder(BaseFinder, BasePath): :returns: A :class:`~pythonfinder.models.PathEntry` instance matching the version requested. """ - version_matcher = operator.methodcaller( - "matches", - major=major, - minor=minor, - patch=patch, - pre=pre, - dev=dev, - arch=arch, - name=name, + sub_finder = operator.methodcaller( + "find_python_version", major, minor, patch, pre, dev, arch, name ) - pythons = filter(None, [p.as_python for p in self.pythons.values()]) - matching_versions = filter(lambda py: version_matcher(py), pythons) - version_sort = operator.attrgetter("version_sort") - return next(iter(c for c in sorted(matching_versions, key=version_sort, reverse=True)), None) + version_sort = operator.attrgetter("as_python.version_sort") + unnested = [sub_finder(self.roots[path]) for path in self.roots] + unnested = [ + p + for p in unnested + if p is not None and p.is_python and p.as_python is not None + ] + paths = sorted(list(unnested), key=version_sort, reverse=True) + return next(iter(p for p in paths if p is not None), None) + + def which(self, name): + # type: (str) -> Optional[PathEntry] + """Search in this path for an executable. + + :param executable: The name of an executable to search for. + :type executable: str + :returns: :class:`~pythonfinder.models.PathEntry` instance. + """ + + matches = (p.which(name) for p in self.paths) + non_empty_match = next(iter(m for m in matches if m is not None), None) + return non_empty_match @attr.s(slots=True) class PythonVersion(object): - major = attr.ib(default=0) - minor = attr.ib(default=None) - patch = attr.ib(default=0) - is_prerelease = attr.ib(default=False) - is_postrelease = attr.ib(default=False) - is_devrelease = attr.ib(default=False) - is_debug = attr.ib(default=False) - version = attr.ib(default=None, validator=optional_instance_of(Version)) - architecture = attr.ib(default=None) - comes_from = attr.ib(default=None) - executable = attr.ib(default=None) - name = attr.ib(default=None) + major = attr.ib(default=0, type=int) + minor = attr.ib(default=None) # type: Optional[int] + patch = attr.ib(default=None) # type: Optional[int] + is_prerelease = attr.ib(default=False, type=bool) + is_postrelease = attr.ib(default=False, type=bool) + is_devrelease = attr.ib(default=False, type=bool) + is_debug = attr.ib(default=False, type=bool) + version = attr.ib(default=None) # type: Version + architecture = attr.ib(default=None) # type: Optional[str] + comes_from = attr.ib(default=None) # type: Optional[PathEntry] + executable = attr.ib(default=None) # type: Optional[str] + company = attr.ib(default=None) # type: Optional[str] + name = attr.ib(default=None, type=str) + + def __getattribute__(self, key): + result = super(PythonVersion, self).__getattribute__(key) + if key in ["minor", "patch"] and result is None: + executable = None # type: Optional[str] + if self.executable: + executable = self.executable + elif self.comes_from: + executable = self.comes_from.path.as_posix() + if executable is not None: + if not isinstance(executable, six.string_types): + executable = executable.as_posix() + instance_dict = self.parse_executable(executable) + for k in instance_dict.keys(): + try: + super(PythonVersion, self).__getattribute__(k) + except AttributeError: + continue + else: + setattr(self, k, instance_dict[k]) + result = instance_dict.get(key) + return result @property def version_sort(self): - """version_sort tuple for sorting against other instances of the same class. - - Returns a tuple of the python version but includes a point for non-dev, - and a point for non-prerelease versions. So released versions will have 2 points - for this value. E.g. `(3, 6, 6, 2)` is a release, `(3, 6, 6, 1)` is a prerelease, - `(3, 6, 6, 0)` is a dev release, and `(3, 6, 6, 3)` is a postrelease. + # type: () -> Tuple[int, int, Optional[int], int, int] """ + A tuple for sorting against other instances of the same class. + + Returns a tuple of the python version but includes points for core python, + non-dev, and non-prerelease versions. So released versions will have 2 points + for this value. E.g. ``(1, 3, 6, 6, 2)`` is a release, ``(1, 3, 6, 6, 1)`` is a + prerelease, ``(1, 3, 6, 6, 0)`` is a dev release, and ``(1, 3, 6, 6, 3)`` is a + postrelease. ``(0, 3, 7, 3, 2)`` represents a non-core python release, e.g. by + a repackager of python like Continuum. + """ + company_sort = 1 if (self.company and self.company == "PythonCore") else 0 release_sort = 2 if self.is_postrelease: release_sort = 3 @@ -270,11 +402,19 @@ class PythonVersion(object): release_sort = 0 elif self.is_debug: release_sort = 1 - return (self.major, self.minor, self.patch if self.patch else 0, release_sort) + return ( + company_sort, + self.major, + self.minor, + self.patch if self.patch else 0, + release_sort, + ) @property def version_tuple(self): - """Provides a version tuple for using as a dictionary key. + # type: () -> Tuple[int, Optional[int], Optional[int], bool, bool, bool] + """ + Provides a version tuple for using as a dictionary key. :return: A tuple describing the python version meetadata contained. :rtype: tuple @@ -291,45 +431,52 @@ class PythonVersion(object): def matches( self, - major=None, - minor=None, - patch=None, - pre=False, - dev=False, - arch=None, - debug=False, - name=None, + major=None, # type: Optional[int] + minor=None, # type: Optional[int] + patch=None, # type: Optional[int] + pre=False, # type: bool + dev=False, # type: bool + arch=None, # type: Optional[str] + debug=False, # type: bool + python_name=None, # type: Optional[str] ): + # type: (...) -> bool + result = False if arch: own_arch = self.get_architecture() if arch.isdigit(): arch = "{0}bit".format(arch) - return ( - (major is None or self.major == major) - and (minor is None or self.minor == minor) - and (patch is None or self.patch == patch) + if ( + (major is None or self.major and self.major == major) + and (minor is None or self.minor and self.minor == minor) + and (patch is None or self.patch and self.patch == patch) and (pre is None or self.is_prerelease == pre) and (dev is None or self.is_devrelease == dev) and (arch is None or own_arch == arch) and (debug is None or self.is_debug == debug) and ( - name is None - or (name and self.name) - and (self.name == name or self.name.startswith(name)) + python_name is None + or (python_name and self.name) + and (self.name == python_name or self.name.startswith(python_name)) ) - ) + ): + result = True + return result def as_major(self): + # type: () -> PythonVersion self_dict = attr.asdict(self, recurse=False, filter=_filter_none).copy() self_dict.update({"minor": None, "patch": None}) return self.create(**self_dict) def as_minor(self): + # type: () -> PythonVersion self_dict = attr.asdict(self, recurse=False, filter=_filter_none).copy() self_dict.update({"patch": None}) return self.create(**self_dict) def as_dict(self): + # type: () -> Dict[str, Union[int, bool, Version, None]] return { "major": self.major, "minor": self.minor, @@ -339,37 +486,70 @@ class PythonVersion(object): "is_devrelease": self.is_devrelease, "is_debug": self.is_debug, "version": self.version, + "company": self.company, } + def update_metadata(self, metadata): + # type: (Dict[str, Union[str, int, Version]]) -> None + """ + Update the metadata on the current :class:`pythonfinder.models.python.PythonVersion` + + Given a parsed version dictionary from :func:`pythonfinder.utils.parse_python_version`, + update the instance variables of the current version instance to reflect the newly + supplied values. + """ + + for key in metadata: + try: + current_value = getattr(self, key) + except AttributeError: + continue + else: + setattr(self, key, metadata[key]) + @classmethod + @lru_cache(maxsize=1024) def parse(cls, version): - """Parse a valid version string into a dictionary + # type: (str) -> Dict[str, Union[str, int, Version]] + """ + Parse a valid version string into a dictionary Raises: ValueError -- Unable to parse version string ValueError -- Not a valid python version + TypeError -- NoneType or unparseable type passed in - :param version: A valid version string - :type version: str + :param str version: A valid version string :return: A dictionary with metadata about the specified python version. - :rtype: dict. + :rtype: dict """ + if version is None: + raise TypeError("Must pass a value to parse!") version_dict = parse_python_version(str(version)) if not version_dict: raise ValueError("Not a valid python version: %r" % version) return version_dict def get_architecture(self): + # type: () -> str if self.architecture: return self.architecture - arch, _ = platform.architecture(self.comes_from.path.as_posix()) + arch = None + if self.comes_from is not None: + arch, _ = platform.architecture(self.comes_from.path.as_posix()) + elif self.executable is not None: + arch, _ = platform.architecture(self.executable) + if arch is None: + arch, _ = platform.architecture(sys.executable) self.architecture = arch return self.architecture @classmethod - def from_path(cls, path, name=None): - """Parses a python version from a system path. + def from_path(cls, path, name=None, ignore_unsupported=True, company=None): + # type: (Union[str, PathEntry], Optional[str], bool, Optional[str]) -> PythonVersion + """ + Parses a python version from a system path. Raises: ValueError -- Not a valid python path @@ -377,35 +557,76 @@ class PythonVersion(object): :param path: A string or :class:`~pythonfinder.models.path.PathEntry` :type path: str or :class:`~pythonfinder.models.path.PathEntry` instance :param str name: Name of the python distribution in question + :param bool ignore_unsupported: Whether to ignore or error on unsupported paths. + :param Optional[str] company: The company or vendor packaging the distribution. :return: An instance of a PythonVersion. :rtype: :class:`~pythonfinder.models.python.PythonVersion` """ from .path import PathEntry - from ..environment import IGNORE_UNSUPPORTED if not isinstance(path, PathEntry): path = PathEntry.create(path, is_root=False, only_python=True, name=name) - if not path.is_python and not IGNORE_UNSUPPORTED: - raise ValueError("Not a valid python path: %s" % path.path) - return - py_version = get_python_version(path.path.absolute().as_posix()) - instance_dict = cls.parse(py_version.strip()) - if not isinstance(instance_dict.get("version"), Version) and not IGNORE_UNSUPPORTED: - raise ValueError("Not a valid python path: %s" % path.path) - return - if not name: - name = path.name + from ..environment import IGNORE_UNSUPPORTED + + ignore_unsupported = ignore_unsupported or IGNORE_UNSUPPORTED + path_name = getattr(path, "name", path.path.name) # str + if not path.is_python: + if not (ignore_unsupported or IGNORE_UNSUPPORTED): + raise ValueError("Not a valid python path: %s" % path.path) + try: + instance_dict = cls.parse(path_name) + except Exception: + instance_dict = cls.parse_executable(path.path.absolute().as_posix()) + else: + if instance_dict.get("minor") is None and looks_like_python(path.path.name): + instance_dict = cls.parse_executable(path.path.absolute().as_posix()) + + if ( + not isinstance(instance_dict.get("version"), Version) + and not ignore_unsupported + ): + raise ValueError("Not a valid python path: %s" % path) + if instance_dict.get("patch") is None: + instance_dict = cls.parse_executable(path.path.absolute().as_posix()) + if name is None: + name = path_name + if company is None: + company = guess_company(path.path.as_posix()) instance_dict.update( - {"comes_from": path, "name": name} + {"comes_from": path, "name": name, "executable": path.path.as_posix()} ) - return cls(**instance_dict) + return cls(**instance_dict) # type: ignore @classmethod - def from_windows_launcher(cls, launcher_entry, name=None): + @lru_cache(maxsize=1024) + def parse_executable(cls, path): + # type: (str) -> Dict[str, Optional[Union[str, int, Version]]] + result_dict = {} # type: Dict[str, Optional[Union[str, int, Version]]] + result_version = None # type: Optional[str] + if path is None: + raise TypeError("Must pass a valid path to parse.") + if not isinstance(path, six.string_types): + path = path.as_posix() + # if not looks_like_python(path): + # raise ValueError("Path %r does not look like a valid python path" % path) + try: + result_version = get_python_version(path) + except Exception: + raise ValueError("Not a valid python path: %r" % path) + if result_version is None: + raise ValueError("Not a valid python path: %s" % path) + result_dict = cls.parse(result_version.strip()) + return result_dict + + @classmethod + def from_windows_launcher(cls, launcher_entry, name=None, company=None): + # type: (Environment, Optional[str], Optional[str]) -> PythonVersion """Create a new PythonVersion instance from a Windows Launcher Entry :param launcher_entry: A python launcher environment object. + :param Optional[str] name: The name of the distribution. + :param Optional[str] company: The name of the distributing company. :return: An instance of a PythonVersion. :rtype: :class:`~pythonfinder.models.python.PythonVersion` """ @@ -420,24 +641,26 @@ class PythonVersion(object): exe_path = ensure_path( getattr(launcher_entry.info.install_path, "executable_path", default_path) ) + company = getattr(launcher_entry, "company", guess_company(exe_path.as_posix())) creation_dict.update( { "architecture": getattr( launcher_entry.info, "sys_architecture", SYSTEM_ARCH ), "executable": exe_path, - "name": name + "name": name, + "company": company, } ) py_version = cls.create(**creation_dict) comes_from = PathEntry.create(exe_path, only_python=True, name=name) - comes_from.py_version = copy.deepcopy(py_version) py_version.comes_from = comes_from py_version.name = comes_from.name return py_version @classmethod def create(cls, **kwargs): + # type: (...) -> PythonVersion if "architecture" in kwargs: if kwargs["architecture"].isdigit(): kwargs["architecture"] = "{0}bit".format(kwargs["architecture"]) @@ -446,10 +669,13 @@ class PythonVersion(object): @attr.s class VersionMap(object): - versions = attr.ib(default=attr.Factory(defaultdict(list))) + versions = attr.ib( + factory=defaultdict + ) # type: DefaultDict[Tuple[int, Optional[int], Optional[int], bool, bool, bool], List[PathEntry]] def add_entry(self, entry): - version = entry.as_python + # type: (...) -> None + version = entry.as_python # type: PythonVersion if version: entries = self.versions[version.version_tuple] paths = {p.path for p in self.versions.get(version.version_tuple, [])} @@ -457,13 +683,18 @@ class VersionMap(object): self.versions[version.version_tuple].append(entry) def merge(self, target): + # type: (VersionMap) -> None for version, entries in target.versions.items(): if version not in self.versions: self.versions[version] = entries else: - current_entries = {p.path for p in self.versions.get(version)} + current_entries = { + p.path + for p in self.versions[version] # type: ignore + if version in self.versions + } new_entries = {p.path for p in entries} new_entries -= current_entries - self.versions[version].append( + self.versions[version].extend( [e for e in entries if e.path in new_entries] ) diff --git a/pipenv/vendor/pythonfinder/models/windows.py b/pipenv/vendor/pythonfinder/models/windows.py index f985630f..a0e69b03 100644 --- a/pipenv/vendor/pythonfinder/models/windows.py +++ b/pipenv/vendor/pythonfinder/models/windows.py @@ -2,63 +2,61 @@ from __future__ import absolute_import, print_function import operator - from collections import defaultdict import attr +from ..environment import MYPY_RUNNING from ..exceptions import InvalidPythonVersion from ..utils import ensure_path from .mixins import BaseFinder from .path import PathEntry from .python import PythonVersion, VersionMap +if MYPY_RUNNING: + from typing import DefaultDict, Tuple, List, Optional, Union, TypeVar, Type, Any + + FinderType = TypeVar("FinderType") + @attr.s class WindowsFinder(BaseFinder): - paths = attr.ib(default=attr.Factory(list)) - version_list = attr.ib(default=attr.Factory(list)) - versions = attr.ib() - pythons = attr.ib() + paths = attr.ib(default=attr.Factory(list), type=list) + version_list = attr.ib(default=attr.Factory(list), type=list) + _versions = attr.ib() # type: DefaultDict[Tuple, PathEntry] + _pythons = attr.ib() # type: DefaultDict[str, PathEntry] def find_all_python_versions( self, - major=None, - minor=None, - patch=None, - pre=None, - dev=None, - arch=None, - name=None, + major=None, # type: Optional[Union[str, int]] + minor=None, # type: Optional[int] + patch=None, # type: Optional[int] + pre=None, # type: Optional[bool] + dev=None, # type: Optional[bool] + arch=None, # type: Optional[str] + name=None, # type: Optional[str] ): + # type (...) -> List[PathEntry] version_matcher = operator.methodcaller( - "matches", - major=major, - minor=minor, - patch=patch, - pre=pre, - dev=dev, - arch=arch, - name=name, - ) - py_filter = filter( - None, filter(lambda c: version_matcher(c), self.version_list) + "matches", major, minor, patch, pre, dev, arch, python_name=name ) + pythons = [py for py in self.version_list if version_matcher(py)] version_sort = operator.attrgetter("version_sort") - return [c.comes_from for c in sorted(py_filter, key=version_sort, reverse=True)] + return [c.comes_from for c in sorted(pythons, key=version_sort, reverse=True)] def find_python_version( self, - major=None, - minor=None, - patch=None, - pre=None, - dev=None, - arch=None, - name=None, + major=None, # type: Optional[Union[str, int]] + minor=None, # type: Optional[int] + patch=None, # type: Optional[int] + pre=None, # type: Optional[bool] + dev=None, # type: Optional[bool] + arch=None, # type: Optional[str] + name=None, # type: Optional[str] ): + # type: (...) -> Optional[PathEntry] return next( - ( + iter( v for v in self.find_all_python_versions( major=major, @@ -67,21 +65,24 @@ class WindowsFinder(BaseFinder): pre=pre, dev=dev, arch=arch, - name=None, + name=name, ) ), None, ) - @versions.default + @_versions.default def get_versions(self): - versions = defaultdict(PathEntry) + # type: () -> DefaultDict[Tuple, PathEntry] + versions = defaultdict(PathEntry) # type: DefaultDict[Tuple, PathEntry] from pythonfinder._vendor.pep514tools import environment as pep514env env_versions = pep514env.findall() path = None for version_object in env_versions: install_path = getattr(version_object.info, "install_path", None) + name = getattr(version_object, "tag", None) + company = getattr(version_object, "company", None) if install_path is None: continue try: @@ -89,28 +90,54 @@ class WindowsFinder(BaseFinder): except AttributeError: continue try: - py_version = PythonVersion.from_windows_launcher(version_object) + py_version = PythonVersion.from_windows_launcher( + version_object, name=name, company=company + ) except InvalidPythonVersion: continue + if py_version is None: + continue self.version_list.append(py_version) + python_path = ( + py_version.comes_from.path + if py_version.comes_from + else py_version.executable + ) + python_kwargs = {python_path: py_version} if python_path is not None else {} base_dir = PathEntry.create( - path, - is_root=True, - only_python=True, - pythons={py_version.comes_from.path: py_version}, + path, is_root=True, only_python=True, pythons=python_kwargs ) versions[py_version.version_tuple[:5]] = base_dir self.paths.append(base_dir) return versions - @pythons.default + @property + def versions(self): + # type: () -> DefaultDict[Tuple, PathEntry] + if not self._versions: + self._versions = self.get_versions() + return self._versions + + @_pythons.default def get_pythons(self): - pythons = defaultdict() + # type: () -> DefaultDict[str, PathEntry] + pythons = defaultdict() # type: DefaultDict[str, PathEntry] for version in self.version_list: _path = ensure_path(version.comes_from.path) pythons[_path.as_posix()] = version.comes_from return pythons + @property + def pythons(self): + # type: () -> DefaultDict[str, PathEntry] + return self._pythons + + @pythons.setter + def pythons(self, value): + # type: (DefaultDict[str, PathEntry]) -> None + self._pythons = value + @classmethod - def create(cls): + def create(cls, *args, **kwargs): + # type: (Type[FinderType], Any, Any) -> FinderType return cls() diff --git a/pipenv/vendor/pythonfinder/pep514tools.LICENSE b/pipenv/vendor/pythonfinder/pep514tools.LICENSE new file mode 100644 index 00000000..c7ac395f --- /dev/null +++ b/pipenv/vendor/pythonfinder/pep514tools.LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2016 Steve Dower + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/pipenv/vendor/pythonfinder/pythonfinder.py b/pipenv/vendor/pythonfinder/pythonfinder.py index 011754ea..400a3170 100644 --- a/pipenv/vendor/pythonfinder/pythonfinder.py +++ b/pipenv/vendor/pythonfinder/pythonfinder.py @@ -1,62 +1,127 @@ # -*- coding=utf-8 -*- -from __future__ import print_function, absolute_import -import os -import six +from __future__ import absolute_import, print_function + +import importlib import operator -from .models import SystemPath +import os + +import six +from click import secho from vistir.compat import lru_cache +from . import environment +from .exceptions import InvalidPythonVersion +from .utils import Iterable, filter_pythons, version_re + +if environment.MYPY_RUNNING: + from typing import Optional, Dict, Any, Union, List, Iterator, Text + from .models.path import Path, PathEntry + from .models.windows import WindowsFinder + from .models.path import SystemPath + + STRING_TYPE = Union[str, Text, bytes] + class Finder(object): - def __init__(self, path=None, system=False, global_search=True, ignore_unsupported=True): - """ - Finder A cross-platform Finder for locating python and other executables. - Searches for python and other specified binaries starting in `path`, if supplied, - but searching the bin path of `sys.executable` if `system=True`, and then - searching in the `os.environ['PATH']` if `global_search=True`. When `global_search` - is `False`, this search operation is restricted to the allowed locations of - `path` and `system`. + """ + A cross-platform Finder for locating python and other executables. + + Searches for python and other specified binaries starting in *path*, if supplied, + but searching the bin path of ``sys.executable`` if *system* is ``True``, and then + searching in the ``os.environ['PATH']`` if *global_search* is ``True``. When *global_search* + is ``False``, this search operation is restricted to the allowed locations of + *path* and *system*. + """ + + def __init__( + self, + path=None, + system=False, + global_search=True, + ignore_unsupported=True, + sort_by_path=False, + ): + # type: (Optional[str], bool, bool, bool, bool) -> None + """Create a new :class:`~pythonfinder.pythonfinder.Finder` instance. :param path: A bin-directory search location, defaults to None :param path: str, optional - :param system: Whether to include the bin-dir of `sys.executable`, defaults to False + :param system: Whether to include the bin-dir of ``sys.executable``, defaults to False :param system: bool, optional :param global_search: Whether to search the global path from os.environ, defaults to True :param global_search: bool, optional :param ignore_unsupported: Whether to ignore unsupported python versions, if False, an error is raised, defaults to True :param ignore_unsupported: bool, optional + :param bool sort_by_path: Whether to always sort by path :returns: a :class:`~pythonfinder.pythonfinder.Finder` object. """ - self.path_prepend = path - self.global_search = global_search - self.system = system - self.ignore_unsupported = ignore_unsupported - self._system_path = None - self._windows_finder = None + self.path_prepend = path # type: Optional[str] + self.global_search = global_search # type: bool + self.system = system # type: bool + self.sort_by_path = sort_by_path # type: bool + self.ignore_unsupported = ignore_unsupported # type: bool + self._system_path = None # type: Optional[SystemPath] + self._windows_finder = None # type: Optional[WindowsFinder] def __hash__(self): + # type: () -> int return hash( (self.path_prepend, self.system, self.global_search, self.ignore_unsupported) ) def __eq__(self, other): + # type: (Any) -> bool return self.__hash__() == other.__hash__() + def create_system_path(self): + # type: () -> SystemPath + pyfinder_path = importlib.import_module("pythonfinder.models.path") + return pyfinder_path.SystemPath.create( + path=self.path_prepend, + system=self.system, + global_search=self.global_search, + ignore_unsupported=self.ignore_unsupported, + ) + + def reload_system_path(self): + # type: () -> None + """ + Rebuilds the base system path and all of the contained finders within it. + + This will re-apply any changes to the environment or any version changes on the system. + """ + + if self._system_path is not None: + self._system_path = self._system_path.clear_caches() + self._system_path = None + pyfinder_path = importlib.import_module("pythonfinder.models.path") + six.moves.reload_module(pyfinder_path) + self._system_path = self.create_system_path() + + def rehash(self): + # type: () -> "Finder" + if not self._system_path: + self._system_path = self.create_system_path() + self.find_all_python_versions.cache_clear() + self.find_python_version.cache_clear() + if self._windows_finder is not None: + self._windows_finder = None + filter_pythons.cache_clear() + self.reload_system_path() + return self + @property def system_path(self): - if not self._system_path: - self._system_path = SystemPath.create( - path=self.path_prepend, - system=self.system, - global_search=self.global_search, - ignore_unsupported=self.ignore_unsupported, - ) + # type: () -> SystemPath + if self._system_path is None: + self._system_path = self.create_system_path() return self._system_path @property def windows_finder(self): + # type: () -> Optional[WindowsFinder] if os.name == "nt" and not self._windows_finder: from .models import WindowsFinder @@ -64,13 +129,119 @@ class Finder(object): return self._windows_finder def which(self, exe): + # type: (str) -> Optional[PathEntry] return self.system_path.which(exe) + @classmethod + def parse_major(cls, major, minor=None, patch=None, pre=None, dev=None, arch=None): + # type: (Optional[str], Optional[int], Optional[int], Optional[bool], Optional[bool], Optional[str]) -> Dict[str, Union[int, str, bool, None]] + from .models import PythonVersion + + major_is_str = major and isinstance(major, six.string_types) + is_num = ( + major + and major_is_str + and all(part.isdigit() for part in major.split(".")[:2]) + ) + major_has_arch = ( + arch is None + and major + and major_is_str + and "-" in major + and major[0].isdigit() + ) + name = None + if major and major_has_arch: + orig_string = "{0!s}".format(major) + major, _, arch = major.rpartition("-") + if arch: + arch = arch.lower().lstrip("x").replace("bit", "") + if not (arch.isdigit() and (int(arch) & int(arch) - 1) == 0): + major = orig_string + arch = None + else: + arch = "{0}bit".format(arch) + try: + version_dict = PythonVersion.parse(major) + except (ValueError, InvalidPythonVersion): + if name is None: + name = "{0!s}".format(major) + major = None + version_dict = {} + elif major and major[0].isalpha(): + return {"major": None, "name": major, "arch": arch} + elif major and is_num: + match = version_re.match(major) + version_dict = match.groupdict() if match else {} # type: ignore + version_dict.update( + { + "is_prerelease": bool(version_dict.get("prerel", False)), + "is_devrelease": bool(version_dict.get("dev", False)), + } + ) + else: + version_dict = { + "major": major, + "minor": minor, + "patch": patch, + "pre": pre, + "dev": dev, + "arch": arch, + } + if not version_dict.get("arch") and arch: + version_dict["arch"] = arch + version_dict["minor"] = ( + int(version_dict["minor"]) if version_dict.get("minor") is not None else minor + ) + version_dict["patch"] = ( + int(version_dict["patch"]) if version_dict.get("patch") is not None else patch + ) + version_dict["major"] = ( + int(version_dict["major"]) if version_dict.get("major") is not None else major + ) + if not (version_dict["major"] or version_dict.get("name")): + version_dict["major"] = major + if name: + version_dict["name"] = name + return version_dict + @lru_cache(maxsize=1024) def find_python_version( - self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None, name=None + self, + major=None, # type: Optional[Union[str, int]] + minor=None, # type: Optional[int] + patch=None, # type: Optional[int] + pre=None, # type: Optional[bool] + dev=None, # type: Optional[bool] + arch=None, # type: Optional[str] + name=None, # type: Optional[str] + sort_by_path=False, # type: bool ): - from .models import PythonVersion + # type: (...) -> Optional[PathEntry] + """ + Find the python version which corresponds most closely to the version requested. + + :param Union[str, int] major: The major version to look for, or the full version, or the name of the target version. + :param Optional[int] minor: The minor version. If provided, disables string-based lookups from the major version field. + :param Optional[int] patch: The patch version. + :param Optional[bool] pre: If provided, specifies whether to search pre-releases. + :param Optional[bool] dev: If provided, whether to search dev-releases. + :param Optional[str] arch: If provided, which architecture to search. + :param Optional[str] name: *Name* of the target python, e.g. ``anaconda3-5.3.0`` + :param bool sort_by_path: Whether to sort by path -- default sort is by version(default: False) + :return: A new *PathEntry* pointer at a matching python version, if one can be located. + :rtype: :class:`pythonfinder.models.path.PathEntry` + """ + + minor = int(minor) if minor is not None else minor + patch = int(patch) if patch is not None else patch + + version_dict = { + "minor": minor, + "patch": patch, + "name": name, + "arch": arch, + } # type: Dict[str, Union[str, int, Any]] if ( isinstance(major, six.string_types) @@ -79,71 +250,72 @@ class Finder(object): and dev is None and patch is None ): - if arch is None and "-" in major: - orig_string = "{0!s}".format(major) - major, _, arch = major.rpartition("-") - if arch.startswith("x"): - arch = arch.lstrip("x") - if arch.lower().endswith("bit"): - arch = arch.lower().replace("bit", "") - if not (arch.isdigit() and (int(arch) & int(arch) - 1) == 0): - major = orig_string - arch = None - else: - arch = "{0}bit".format(arch) - try: - version_dict = PythonVersion.parse(major) - except ValueError: - if name is None: - name = "{0!s}".format(major) - major = None - version_dict = {} - major = version_dict.get("major", major) - minor = version_dict.get("minor", minor) - patch = version_dict.get("patch", patch) - pre = version_dict.get("is_prerelease", pre) if pre is None else pre - dev = version_dict.get("is_devrelease", dev) if dev is None else dev - arch = version_dict.get("architecture", arch) if arch is None else arch - if os.name == "nt": - match = self.windows_finder.find_python_version( - major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch, name=name + version_dict = self.parse_major(major, minor=minor, patch=patch, arch=arch) + major = version_dict["major"] + minor = version_dict.get("minor", minor) # type: ignore + patch = version_dict.get("patch", patch) # type: ignore + arch = version_dict.get("arch", arch) # type: ignore + name = version_dict.get("name", name) # type: ignore + _pre = version_dict.get("is_prerelease", pre) + pre = bool(_pre) if _pre is not None else pre + _dev = version_dict.get("is_devrelease", dev) + dev = bool(_dev) if _dev is not None else dev + if "architecture" in version_dict and isinstance( + version_dict["architecture"], six.string_types + ): + arch = version_dict["architecture"] # type: ignore + if os.name == "nt" and self.windows_finder is not None: + found = self.windows_finder.find_python_version( + major=major, + minor=minor, + patch=patch, + pre=pre, + dev=dev, + arch=arch, + name=name, ) - if match: - return match + if found: + return found return self.system_path.find_python_version( - major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch, name=name + major=major, + minor=minor, + patch=patch, + pre=pre, + dev=dev, + arch=arch, + name=name, + sort_by_path=self.sort_by_path, ) @lru_cache(maxsize=1024) def find_all_python_versions( self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None, name=None ): + # type: (Optional[Union[str, int]], Optional[int], Optional[int], Optional[bool], Optional[bool], Optional[str], Optional[str]) -> List[PathEntry] version_sort = operator.attrgetter("as_python.version_sort") python_version_dict = getattr(self.system_path, "python_version_dict") if python_version_dict: - paths = filter( - None, - [ - path - for version in python_version_dict.values() - for path in version - if path.as_python - ], + paths = ( + path + for version in python_version_dict.values() + for path in version + if path is not None and path.as_python ) - paths = sorted(paths, key=version_sort, reverse=True) - return paths + path_list = sorted(paths, key=version_sort, reverse=True) + return path_list versions = self.system_path.find_all_python_versions( major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch, name=name ) - if not isinstance(versions, list): + if not isinstance(versions, Iterable): versions = [versions] - paths = sorted(versions, key=version_sort, reverse=True) - path_map = {} - for path in paths: + # This list has already been mostly sorted on windows, we don't need to reverse it again + path_list = sorted(versions, key=version_sort, reverse=True) + path_map = {} # type: Dict[str, PathEntry] + for path in path_list: try: resolved_path = path.path.resolve() except OSError: resolved_path = path.path.absolute() if not path_map.get(resolved_path.as_posix()): path_map[resolved_path.as_posix()] = path - return list(path_map.values()) + return path_list diff --git a/pipenv/vendor/pythonfinder/utils.py b/pipenv/vendor/pythonfinder/utils.py index d8edb239..477f3668 100644 --- a/pipenv/vendor/pythonfinder/utils.py +++ b/pipenv/vendor/pythonfinder/utils.py @@ -1,60 +1,88 @@ # -*- coding=utf-8 -*- from __future__ import absolute_import, print_function +import io import itertools import os - +import re from fnmatch import fnmatch +from threading import Timer import attr -import io -import re import six - import vistir - from packaging.version import LegacyVersion, Version -from .environment import PYENV_ROOT, ASDF_DATA_DIR, MYPY_RUNNING +from .environment import MYPY_RUNNING, PYENV_ROOT, SUBPROCESS_TIMEOUT from .exceptions import InvalidPythonVersion -six.add_move(six.MovedAttribute("Iterable", "collections", "collections.abc")) -from six.moves import Iterable +six.add_move( + six.MovedAttribute("Iterable", "collections", "collections.abc") +) # type: ignore # noqa +six.add_move( + six.MovedAttribute("Sequence", "collections", "collections.abc") +) # type: ignore # noqa +# fmt: off +from six.moves import Iterable # type: ignore # noqa # isort:skip +from six.moves import Sequence # type: ignore # noqa # isort:skip +# fmt: on try: from functools import lru_cache except ImportError: - from backports.functools_lru_cache import lru_cache + from backports.functools_lru_cache import lru_cache # type: ignore # noqa if MYPY_RUNNING: - from typing import Any, Union, List, Callable, Iterable, Set, Tuple, Dict, Optional - from attr.validators import _OptionalValidator + from typing import Any, Union, List, Callable, Set, Tuple, Dict, Optional, Iterator + from attr.validators import _OptionalValidator # type: ignore + from .models.path import PathEntry -version_re = re.compile(r"(?P\d+)\.(?P\d+)\.?(?P(?<=\.)[0-9]+)?\.?" - r"(?:(?P[abc]|rc|dev)(?:(?P\d+(?:\.\d+)*))?)" - r"?(?P(\.post(?P\d+))?(\.dev(?P\d+))?)?") +version_re_str = ( + r"(?P\d+)(?:\.(?P\d+))?(?:\.(?P(?<=\.)[0-9]+))?\.?" + r"(?:(?P[abc]|rc|dev)(?:(?P\d+(?:\.\d+)*))?)" + r"?(?P(\.post(?P\d+))?(\.dev(?P\d+))?)?" +) +version_re = re.compile(version_re_str) PYTHON_IMPLEMENTATIONS = ( - "python", "ironpython", "jython", "pypy", "anaconda", "miniconda", - "stackless", "activepython", "micropython" + "python", + "ironpython", + "jython", + "pypy", + "anaconda", + "miniconda", + "stackless", + "activepython", + "pyston", + "micropython", ) -RULES_BASE = ["*{0}", "*{0}?", "*{0}?.?", "*{0}?.?m"] -RULES = [rule.format(impl) for impl in PYTHON_IMPLEMENTATIONS for rule in RULES_BASE] - KNOWN_EXTS = {"exe", "py", "fish", "sh", ""} KNOWN_EXTS = KNOWN_EXTS | set( filter(None, os.environ.get("PATHEXT", "").split(os.pathsep)) ) +PY_MATCH_STR = r"((?P{0})(?:\d?(?:\.\d[cpm]{{0,3}}))?(?:-?[\d\.]+)*[^zw])".format( + "|".join(PYTHON_IMPLEMENTATIONS) +) +EXE_MATCH_STR = r"{0}(?:\.(?P{1}))?".format(PY_MATCH_STR, "|".join(KNOWN_EXTS)) +RE_MATCHER = re.compile(r"({0}|{1})".format(version_re_str, PY_MATCH_STR)) +EXE_MATCHER = re.compile(EXE_MATCH_STR) +RULES_BASE = [ + "*{0}", + "*{0}?", + "*{0}?.?", + "*{0}?.?m", + "{0}?-?.?", + "{0}?-?.?.?", + "{0}?.?-?.?.?", +] +RULES = [rule.format(impl) for impl in PYTHON_IMPLEMENTATIONS for rule in RULES_BASE] MATCH_RULES = [] for rule in RULES: MATCH_RULES.extend( - [ - "{0}.{1}".format(rule, ext) if ext else "{0}".format(rule) - for ext in KNOWN_EXTS - ] + ["{0}.{1}".format(rule, ext) if ext else "{0}".format(rule) for ext in KNOWN_EXTS] ) @@ -62,10 +90,21 @@ for rule in RULES: def get_python_version(path): # type: (str) -> str """Get python version string using subprocess from a given path.""" - version_cmd = [path, "-c", "import sys; print(sys.version.split()[0])"] + version_cmd = [ + path, + "-c", + "import sys; print('.'.join([str(i) for i in sys.version_info[:3]]))", + ] try: - c = vistir.misc.run(version_cmd, block=True, nospin=True, return_object=True, - combine_stderr=False, write_to_stdout=False) + c = vistir.misc.run( + version_cmd, + block=True, + nospin=True, + return_object=True, + combine_stderr=False, + write_to_stdout=False, + ) + timer = Timer(SUBPROCESS_TIMEOUT, c.kill) except OSError: raise InvalidPythonVersion("%s is not a valid python path" % path) if not c.out: @@ -77,14 +116,15 @@ def get_python_version(path): def parse_python_version(version_str): # type: (str) -> Dict[str, Union[str, int, Version]] from packaging.version import parse as parse_version + is_debug = False if version_str.endswith("-debug"): is_debug = True version_str, _, _ = version_str.rpartition("-") - m = version_re.match(version_str) - if not m: + match = version_re.match(version_str) + if not match: raise InvalidPythonVersion("%s is not a python version" % version_str) - version_dict = m.groupdict() # type: Dict[str, str] + version_dict = match.groupdict() # type: Dict[str, str] major = int(version_dict.get("major", 0)) if version_dict.get("major") else None minor = int(version_dict.get("minor", 0)) if version_dict.get("minor") else None patch = int(version_dict.get("patch", 0)) if version_dict.get("patch") else None @@ -97,8 +137,18 @@ def parse_python_version(version_str): try: version = parse_version(version_str) except TypeError: - version_parts = [str(v) for v in [major, minor, patch] if v is not None] - version = parse_version(".".join(version_parts)) + version = None + if isinstance(version, LegacyVersion) or version is None: + v_dict = version_dict.copy() + pre = "" + if v_dict.get("prerel") and v_dict.get("prerelversion"): + pre = v_dict.pop("prerel") + pre = "{0}{1}".format(pre, v_dict.pop("prerelversion")) + v_dict["pre"] = pre + keys = ["major", "minor", "patch", "pre", "postdev", "post", "dev"] + values = [v_dict.get(val) for val in keys] + version_str = ".".join([str(v) for v in values if v]) + version = parse_version(version_str) return { "major": major, "minor": minor, @@ -107,7 +157,7 @@ def parse_python_version(version_str): "is_prerelease": is_prerelease, "is_devrelease": is_devrelease, "is_debug": is_debug, - "version": version + "version": version, } @@ -168,7 +218,10 @@ def looks_like_python(name): if not any(name.lower().startswith(py_name) for py_name in PYTHON_IMPLEMENTATIONS): return False - return any(fnmatch(name, rule) for rule in MATCH_RULES) + match = RE_MATCHER.match(name) + if match: + return any(fnmatch(name, rule) for rule in MATCH_RULES) + return False @lru_cache(maxsize=1024) @@ -186,9 +239,43 @@ def path_is_python(path): return path_is_executable(path) and looks_like_python(path.name) +@lru_cache(maxsize=1024) +def guess_company(path): + # type: (str) -> Optional[str] + """Given a path to python, guess the company who created it + + :param str path: The path to guess about + :return: The guessed company + :rtype: Optional[str] + """ + non_core_pythons = [impl for impl in PYTHON_IMPLEMENTATIONS if impl != "python"] + return next( + iter(impl for impl in non_core_pythons if impl in path.lower()), "PythonCore" + ) + + +@lru_cache(maxsize=1024) +def path_is_pythoncore(path): + # type: (str) -> bool + """Given a path, determine whether it appears to be pythoncore. + + Does not verify whether the path is in fact a path to python, but simply + does an exclusionary check on the possible known python implementations + to see if their names are present in the path (fairly dumb check). + + :param str path: The path to check + :return: Whether that path is a PythonCore path or not + :rtype: bool + """ + company = guess_company(path) + if company: + return company == "PythonCore" + return False + + @lru_cache(maxsize=1024) def ensure_path(path): - # type: (Union[vistir.compat.Path, str]) -> bool + # type: (Union[vistir.compat.Path, str]) -> vistir.compat.Path """ Given a path (either a string or a Path object), expand variables and return a Path object. @@ -214,9 +301,11 @@ def _filter_none(k, v): # TODO: Reimplement in vistir def normalize_path(path): # type: (str) -> str - return os.path.normpath(os.path.normcase( - os.path.abspath(os.path.expandvars(os.path.expanduser(str(path)))) - )) + return os.path.normpath( + os.path.normcase( + os.path.abspath(os.path.expandvars(os.path.expanduser(str(path)))) + ) + ) @lru_cache(maxsize=1024) @@ -238,13 +327,16 @@ def unnest(item): item, target = itertools.tee(item, 2) else: target = item - for el in target: - if isinstance(el, Iterable) and not isinstance(el, six.string_types): - el, el_copy = itertools.tee(el, 2) - for sub in unnest(el_copy): - yield sub - else: - yield el + if getattr(target, "__iter__", None): + for el in target: + if isinstance(el, Iterable) and not isinstance(el, six.string_types): + el, el_copy = itertools.tee(el, 2) + for sub in unnest(el_copy): + yield sub + else: + yield el + else: + yield target def parse_pyenv_version_order(filename="version"): @@ -264,16 +356,81 @@ def parse_asdf_version_order(filename=".tool-versions"): if os.path.exists(version_order_file) and os.path.isfile(version_order_file): with io.open(version_order_file, encoding="utf-8") as fh: contents = fh.read() - python_section = next(iter( - line for line in contents.splitlines() if line.startswith("python") - ), None) + python_section = next( + iter(line for line in contents.splitlines() if line.startswith("python")), + None, + ) if python_section: - python_key, _, versions = python_section.partition(" ") + # python_key, _, versions + _, _, versions = python_section.partition(" ") if versions: return versions.split() return [] +def split_version_and_name( + major=None, # type: Optional[Union[str, int]] + minor=None, # type: Optional[Union[str, int]] + patch=None, # type: Optional[Union[str, int]] + name=None, # type: Optional[str] +): + # type: (...) -> Tuple[Optional[Union[str, int]], Optional[Union[str, int]], Optional[Union[str, int]], Optional[str]] + if isinstance(major, six.string_types) and not minor and not patch: + # Only proceed if this is in the format "x.y.z" or similar + if major.isdigit() or (major.count(".") > 0 and major[0].isdigit()): + version = major.split(".", 2) + if isinstance(version, (tuple, list)): + if len(version) > 3: + major, minor, patch, _ = version + elif len(version) == 3: + major, minor, patch = version + elif len(version) == 2: + major, minor = version + else: + major = major[0] + else: + major = major + name = None + else: + name = "{0!s}".format(major) + major = None + return (major, minor, patch, name) + + # TODO: Reimplement in vistir def is_in_path(path, parent): return normalize_path(str(path)).startswith(normalize_path(str(parent))) + + +def expand_paths(path, only_python=True): + # type: (Union[Sequence, PathEntry], bool) -> Iterator + """ + Recursively expand a list or :class:`~pythonfinder.models.path.PathEntry` instance + + :param Union[Sequence, PathEntry] path: The path or list of paths to expand + :param bool only_python: Whether to filter to include only python paths, default True + :returns: An iterator over the expanded set of path entries + :rtype: Iterator[PathEntry] + """ + + if path is not None and ( + isinstance(path, Sequence) + and not getattr(path.__class__, "__name__", "") == "PathEntry" + ): + for p in unnest(path): + if p is None: + continue + for expanded in itertools.chain.from_iterable( + expand_paths(p, only_python=only_python) + ): + yield expanded + elif path is not None and path.is_dir: + for p in path.children.values(): + if p is not None and p.is_python and p.as_python is not None: + for sub_path in itertools.chain.from_iterable( + expand_paths(p, only_python=only_python) + ): + yield sub_path + else: + if path is not None and path.is_python and path.as_python is not None: + yield path diff --git a/pipenv/vendor/pytoml/LICENSE b/pipenv/vendor/pytoml/LICENSE new file mode 100644 index 00000000..9739fc67 --- /dev/null +++ b/pipenv/vendor/pytoml/LICENSE @@ -0,0 +1,16 @@ +No-notice MIT License + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/pipenv/vendor/pytoml/__init__.py b/pipenv/vendor/pytoml/__init__.py new file mode 100644 index 00000000..8ed060ff --- /dev/null +++ b/pipenv/vendor/pytoml/__init__.py @@ -0,0 +1,4 @@ +from .core import TomlError +from .parser import load, loads +from .test import translate_to_test +from .writer import dump, dumps \ No newline at end of file diff --git a/pipenv/vendor/pytoml/core.py b/pipenv/vendor/pytoml/core.py new file mode 100644 index 00000000..c182734e --- /dev/null +++ b/pipenv/vendor/pytoml/core.py @@ -0,0 +1,13 @@ +class TomlError(RuntimeError): + def __init__(self, message, line, col, filename): + RuntimeError.__init__(self, message, line, col, filename) + self.message = message + self.line = line + self.col = col + self.filename = filename + + def __str__(self): + return '{}({}, {}): {}'.format(self.filename, self.line, self.col, self.message) + + def __repr__(self): + return 'TomlError({!r}, {!r}, {!r}, {!r})'.format(self.message, self.line, self.col, self.filename) diff --git a/pipenv/vendor/pytoml/parser.py b/pipenv/vendor/pytoml/parser.py new file mode 100644 index 00000000..3493aa64 --- /dev/null +++ b/pipenv/vendor/pytoml/parser.py @@ -0,0 +1,341 @@ +import string, re, sys, datetime +from .core import TomlError +from .utils import rfc3339_re, parse_rfc3339_re + +if sys.version_info[0] == 2: + _chr = unichr +else: + _chr = chr + +def load(fin, translate=lambda t, x, v: v, object_pairs_hook=dict): + return loads(fin.read(), translate=translate, object_pairs_hook=object_pairs_hook, filename=getattr(fin, 'name', repr(fin))) + +def loads(s, filename='', translate=lambda t, x, v: v, object_pairs_hook=dict): + if isinstance(s, bytes): + s = s.decode('utf-8') + + s = s.replace('\r\n', '\n') + + root = object_pairs_hook() + tables = object_pairs_hook() + scope = root + + src = _Source(s, filename=filename) + ast = _p_toml(src, object_pairs_hook=object_pairs_hook) + + def error(msg): + raise TomlError(msg, pos[0], pos[1], filename) + + def process_value(v, object_pairs_hook): + kind, text, value, pos = v + if kind == 'str' and value.startswith('\n'): + value = value[1:] + if kind == 'array': + if value and any(k != value[0][0] for k, t, v, p in value[1:]): + error('array-type-mismatch') + value = [process_value(item, object_pairs_hook=object_pairs_hook) for item in value] + elif kind == 'table': + value = object_pairs_hook([(k, process_value(value[k], object_pairs_hook=object_pairs_hook)) for k in value]) + return translate(kind, text, value) + + for kind, value, pos in ast: + if kind == 'kv': + k, v = value + if k in scope: + error('duplicate_keys. Key "{0}" was used more than once.'.format(k)) + scope[k] = process_value(v, object_pairs_hook=object_pairs_hook) + else: + is_table_array = (kind == 'table_array') + cur = tables + for name in value[:-1]: + if isinstance(cur.get(name), list): + d, cur = cur[name][-1] + else: + d, cur = cur.setdefault(name, (None, object_pairs_hook())) + + scope = object_pairs_hook() + name = value[-1] + if name not in cur: + if is_table_array: + cur[name] = [(scope, object_pairs_hook())] + else: + cur[name] = (scope, object_pairs_hook()) + elif isinstance(cur[name], list): + if not is_table_array: + error('table_type_mismatch') + cur[name].append((scope, object_pairs_hook())) + else: + if is_table_array: + error('table_type_mismatch') + old_scope, next_table = cur[name] + if old_scope is not None: + error('duplicate_tables') + cur[name] = (scope, next_table) + + def merge_tables(scope, tables): + if scope is None: + scope = object_pairs_hook() + for k in tables: + if k in scope: + error('key_table_conflict') + v = tables[k] + if isinstance(v, list): + scope[k] = [merge_tables(sc, tbl) for sc, tbl in v] + else: + scope[k] = merge_tables(v[0], v[1]) + return scope + + return merge_tables(root, tables) + +class _Source: + def __init__(self, s, filename=None): + self.s = s + self._pos = (1, 1) + self._last = None + self._filename = filename + self.backtrack_stack = [] + + def last(self): + return self._last + + def pos(self): + return self._pos + + def fail(self): + return self._expect(None) + + def consume_dot(self): + if self.s: + self._last = self.s[0] + self.s = self[1:] + self._advance(self._last) + return self._last + return None + + def expect_dot(self): + return self._expect(self.consume_dot()) + + def consume_eof(self): + if not self.s: + self._last = '' + return True + return False + + def expect_eof(self): + return self._expect(self.consume_eof()) + + def consume(self, s): + if self.s.startswith(s): + self.s = self.s[len(s):] + self._last = s + self._advance(s) + return True + return False + + def expect(self, s): + return self._expect(self.consume(s)) + + def consume_re(self, re): + m = re.match(self.s) + if m: + self.s = self.s[len(m.group(0)):] + self._last = m + self._advance(m.group(0)) + return m + return None + + def expect_re(self, re): + return self._expect(self.consume_re(re)) + + def __enter__(self): + self.backtrack_stack.append((self.s, self._pos)) + + def __exit__(self, type, value, traceback): + if type is None: + self.backtrack_stack.pop() + else: + self.s, self._pos = self.backtrack_stack.pop() + return type == TomlError + + def commit(self): + self.backtrack_stack[-1] = (self.s, self._pos) + + def _expect(self, r): + if not r: + raise TomlError('msg', self._pos[0], self._pos[1], self._filename) + return r + + def _advance(self, s): + suffix_pos = s.rfind('\n') + if suffix_pos == -1: + self._pos = (self._pos[0], self._pos[1] + len(s)) + else: + self._pos = (self._pos[0] + s.count('\n'), len(s) - suffix_pos) + +_ews_re = re.compile(r'(?:[ \t]|#[^\n]*\n|#[^\n]*\Z|\n)*') +def _p_ews(s): + s.expect_re(_ews_re) + +_ws_re = re.compile(r'[ \t]*') +def _p_ws(s): + s.expect_re(_ws_re) + +_escapes = { 'b': '\b', 'n': '\n', 'r': '\r', 't': '\t', '"': '"', + '\\': '\\', 'f': '\f' } + +_basicstr_re = re.compile(r'[^"\\\000-\037]*') +_short_uni_re = re.compile(r'u([0-9a-fA-F]{4})') +_long_uni_re = re.compile(r'U([0-9a-fA-F]{8})') +_escapes_re = re.compile(r'[btnfr\"\\]') +_newline_esc_re = re.compile('\n[ \t\n]*') +def _p_basicstr_content(s, content=_basicstr_re): + res = [] + while True: + res.append(s.expect_re(content).group(0)) + if not s.consume('\\'): + break + if s.consume_re(_newline_esc_re): + pass + elif s.consume_re(_short_uni_re) or s.consume_re(_long_uni_re): + v = int(s.last().group(1), 16) + if 0xd800 <= v < 0xe000: + s.fail() + res.append(_chr(v)) + else: + s.expect_re(_escapes_re) + res.append(_escapes[s.last().group(0)]) + return ''.join(res) + +_key_re = re.compile(r'[0-9a-zA-Z-_]+') +def _p_key(s): + with s: + s.expect('"') + r = _p_basicstr_content(s, _basicstr_re) + s.expect('"') + return r + if s.consume('\''): + if s.consume('\'\''): + r = s.expect_re(_litstr_ml_re).group(0) + s.expect('\'\'\'') + else: + r = s.expect_re(_litstr_re).group(0) + s.expect('\'') + return r + return s.expect_re(_key_re).group(0) + +_float_re = re.compile(r'[+-]?(?:0|[1-9](?:_?\d)*)(?:\.\d(?:_?\d)*)?(?:[eE][+-]?(?:\d(?:_?\d)*))?') + +_basicstr_ml_re = re.compile(r'(?:""?(?!")|[^"\\\000-\011\013-\037])*') +_litstr_re = re.compile(r"[^'\000\010\012-\037]*") +_litstr_ml_re = re.compile(r"(?:(?:|'|'')(?:[^'\000-\010\013-\037]))*") +def _p_value(s, object_pairs_hook): + pos = s.pos() + + if s.consume('true'): + return 'bool', s.last(), True, pos + if s.consume('false'): + return 'bool', s.last(), False, pos + + if s.consume('"'): + if s.consume('""'): + r = _p_basicstr_content(s, _basicstr_ml_re) + s.expect('"""') + else: + r = _p_basicstr_content(s, _basicstr_re) + s.expect('"') + return 'str', r, r, pos + + if s.consume('\''): + if s.consume('\'\''): + r = s.expect_re(_litstr_ml_re).group(0) + s.expect('\'\'\'') + else: + r = s.expect_re(_litstr_re).group(0) + s.expect('\'') + return 'str', r, r, pos + + if s.consume_re(rfc3339_re): + m = s.last() + return 'datetime', m.group(0), parse_rfc3339_re(m), pos + + if s.consume_re(_float_re): + m = s.last().group(0) + r = m.replace('_','') + if '.' in m or 'e' in m or 'E' in m: + return 'float', m, float(r), pos + else: + return 'int', m, int(r, 10), pos + + if s.consume('['): + items = [] + with s: + while True: + _p_ews(s) + items.append(_p_value(s, object_pairs_hook=object_pairs_hook)) + s.commit() + _p_ews(s) + s.expect(',') + s.commit() + _p_ews(s) + s.expect(']') + return 'array', None, items, pos + + if s.consume('{'): + _p_ws(s) + items = object_pairs_hook() + if not s.consume('}'): + k = _p_key(s) + _p_ws(s) + s.expect('=') + _p_ws(s) + items[k] = _p_value(s, object_pairs_hook=object_pairs_hook) + _p_ws(s) + while s.consume(','): + _p_ws(s) + k = _p_key(s) + _p_ws(s) + s.expect('=') + _p_ws(s) + items[k] = _p_value(s, object_pairs_hook=object_pairs_hook) + _p_ws(s) + s.expect('}') + return 'table', None, items, pos + + s.fail() + +def _p_stmt(s, object_pairs_hook): + pos = s.pos() + if s.consume( '['): + is_array = s.consume('[') + _p_ws(s) + keys = [_p_key(s)] + _p_ws(s) + while s.consume('.'): + _p_ws(s) + keys.append(_p_key(s)) + _p_ws(s) + s.expect(']') + if is_array: + s.expect(']') + return 'table_array' if is_array else 'table', keys, pos + + key = _p_key(s) + _p_ws(s) + s.expect('=') + _p_ws(s) + value = _p_value(s, object_pairs_hook=object_pairs_hook) + return 'kv', (key, value), pos + +_stmtsep_re = re.compile(r'(?:[ \t]*(?:#[^\n]*)?\n)+[ \t]*') +def _p_toml(s, object_pairs_hook): + stmts = [] + _p_ews(s) + with s: + stmts.append(_p_stmt(s, object_pairs_hook=object_pairs_hook)) + while True: + s.commit() + s.expect_re(_stmtsep_re) + stmts.append(_p_stmt(s, object_pairs_hook=object_pairs_hook)) + _p_ews(s) + s.expect_eof() + return stmts diff --git a/pipenv/vendor/pytoml/test.py b/pipenv/vendor/pytoml/test.py new file mode 100644 index 00000000..ec8abfc6 --- /dev/null +++ b/pipenv/vendor/pytoml/test.py @@ -0,0 +1,30 @@ +import datetime +from .utils import format_rfc3339 + +try: + _string_types = (str, unicode) + _int_types = (int, long) +except NameError: + _string_types = str + _int_types = int + +def translate_to_test(v): + if isinstance(v, dict): + return { k: translate_to_test(v) for k, v in v.items() } + if isinstance(v, list): + a = [translate_to_test(x) for x in v] + if v and isinstance(v[0], dict): + return a + else: + return {'type': 'array', 'value': a} + if isinstance(v, datetime.datetime): + return {'type': 'datetime', 'value': format_rfc3339(v)} + if isinstance(v, bool): + return {'type': 'bool', 'value': 'true' if v else 'false'} + if isinstance(v, _int_types): + return {'type': 'integer', 'value': str(v)} + if isinstance(v, float): + return {'type': 'float', 'value': '{:.17}'.format(v)} + if isinstance(v, _string_types): + return {'type': 'string', 'value': v} + raise RuntimeError('unexpected value: {!r}'.format(v)) diff --git a/pipenv/vendor/pytoml/utils.py b/pipenv/vendor/pytoml/utils.py new file mode 100644 index 00000000..636a680b --- /dev/null +++ b/pipenv/vendor/pytoml/utils.py @@ -0,0 +1,67 @@ +import datetime +import re + +rfc3339_re = re.compile(r'(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(\.\d+)?(?:Z|([+-]\d{2}):(\d{2}))') + +def parse_rfc3339(v): + m = rfc3339_re.match(v) + if not m or m.group(0) != v: + return None + return parse_rfc3339_re(m) + +def parse_rfc3339_re(m): + r = map(int, m.groups()[:6]) + if m.group(7): + micro = float(m.group(7)) + else: + micro = 0 + + if m.group(8): + g = int(m.group(8), 10) * 60 + int(m.group(9), 10) + tz = _TimeZone(datetime.timedelta(0, g * 60)) + else: + tz = _TimeZone(datetime.timedelta(0, 0)) + + y, m, d, H, M, S = r + return datetime.datetime(y, m, d, H, M, S, int(micro * 1000000), tz) + + +def format_rfc3339(v): + offs = v.utcoffset() + offs = int(offs.total_seconds()) // 60 if offs is not None else 0 + + if offs == 0: + suffix = 'Z' + else: + if offs > 0: + suffix = '+' + else: + suffix = '-' + offs = -offs + suffix = '{0}{1:02}:{2:02}'.format(suffix, offs // 60, offs % 60) + + if v.microsecond: + return v.strftime('%Y-%m-%dT%H:%M:%S.%f') + suffix + else: + return v.strftime('%Y-%m-%dT%H:%M:%S') + suffix + +class _TimeZone(datetime.tzinfo): + def __init__(self, offset): + self._offset = offset + + def utcoffset(self, dt): + return self._offset + + def dst(self, dt): + return None + + def tzname(self, dt): + m = self._offset.total_seconds() // 60 + if m < 0: + res = '-' + m = -m + else: + res = '+' + h = m // 60 + m = m - h * 60 + return '{}{:.02}{:.02}'.format(res, h, m) diff --git a/pipenv/vendor/pytoml/writer.py b/pipenv/vendor/pytoml/writer.py new file mode 100644 index 00000000..73b5089c --- /dev/null +++ b/pipenv/vendor/pytoml/writer.py @@ -0,0 +1,106 @@ +from __future__ import unicode_literals +import io, datetime, math, string, sys + +from .utils import format_rfc3339 + +if sys.version_info[0] == 3: + long = int + unicode = str + + +def dumps(obj, sort_keys=False): + fout = io.StringIO() + dump(obj, fout, sort_keys=sort_keys) + return fout.getvalue() + + +_escapes = {'\n': 'n', '\r': 'r', '\\': '\\', '\t': 't', '\b': 'b', '\f': 'f', '"': '"'} + + +def _escape_string(s): + res = [] + start = 0 + + def flush(): + if start != i: + res.append(s[start:i]) + return i + 1 + + i = 0 + while i < len(s): + c = s[i] + if c in '"\\\n\r\t\b\f': + start = flush() + res.append('\\' + _escapes[c]) + elif ord(c) < 0x20: + start = flush() + res.append('\\u%04x' % ord(c)) + i += 1 + + flush() + return '"' + ''.join(res) + '"' + + +_key_chars = string.digits + string.ascii_letters + '-_' +def _escape_id(s): + if any(c not in _key_chars for c in s): + return _escape_string(s) + return s + + +def _format_value(v): + if isinstance(v, bool): + return 'true' if v else 'false' + if isinstance(v, int) or isinstance(v, long): + return unicode(v) + if isinstance(v, float): + if math.isnan(v) or math.isinf(v): + raise ValueError("{0} is not a valid TOML value".format(v)) + else: + return repr(v) + elif isinstance(v, unicode) or isinstance(v, bytes): + return _escape_string(v) + elif isinstance(v, datetime.datetime): + return format_rfc3339(v) + elif isinstance(v, list): + return '[{0}]'.format(', '.join(_format_value(obj) for obj in v)) + elif isinstance(v, dict): + return '{{{0}}}'.format(', '.join('{} = {}'.format(_escape_id(k), _format_value(obj)) for k, obj in v.items())) + else: + raise RuntimeError(v) + + +def dump(obj, fout, sort_keys=False): + tables = [((), obj, False)] + + while tables: + name, table, is_array = tables.pop() + if name: + section_name = '.'.join(_escape_id(c) for c in name) + if is_array: + fout.write('[[{0}]]\n'.format(section_name)) + else: + fout.write('[{0}]\n'.format(section_name)) + + table_keys = sorted(table.keys()) if sort_keys else table.keys() + new_tables = [] + has_kv = False + for k in table_keys: + v = table[k] + if isinstance(v, dict): + new_tables.append((name + (k,), v, False)) + elif isinstance(v, list) and v and all(isinstance(o, dict) for o in v): + new_tables.extend((name + (k,), d, True) for d in v) + elif v is None: + # based on mojombo's comment: https://github.com/toml-lang/toml/issues/146#issuecomment-25019344 + fout.write( + '#{} = null # To use: uncomment and replace null with value\n'.format(_escape_id(k))) + has_kv = True + else: + fout.write('{0} = {1}\n'.format(_escape_id(k), _format_value(v))) + has_kv = True + + tables.extend(reversed(new_tables)) + + if (name or has_kv) and tables: + fout.write('\n') diff --git a/pipenv/vendor/requests/__init__.py b/pipenv/vendor/requests/__init__.py index bc168ee5..9a899df6 100644 --- a/pipenv/vendor/requests/__init__.py +++ b/pipenv/vendor/requests/__init__.py @@ -57,10 +57,10 @@ def check_compatibility(urllib3_version, chardet_version): # Check urllib3 for compatibility. major, minor, patch = urllib3_version # noqa: F811 major, minor, patch = int(major), int(minor), int(patch) - # urllib3 >= 1.21.1, <= 1.24 + # urllib3 >= 1.21.1, <= 1.25 assert major == 1 assert minor >= 21 - assert minor <= 24 + assert minor <= 25 # Check chardet for compatibility. major, minor, patch = chardet_version.split('.')[:3] diff --git a/pipenv/vendor/requests/__version__.py b/pipenv/vendor/requests/__version__.py index 803773a0..9844f740 100644 --- a/pipenv/vendor/requests/__version__.py +++ b/pipenv/vendor/requests/__version__.py @@ -5,10 +5,10 @@ __title__ = 'requests' __description__ = 'Python HTTP for Humans.' __url__ = 'http://python-requests.org' -__version__ = '2.20.1' -__build__ = 0x022001 +__version__ = '2.22.0' +__build__ = 0x022200 __author__ = 'Kenneth Reitz' __author_email__ = 'me@kennethreitz.org' __license__ = 'Apache 2.0' -__copyright__ = 'Copyright 2018 Kenneth Reitz' +__copyright__ = 'Copyright 2019 Kenneth Reitz' __cake__ = u'\u2728 \U0001f370 \u2728' diff --git a/pipenv/vendor/requests/api.py b/pipenv/vendor/requests/api.py index abada96d..ef71d075 100644 --- a/pipenv/vendor/requests/api.py +++ b/pipenv/vendor/requests/api.py @@ -19,7 +19,7 @@ def request(method, url, **kwargs): :param method: method for the new :class:`Request` object. :param url: URL for the new :class:`Request` object. :param params: (optional) Dictionary, list of tuples or bytes to send - in the body of the :class:`Request`. + in the query string for the :class:`Request`. :param data: (optional) Dictionary, list of tuples, bytes, or file-like object to send in the body of the :class:`Request`. :param json: (optional) A JSON serializable Python object to send in the body of the :class:`Request`. @@ -65,7 +65,7 @@ def get(url, params=None, **kwargs): :param url: URL for the new :class:`Request` object. :param params: (optional) Dictionary, list of tuples or bytes to send - in the body of the :class:`Request`. + in the query string for the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :return: :class:`Response ` object :rtype: requests.Response diff --git a/pipenv/vendor/requests/models.py b/pipenv/vendor/requests/models.py index 3dded57e..62dcd0b7 100644 --- a/pipenv/vendor/requests/models.py +++ b/pipenv/vendor/requests/models.py @@ -781,7 +781,7 @@ class Response(object): return chunks - def iter_lines(self, chunk_size=ITER_CHUNK_SIZE, decode_unicode=None, delimiter=None): + def iter_lines(self, chunk_size=ITER_CHUNK_SIZE, decode_unicode=False, delimiter=None): """Iterates over the response data, one line at a time. When stream=True is set on the request, this avoids reading the content at once into memory for large responses. diff --git a/pipenv/vendor/requirementslib/__init__.py b/pipenv/vendor/requirementslib/__init__.py index 32415c61..70b604a8 100644 --- a/pipenv/vendor/requirementslib/__init__.py +++ b/pipenv/vendor/requirementslib/__init__.py @@ -1,16 +1,21 @@ # -*- coding=utf-8 -*- -__version__ = '1.3.1' +from __future__ import absolute_import, print_function import logging import warnings + from vistir.compat import ResourceWarning +from .models.lockfile import Lockfile +from .models.pipfile import Pipfile +from .models.requirements import Requirement + +__version__ = "1.5.2.dev0" + + logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) warnings.filterwarnings("ignore", category=ResourceWarning) -from .models.requirements import Requirement -from .models.lockfile import Lockfile -from .models.pipfile import Pipfile __all__ = ["Lockfile", "Pipfile", "Requirement"] diff --git a/pipenv/vendor/requirementslib/environment.py b/pipenv/vendor/requirementslib/environment.py new file mode 100644 index 00000000..2a7d9b0e --- /dev/null +++ b/pipenv/vendor/requirementslib/environment.py @@ -0,0 +1,17 @@ +# -*- coding=utf-8 -*- +from __future__ import print_function, absolute_import + +import os +from appdirs import user_cache_dir + + +def is_type_checking(): + try: + from typing import TYPE_CHECKING + except ImportError: + return False + return TYPE_CHECKING + + +REQUIREMENTSLIB_CACHE_DIR = os.getenv("REQUIREMENTSLIB_CACHE_DIR", user_cache_dir("pipenv")) +MYPY_RUNNING = os.environ.get("MYPY_RUNNING", is_type_checking()) diff --git a/pipenv/vendor/requirementslib/exceptions.py b/pipenv/vendor/requirementslib/exceptions.py index 17b884eb..d11dbce9 100644 --- a/pipenv/vendor/requirementslib/exceptions.py +++ b/pipenv/vendor/requirementslib/exceptions.py @@ -1,19 +1,21 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function + import errno import os -import six import sys - +import six from vistir.compat import FileNotFoundError - if six.PY2: + class FileExistsError(OSError): def __init__(self, *args, **kwargs): self.errno = errno.EEXIST super(FileExistsError, self).__init__(*args, **kwargs) + + else: from six.moves.builtins import FileExistsError @@ -24,8 +26,15 @@ class RequirementError(Exception): class MissingParameter(Exception): def __init__(self, param): - Exception.__init__(self) - print("Missing parameter: %s" % param, file=sys.stderr, flush=True) + self.message = self.get_message(param) + super(MissingParameter, self).__init__(self.message) + + @classmethod + def get_message(cls, param): + return "Missing Parameter: %s" % param + + def show(self, param): + print(self.message, file=sys.stderr, flush=True) class FileCorruptException(OSError): @@ -35,58 +44,67 @@ class FileCorruptException(OSError): if not backup_path and args: args = reversed(args) backup_path = args.pop() - if not isinstance(backup_path, six.string_types) or not os.path.exists(os.path.abspath(os.path.dirname(backup_path))): + if not isinstance(backup_path, six.string_types) or not os.path.exists( + os.path.abspath(os.path.dirname(backup_path)) + ): args.append(backup_path) backup_path = None if args: args = reversed(args) - self.path = path - self.backup_path = backup_path - self.show(self.path, self.backup_path) - OSError.__init__(self, path, *args, **kwargs) + self.message = self.get_message(path, backup_path=backup_path) + super(FileCorruptException, self).__init__(self.message) - @classmethod - def show(cls, path, backup_path=None): - print("ERROR: Failed to load file at %s" % path, file=sys.stderr, flush=True) + def get_message(self, path, backup_path=None): + message = "ERROR: Failed to load file at %s" % path if backup_path: msg = "it will be backed up to %s and removed" % backup_path else: - msg = "it will be removed and replaced." - print("The file is corrupt, %s" % msg, file=sys.stderr, flush=True) + msg = "it will be removed and replaced on the next lock." + message = "{0}\nYour lockfile is corrupt, {1}".format(message, msg) + return message + + def show(self): + print(self.message, file=sys.stderr, flush=True) class LockfileCorruptException(FileCorruptException): + def __init__(self, path, backup_path=None): + self.message = self.get_message(path, backup_path=backup_path) + super(LockfileCorruptException, self).__init__(self.message) - @classmethod - def show(cls, path, backup_path=None): - print("ERROR: Failed to load lockfile at %s" % path, file=sys.stderr, flush=True) + def get_message(self, path, backup_path=None): + message = "ERROR: Failed to load lockfile at %s" % path if backup_path: msg = "it will be backed up to %s and removed" % backup_path else: msg = "it will be removed and replaced on the next lock." - print("Your lockfile is corrupt, %s" % msg, file=sys.stderr, flush=True) + message = "{0}\nYour lockfile is corrupt, {1}".format(message, msg) + return message + + def show(self, path, backup_path=None): + print(self.message, file=sys.stderr, flush=True) class PipfileCorruptException(FileCorruptException): + def __init__(self, path, backup_path=None): + self.message = self.get_message(path, backup_path=backup_path) + super(PipfileCorruptException, self).__init__(self.message) - @classmethod - def show(cls, path, backup_path=None): - print("ERROR: Failed to load Pipfile at %s" % path, file=sys.stderr, flush=True) + def get_message(self, path, backup_path=None): + message = "ERROR: Failed to load Pipfile at %s" % path if backup_path: msg = "it will be backed up to %s and removed" % backup_path else: msg = "it will be removed and replaced on the next lock." - print("Your Pipfile is corrupt, %s" % msg, file=sys.stderr, flush=True) + message = "{0}\nYour Pipfile is corrupt, {1}".format(message, msg) + return message + + def show(self, path, backup_path=None): + print(self.message, file=sys.stderr, flush=True) class PipfileNotFound(FileNotFoundError): def __init__(self, path, *args, **kwargs): self.errno = errno.ENOENT - self.path = path - self.show(path) - super(PipfileNotFound, self).__init__(*args, **kwargs) - - @classmethod - def show(cls, path): - print("ERROR: The file could not be found: %s" % path, file=sys.stderr, flush=True) - print("Aborting...", file=sys.stderr, flush=True) + self.filename = path + super(PipfileNotFound, self).__init__(self.filename) diff --git a/pipenv/vendor/requirementslib/models/dependencies.py b/pipenv/vendor/requirementslib/models/dependencies.py index f87fd585..82eaba5f 100644 --- a/pipenv/vendor/requirementslib/models/dependencies.py +++ b/pipenv/vendor/requirementslib/models/dependencies.py @@ -9,24 +9,58 @@ import os import attr import packaging.markers import packaging.version +import pip_shims.shims import requests - from first import first from packaging.utils import canonicalize_name - -import pip_shims.shims -from vistir.compat import JSONDecodeError, fs_str, ResourceWarning +from vistir.compat import JSONDecodeError, fs_str from vistir.contextmanagers import cd, temp_environ from vistir.misc import partialclass from vistir.path import create_tracked_tempdir -from ..utils import prepare_pip_source_args, _ensure_dir from .cache import CACHE_DIR, DependencyCache from .utils import ( - clean_requires_python, fix_requires_python_marker, format_requirement, - full_groupby, is_pinned_requirement, key_from_ireq, - make_install_requirement, name_from_req, version_from_ireq + clean_requires_python, + fix_requires_python_marker, + format_requirement, + full_groupby, + is_pinned_requirement, + key_from_ireq, + make_install_requirement, + name_from_req, + version_from_ireq, ) +from ..environment import MYPY_RUNNING +from ..utils import _ensure_dir, prepare_pip_source_args + +if MYPY_RUNNING: + from typing import ( + Any, + Dict, + List, + Generator, + Optional, + Union, + Tuple, + TypeVar, + Text, + Set, + ) + from pip_shims.shims import ( + InstallRequirement, + InstallationCandidate, + PackageFinder, + Command, + ) + from packaging.requirements import Requirement as PackagingRequirement + + TRequirement = TypeVar("TRequirement") + RequirementType = TypeVar( + "RequirementType", covariant=True, bound=PackagingRequirement + ) + MarkerType = TypeVar("MarkerType", covariant=True, bound=Marker) + STRING_TYPE = Union[str, bytes, Text] + S = TypeVar("S", bytes, str, Text) PKGS_DOWNLOAD_DIR = fs_str(os.path.join(CACHE_DIR, "pkgs")) @@ -43,6 +77,7 @@ def _get_filtered_versions(ireq, versions, prereleases): def find_all_matches(finder, ireq, pre=False): + # type: (PackageFinder, InstallRequirement, bool) -> List[InstallationCandidate] """Find all matching dependencies using the supplied finder and the given ireq. @@ -54,7 +89,6 @@ def find_all_matches(finder, ireq, pre=False): :rtype: list[:class:`~pip._internal.index.InstallationCandidate`] """ - candidates = clean_requires_python(finder.find_all_candidates(ireq.name)) versions = {candidate.version for candidate in candidates} allowed_versions = _get_filtered_versions(ireq, versions, pre) @@ -65,6 +99,7 @@ def find_all_matches(finder, ireq, pre=False): def get_pip_command(): + # type: () -> Command # Use pip's parser for pip.conf management and defaults. # General options (find_links, index_url, extra_index_url, trusted_host, # and pre) are defered to pip. @@ -89,7 +124,7 @@ def get_pip_command(): @attr.s class AbstractDependency(object): - name = attr.ib() + name = attr.ib() # type: STRING_TYPE specifiers = attr.ib() markers = attr.ib() candidates = attr.ib() @@ -144,10 +179,14 @@ class AbstractDependency(object): elif len(other.candidates) == 1 and first(other.candidates).editable: return other new_specifiers = self.specifiers & other.specifiers - markers = set(self.markers,) if self.markers else set() + markers = set(self.markers) if self.markers else set() if other.markers: markers.add(other.markers) - new_markers = packaging.markers.Marker(" or ".join(str(m) for m in sorted(markers))) + new_markers = None + if markers: + new_markers = packaging.markers.Marker( + " or ".join(str(m) for m in sorted(markers)) + ) new_ireq = copy.deepcopy(self.requirement.ireq) new_ireq.req.specifier = new_specifiers new_ireq.req.marker = new_markers @@ -173,7 +212,7 @@ class AbstractDependency(object): requirement=new_requirement, parent=self.parent, dep_dict=dep_dict, - finder=self.finder + finder=self.finder, ) def get_deps(self, candidate): @@ -190,7 +229,7 @@ class AbstractDependency(object): from .requirements import Requirement req = Requirement.from_line(key) - req.merge_markers(self.markers) + req = req.merge_markers(self.markers) self.dep_dict[key] = req.get_abstract_dependencies() return self.dep_dict[key] @@ -216,13 +255,18 @@ class AbstractDependency(object): if not is_pinned and not requirement.editable: for r in requirement.find_all_matches(finder=finder): req = make_install_requirement( - name, r.version, extras=extras, markers=markers, constraint=is_constraint, + name, + r.version, + extras=extras, + markers=markers, + constraint=is_constraint, ) req.req.link = r.location req.parent = parent candidates.append(req) candidates = sorted( - set(candidates), key=lambda k: packaging.version.parse(version_from_ireq(k)), + set(candidates), + key=lambda k: packaging.version.parse(version_from_ireq(k)), ) else: candidates = [requirement.ireq] @@ -265,9 +309,7 @@ def get_abstract_dependencies(reqs, sources=None, parent=None): for req in reqs: if isinstance(req, pip_shims.shims.InstallRequirement): - requirement = Requirement.from_line( - "{0}{1}".format(req.name, req.specifier) - ) + requirement = Requirement.from_line("{0}{1}".format(req.name, req.specifier)) if req.link: requirement.req.link = req.link requirement.markers = req.markers @@ -284,6 +326,7 @@ def get_abstract_dependencies(reqs, sources=None, parent=None): def get_dependencies(ireq, sources=None, parent=None): + # type: (Union[InstallRequirement, InstallationCandidate], Optional[List[Dict[S, Union[S, bool]]]], Optional[AbstractDependency]) -> Set[S, ...] """Get all dependencies for a given install requirement. :param ireq: A single InstallRequirement @@ -296,27 +339,26 @@ def get_dependencies(ireq, sources=None, parent=None): :rtype: set(str) """ if not isinstance(ireq, pip_shims.shims.InstallRequirement): - name = getattr( - ireq, "project_name", - getattr(ireq, "project", ireq.name), - ) + name = getattr(ireq, "project_name", getattr(ireq, "project", ireq.name)) version = getattr(ireq, "version", None) if not version: ireq = pip_shims.shims.InstallRequirement.from_line("{0}".format(name)) else: - ireq = pip_shims.shims.InstallRequirement.from_line("{0}=={1}".format(name, version)) + ireq = pip_shims.shims.InstallRequirement.from_line( + "{0}=={1}".format(name, version) + ) pip_options = get_pip_options(sources=sources) getters = [ get_dependencies_from_cache, get_dependencies_from_wheel_cache, get_dependencies_from_json, - functools.partial(get_dependencies_from_index, pip_options=pip_options) + functools.partial(get_dependencies_from_index, pip_options=pip_options), ] for getter in getters: deps = getter(ireq) if deps is not None: return deps - raise RuntimeError('failed to get dependencies for {}'.format(ireq)) + raise RuntimeError("failed to get dependencies for {}".format(ireq)) def get_dependencies_from_wheel_cache(ireq): @@ -374,7 +416,7 @@ def get_dependencies_from_json(ireq): finally: session.close() requires_dist = info.get("requires_dist", info.get("requires")) - if not requires_dist: # The API can return None for this. + if not requires_dist: # The API can return None for this. return for requires in requires_dist: i = pip_shims.shims.InstallRequirement.from_line(requires) @@ -415,9 +457,9 @@ def get_dependencies_from_cache(ireq): dep_ireq = pip_shims.shims.InstallRequirement.from_line(line) name = canonicalize_name(dep_ireq.name) if _marker_contains_extra(dep_ireq): - broken = True # The "extra =" marker breaks everything. + broken = True # The "extra =" marker breaks everything. elif name == canonicalize_name(ireq.name): - broken = True # A package cannot depend on itself. + broken = True # A package cannot depend on itself. if broken: break except Exception: @@ -431,7 +473,7 @@ def get_dependencies_from_cache(ireq): def is_python(section): - return section.startswith('[') and ':' in section + return section.startswith("[") and ":" in section def get_dependencies_from_index(dep, sources=None, pip_options=None, wheel_cache=None): @@ -453,12 +495,15 @@ def get_dependencies_from_index(dep, sources=None, pip_options=None, wheel_cache reqset.add_requirement(dep) requirements = None setup_requires = {} - with temp_environ(), start_resolver(finder=finder, wheel_cache=wheel_cache) as resolver: - os.environ['PIP_EXISTS_ACTION'] = 'i' + with temp_environ(), start_resolver( + finder=finder, wheel_cache=wheel_cache + ) as resolver: + os.environ["PIP_EXISTS_ACTION"] = "i" dist = None if dep.editable and not dep.prepared and not dep.req: with cd(dep.setup_py_dir): from setuptools.dist import distutils + try: dist = distutils.core.run_setup(dep.setup_py) except (ImportError, TypeError, AttributeError): @@ -489,7 +534,7 @@ def get_dependencies_from_index(dep, sources=None, pip_options=None, wheel_cache add_marker = fix_requires_python_marker(requires_python) reqset.remove(dep) if dep.req.marker: - dep.req.marker._markers.extend(['and',].extend(add_marker._markers)) + dep.req.marker._markers.extend(["and"].extend(add_marker._markers)) else: dep.req.marker = add_marker reqset.add(dep) @@ -497,7 +542,7 @@ def get_dependencies_from_index(dep, sources=None, pip_options=None, wheel_cache for r in results: if requires_python: if r.req.marker: - r.req.marker._markers.extend(['and',].extend(add_marker._markers)) + r.req.marker._markers.extend(["and"].extend(add_marker._markers)) else: r.req.marker = add_marker requirements.add(format_requirement(r)) @@ -516,10 +561,16 @@ def get_dependencies_from_index(dep, sources=None, pip_options=None, wheel_cache else: not_python = True - if ':' not in value and not_python: + if ":" not in value and not_python: try: - requirement_str = "{0}{1}".format(value, python_version).replace(":", ";") - requirements.add(format_requirement(make_install_requirement(requirement_str).ireq)) + requirement_str = "{0}{1}".format(value, python_version).replace( + ":", ";" + ) + requirements.add( + format_requirement( + make_install_requirement(requirement_str).ireq + ) + ) # Anything could go wrong here -- can't be too careful. except Exception: pass @@ -544,9 +595,7 @@ def get_pip_options(args=[], sources=None, pip_command=None): if not pip_command: pip_command = get_pip_command() if not sources: - sources = [ - {"url": "https://pypi.org/simple", "name": "pypi", "verify_ssl": True} - ] + sources = [{"url": "https://pypi.org/simple", "name": "pypi", "verify_ssl": True}] _ensure_dir(CACHE_DIR) pip_args = args pip_args = prepare_pip_source_args(sources, pip_args) @@ -556,6 +605,7 @@ def get_pip_options(args=[], sources=None, pip_command=None): def get_finder(sources=None, pip_command=None, pip_options=None): + # type: (List[Dict[S, Union[S, bool]]], Optional[Command], Any) -> PackageFinder """Get a package finder for looking up candidates to install :param sources: A list of pipfile-formatted sources, defaults to None @@ -571,9 +621,7 @@ def get_finder(sources=None, pip_command=None, pip_options=None): if not pip_command: pip_command = get_pip_command() if not sources: - sources = [ - {"url": "https://pypi.org/simple", "name": "pypi", "verify_ssl": True} - ] + sources = [{"url": "https://pypi.org/simple", "name": "pypi", "verify_ssl": True}] if not pip_options: pip_options = get_pip_options(sources=sources, pip_command=pip_command) session = pip_command._build_session(pip_options) @@ -636,7 +684,9 @@ def start_resolver(finder=None, wheel_cache=None): use_user_site=False, ) try: - if packaging.version.parse(pip_shims.shims.pip_version) >= packaging.version.parse('18'): + if packaging.version.parse( + pip_shims.shims.pip_version + ) >= packaging.version.parse("18"): with pip_shims.shims.RequirementTracker() as req_tracker: preparer = preparer(req_tracker=req_tracker) yield resolver(preparer=preparer) diff --git a/pipenv/vendor/requirementslib/models/lockfile.py b/pipenv/vendor/requirementslib/models/lockfile.py index 54b2761c..42248868 100644 --- a/pipenv/vendor/requirementslib/models/lockfile.py +++ b/pipenv/vendor/requirementslib/models/lockfile.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import +from __future__ import absolute_import, print_function import copy import os diff --git a/pipenv/vendor/requirementslib/models/markers.py b/pipenv/vendor/requirementslib/models/markers.py index 70fe3bc0..fc85fbdd 100644 --- a/pipenv/vendor/requirementslib/models/markers.py +++ b/pipenv/vendor/requirementslib/models/markers.py @@ -1,19 +1,44 @@ # -*- coding: utf-8 -*- +import itertools +import operator + import attr - +import distlib.markers +import packaging.version +import six from packaging.markers import InvalidMarker, Marker +from packaging.specifiers import Specifier, SpecifierSet +from vistir.compat import Mapping, Set, lru_cache +from vistir.misc import dedup -from ..exceptions import RequirementError from .utils import filter_none, validate_markers +from ..environment import MYPY_RUNNING +from ..exceptions import RequirementError + +from six.moves import reduce # isort:skip + + +if MYPY_RUNNING: + from typing import Optional, List, Type, Any, Tuple, Union, AnyStr, Text, Iterator + + STRING_TYPE = Union[str, bytes, Text] + + +MAX_VERSIONS = {2: 7, 3: 10} + + +def is_instance(item, cls): + # type: (Any, Type) -> bool + if isinstance(item, cls) or item.__class__.__name__ == cls.__name__: + return True + return False @attr.s class PipenvMarkers(object): """System-level requirements - see PEP508 for more detail""" - os_name = attr.ib( - default=None, validator=attr.validators.optional(validate_markers) - ) + os_name = attr.ib(default=None, validator=attr.validators.optional(validate_markers)) sys_platform = attr.ib( default=None, validator=attr.validators.optional(validate_markers) ) @@ -92,3 +117,556 @@ class PipenvMarkers(object): pass else: return combined_marker + + +@lru_cache(maxsize=1024) +def _tuplize_version(version): + # type: (STRING_TYPE) -> Tuple[int, ...] + return tuple(int(x) for x in filter(lambda i: i != "*", version.split("."))) + + +@lru_cache(maxsize=1024) +def _format_version(version): + # type: (Tuple[int, ...]) -> STRING_TYPE + if not isinstance(version, six.string_types): + return ".".join(str(i) for i in version) + return version + + +# Prefer [x,y) ranges. +REPLACE_RANGES = {">": ">=", "<=": "<"} + + +@lru_cache(maxsize=1024) +def _format_pyspec(specifier): + # type: (Union[STRING_TYPE, Specifier]) -> Specifier + if isinstance(specifier, str): + if not any(op in specifier for op in Specifier._operators.keys()): + specifier = "=={0}".format(specifier) + specifier = Specifier(specifier) + version = getattr(specifier, "version", specifier).rstrip() + if version and version.endswith("*"): + if version.endswith(".*"): + version = version.rstrip(".*") + else: + version = version.rstrip("*") + specifier = Specifier("{0}{1}".format(specifier.operator, version)) + try: + op = REPLACE_RANGES[specifier.operator] + except KeyError: + return specifier + curr_tuple = _tuplize_version(version) + try: + next_tuple = (curr_tuple[0], curr_tuple[1] + 1) + except IndexError: + next_tuple = (curr_tuple[0], 1) + if not next_tuple[1] <= MAX_VERSIONS[next_tuple[0]]: + if specifier.operator == "<" and curr_tuple[1] <= MAX_VERSIONS[next_tuple[0]]: + op = "<=" + next_tuple = (next_tuple[0], curr_tuple[1]) + else: + return specifier + specifier = Specifier("{0}{1}".format(op, _format_version(next_tuple))) + return specifier + + +@lru_cache(maxsize=1024) +def _get_specs(specset): + if specset is None: + return + if is_instance(specset, Specifier): + new_specset = SpecifierSet() + specs = set() + specs.add(specset) + new_specset._specs = frozenset(specs) + specset = new_specset + if isinstance(specset, str): + specset = SpecifierSet(specset) + result = [] + for spec in set(specset): + version = spec.version + op = spec.operator + if op in ("in", "not in"): + versions = version.split(",") + op = "==" if op == "in" else "!=" + for ver in versions: + result.append((op, _tuplize_version(ver.strip()))) + else: + result.append((spec.operator, _tuplize_version(spec.version))) + return sorted(result, key=operator.itemgetter(1)) + + +def _group_by_op(specs): + # type: (Union[Set[Specifier], SpecifierSet]) -> Iterator + specs = [_get_specs(x) for x in list(specs)] + flattened = [(op, version) for spec in specs for op, version in spec] + specs = sorted(flattened) + grouping = itertools.groupby(specs, key=operator.itemgetter(0)) + return grouping + + +def normalize_specifier_set(specs): + # type: (Union[str, SpecifierSet]) -> Optional[Set[Specifier]] + """Given a specifier set, a string, or an iterable, normalize the specifiers + + .. note:: This function exists largely to deal with ``pyzmq`` which handles + the ``requires_python`` specifier incorrectly, using ``3.7*`` rather than + the correct form of ``3.7.*``. This workaround can likely go away if + we ever introduce enforcement for metadata standards on PyPI. + + :param Union[str, SpecifierSet] specs: Supplied specifiers to normalize + :return: A new set of specifiers or specifierset + :rtype: Union[Set[Specifier], :class:`~packaging.specifiers.SpecifierSet`] + """ + if not specs: + return None + if isinstance(specs, set): + return specs + # when we aren't dealing with a string at all, we can normalize this as usual + elif not isinstance(specs, six.string_types): + return {_format_pyspec(spec) for spec in specs} + spec_list = [] + for spec in specs.split(","): + if spec.endswith(".*"): + spec = spec.rstrip(".*") + elif spec.endswith("*"): + spec = spec.rstrip("*") + spec_list.append(spec) + return normalize_specifier_set(SpecifierSet(",".join(spec_list))) + + +def get_sorted_version_string(version_set): + # type: (Set[AnyStr]) -> AnyStr + version_list = sorted( + "{0}".format(_format_version(version)) for version in version_set + ) + version = ", ".join(version_list) + return version + + +@lru_cache(maxsize=1024) +def cleanup_pyspecs(specs, joiner="or"): + specs = normalize_specifier_set(specs) + # for != operator we want to group by version + # if all are consecutive, join as a list + results = {} + translation_map = { + # if we are doing an or operation, we need to use the min for >= + # this way OR(>=2.6, >=2.7, >=3.6) picks >=2.6 + # if we do an AND operation we need to use MAX to be more selective + (">", ">="): { + "or": lambda x: _format_version(min(x)), + "and": lambda x: _format_version(max(x)), + }, + # we use inverse logic here so we will take the max value if we are + # using OR but the min value if we are using AND + ("<", "<="): { + "or": lambda x: _format_version(max(x)), + "and": lambda x: _format_version(min(x)), + }, + # leave these the same no matter what operator we use + ("!=", "==", "~=", "==="): { + "or": get_sorted_version_string, + "and": get_sorted_version_string, + }, + } + op_translations = { + "!=": lambda x: "not in" if len(x) > 1 else "!=", + "==": lambda x: "in" if len(x) > 1 else "==", + } + translation_keys = list(translation_map.keys()) + for op, versions in _group_by_op(tuple(specs)): + versions = [version[1] for version in versions] + versions = sorted(dedup(versions)) + op_key = next(iter(k for k in translation_keys if op in k), None) + version_value = versions + if op_key is not None: + version_value = translation_map[op_key][joiner](versions) + if op in op_translations: + op = op_translations[op](versions) + results[op] = version_value + return sorted([(k, v) for k, v in results.items()], key=operator.itemgetter(1)) + + +@lru_cache(maxsize=1024) +def fix_version_tuple(version_tuple): + # type: (Tuple[AnyStr, AnyStr]) -> Tuple[AnyStr, AnyStr] + op, version = version_tuple + max_major = max(MAX_VERSIONS.keys()) + if version[0] > max_major: + return (op, (max_major, MAX_VERSIONS[max_major])) + max_allowed = MAX_VERSIONS[version[0]] + if op == "<" and version[1] > max_allowed and version[1] - 1 <= max_allowed: + op = "<=" + version = (version[0], version[1] - 1) + return (op, version) + + +@lru_cache(maxsize=128) +def get_versions(specset, group_by_operator=True): + # type: (Union[Set[Specifier], SpecifierSet], bool) -> List[Tuple[STRING_TYPE, STRING_TYPE]] + specs = [_get_specs(x) for x in list(tuple(specset))] + initial_sort_key = lambda k: (k[0], k[1]) + initial_grouping_key = operator.itemgetter(0) + if not group_by_operator: + initial_grouping_key = operator.itemgetter(1) + initial_sort_key = operator.itemgetter(1) + version_tuples = sorted( + set((op, version) for spec in specs for op, version in spec), key=initial_sort_key + ) + version_tuples = [fix_version_tuple(t) for t in version_tuples] + op_groups = [ + (grp, list(map(operator.itemgetter(1), keys))) + for grp, keys in itertools.groupby(version_tuples, key=initial_grouping_key) + ] + versions = [ + (op, packaging.version.parse(".".join(str(v) for v in val))) + for op, vals in op_groups + for val in vals + ] + return sorted(versions, key=operator.itemgetter(1)) + + +def _ensure_marker(marker): + # type: (Union[STRING_TYPE, Marker]) -> Marker + if not is_instance(marker, Marker): + return Marker(str(marker)) + return marker + + +def gen_marker(mkr): + # type: (List[STRING_TYPE]) -> Marker + m = Marker("python_version == '1'") + m._markers.pop() + m._markers.append(mkr) + return m + + +def _strip_extra(elements): + """Remove the "extra == ..." operands from the list.""" + + return _strip_marker_elem("extra", elements) + + +def _strip_pyversion(elements): + return _strip_marker_elem("python_version", elements) + + +def _strip_marker_elem(elem_name, elements): + """Remove the supplied element from the marker. + + This is not a comprehensive implementation, but relies on an important + characteristic of metadata generation: The element's operand is always + associated with an "and" operator. This means that we can simply remove the + operand and the "and" operator associated with it. + """ + + extra_indexes = [] + preceding_operators = ["and"] if elem_name == "extra" else ["and", "or"] + for i, element in enumerate(elements): + if isinstance(element, list): + cancelled = _strip_marker_elem(elem_name, element) + if cancelled: + extra_indexes.append(i) + elif isinstance(element, tuple) and element[0].value == elem_name: + extra_indexes.append(i) + for i in reversed(extra_indexes): + del elements[i] + if i > 0 and elements[i - 1] in preceding_operators: + # Remove the "and" before it. + del elements[i - 1] + elif elements: + # This shouldn't ever happen, but is included for completeness. + # If there is not an "and" before this element, try to remove the + # operator after it. + del elements[0] + return not elements + + +def _get_stripped_marker(marker, strip_func): + """Build a new marker which is cleaned according to `strip_func`""" + + if not marker: + return None + marker = _ensure_marker(marker) + elements = marker._markers + strip_func(elements) + if elements: + return marker + return None + + +def get_without_extra(marker): + """Build a new marker without the `extra == ...` part. + + The implementation relies very deep into packaging's internals, but I don't + have a better way now (except implementing the whole thing myself). + + This could return `None` if the `extra == ...` part is the only one in the + input marker. + """ + + return _get_stripped_marker(marker, _strip_extra) + + +def get_without_pyversion(marker): + """Built a new marker without the `python_version` part. + + This could return `None` if the `python_version` section is the only section in the + marker. + """ + + return _get_stripped_marker(marker, _strip_pyversion) + + +def _markers_collect_extras(markers, collection): + # Optimization: the marker element is usually appended at the end. + for el in reversed(markers): + if isinstance(el, tuple) and el[0].value == "extra" and el[1].value == "==": + collection.add(el[2].value) + elif isinstance(el, list): + _markers_collect_extras(el, collection) + + +def _markers_collect_pyversions(markers, collection): + local_collection = [] + marker_format_str = "{0}" + for i, el in enumerate(reversed(markers)): + if isinstance(el, tuple) and el[0].value == "python_version": + new_marker = str(gen_marker(el)) + local_collection.append(marker_format_str.format(new_marker)) + elif isinstance(el, list): + _markers_collect_pyversions(el, local_collection) + if local_collection: + # local_collection = "{0}".format(" ".join(local_collection)) + collection.extend(local_collection) + + +def _markers_contains_extra(markers): + # Optimization: the marker element is usually appended at the end. + return _markers_contains_key(markers, "extra") + + +def _markers_contains_pyversion(markers): + return _markers_contains_key(markers, "python_version") + + +def _markers_contains_key(markers, key): + for element in reversed(markers): + if isinstance(element, tuple) and element[0].value == key: + return True + elif isinstance(element, list): + if _markers_contains_key(element, key): + return True + return False + + +@lru_cache(maxsize=128) +def get_contained_extras(marker): + """Collect "extra == ..." operands from a marker. + + Returns a list of str. Each str is a speficied extra in this marker. + """ + if not marker: + return set() + extras = set() + marker = _ensure_marker(marker) + _markers_collect_extras(marker._markers, extras) + return extras + + +@lru_cache(maxsize=1024) +def get_contained_pyversions(marker): + """Collect all `python_version` operands from a marker. + """ + + collection = [] + if not marker: + return set() + marker = _ensure_marker(marker) + # Collect the (Variable, Op, Value) tuples and string joiners from the marker + _markers_collect_pyversions(marker._markers, collection) + marker_str = " and ".join(sorted(collection)) + if not marker_str: + return set() + # Use the distlib dictionary parser to create a dictionary 'trie' which is a bit + # easier to reason about + marker_dict = distlib.markers.parse_marker(marker_str)[0] + version_set = set() + pyversions, _ = parse_marker_dict(marker_dict) + if isinstance(pyversions, set): + version_set.update(pyversions) + elif pyversions is not None: + version_set.add(pyversions) + # Each distinct element in the set was separated by an "and" operator in the marker + # So we will need to reduce them with an intersection here rather than a union + # in order to find the boundaries + versions = set() + if version_set: + versions = reduce(lambda x, y: x & y, version_set) + return versions + + +@lru_cache(maxsize=128) +def contains_extra(marker): + """Check whehter a marker contains an "extra == ..." operand. + """ + if not marker: + return False + marker = _ensure_marker(marker) + return _markers_contains_extra(marker._markers) + + +@lru_cache(maxsize=128) +def contains_pyversion(marker): + """Check whether a marker contains a python_version operand. + """ + + if not marker: + return False + marker = _ensure_marker(marker) + return _markers_contains_pyversion(marker._markers) + + +def get_specset(marker_list): + # type: (List) -> Optional[SpecifierSet] + specset = set() + _last_str = "and" + for marker_parts in marker_list: + if isinstance(marker_parts, tuple): + variable, op, value = marker_parts + if variable.value != "python_version": + continue + if op.value == "in": + values = [v.strip() for v in value.value.split(",")] + specset.update(Specifier("=={0}".format(v)) for v in values) + elif op.value == "not in": + values = [v.strip() for v in value.value.split(",")] + bad_versions = ["3.0", "3.1", "3.2", "3.3"] + if len(values) >= 2 and any(v in values for v in bad_versions): + values = bad_versions + specset.update( + Specifier("!={0}".format(v.strip())) for v in sorted(bad_versions) + ) + else: + specset.add(Specifier("{0}{1}".format(op.value, value.value))) + elif isinstance(marker_parts, list): + parts = get_specset(marker_parts) + if parts: + specset.update(parts) + elif isinstance(marker_parts, str): + _last_str = marker_parts + specifiers = SpecifierSet() + specifiers._specs = frozenset(specset) + return specifiers + + +def parse_marker_dict(marker_dict): + op = marker_dict["op"] + lhs = marker_dict["lhs"] + rhs = marker_dict["rhs"] + # This is where the spec sets for each side land if we have an "or" operator + side_spec_list = [] + side_markers_list = [] + finalized_marker = "" + # And if we hit the end of the parse tree we use this format string to make a marker + format_string = "{lhs} {op} {rhs}" + specset = SpecifierSet() + specs = set() + # Essentially we will iterate over each side of the parsed marker if either one is + # A mapping instance (i.e. a dictionary) and recursively parse and reduce the specset + # Union the "and" specs, intersect the "or"s to find the most appropriate range + if any(issubclass(type(side), Mapping) for side in (lhs, rhs)): + for side in (lhs, rhs): + side_specs = set() + side_markers = set() + if issubclass(type(side), Mapping): + merged_side_specs, merged_side_markers = parse_marker_dict(side) + side_specs.update(merged_side_specs) + side_markers.update(merged_side_markers) + else: + marker = _ensure_marker(side) + marker_parts = getattr(marker, "_markers", []) + if marker_parts[0][0].value == "python_version": + side_specs |= set(get_specset(marker_parts)) + else: + side_markers.add(str(marker)) + side_spec_list.append(side_specs) + side_markers_list.append(side_markers) + if op == "and": + # When we are "and"-ing things together, it probably makes the most sense + # to reduce them here into a single PySpec instance + specs = reduce(lambda x, y: set(x) | set(y), side_spec_list) + markers = reduce(lambda x, y: set(x) | set(y), side_markers_list) + if not specs and not markers: + return specset, finalized_marker + if markers and isinstance(markers, (tuple, list, Set)): + finalized_marker = Marker(" and ".join([m for m in markers if m])) + elif markers: + finalized_marker = str(markers) + specset._specs = frozenset(specs) + return specset, finalized_marker + # Actually when we "or" things as well we can also just turn them into a reduced + # set using this logic now + sides = reduce(lambda x, y: set(x) & set(y), side_spec_list) + finalized_marker = " or ".join( + [normalize_marker_str(m) for m in side_markers_list] + ) + specset._specs = frozenset(sorted(sides)) + return specset, finalized_marker + else: + # At the tip of the tree we are dealing with strings all around and they just need + # to be smashed together + specs = set() + if lhs == "python_version": + format_string = "{lhs}{op}{rhs}" + marker = Marker(format_string.format(**marker_dict)) + marker_parts = getattr(marker, "_markers", []) + _set = get_specset(marker_parts) + if _set: + specs |= set(_set) + specset._specs = frozenset(specs) + return specset, finalized_marker + + +def format_pyversion(parts): + op, val = parts + return "python_version {0} '{1}'".format(op, val) + + +def normalize_marker_str(marker): + # type: (Union[Marker, STRING_TYPE]) + marker_str = "" + if not marker: + return None + if not is_instance(marker, Marker): + marker = _ensure_marker(marker) + pyversion = get_contained_pyversions(marker) + marker = get_without_pyversion(marker) + if pyversion: + parts = cleanup_pyspecs(pyversion) + marker_str = " and ".join([format_pyversion(pv) for pv in parts]) + if marker: + if marker_str: + marker_str = "{0!s} and {1!s}".format(marker_str, marker) + else: + marker_str = "{0!s}".format(marker) + return marker_str.replace('"', "'") + + +@lru_cache(maxsize=1024) +def marker_from_specifier(spec): + # type: (STRING_TYPE) -> Marker + if not any(spec.startswith(k) for k in Specifier._operators.keys()): + if spec.strip().lower() in ["any", "", "*"]: + return None + spec = "=={0}".format(spec) + elif spec.startswith("==") and spec.count("=") > 3: + spec = "=={0}".format(spec.lstrip("=")) + if not spec: + return None + marker_segments = [] + for marker_segment in cleanup_pyspecs(spec): + marker_segments.append(format_pyversion(marker_segment)) + marker_str = " and ".join(marker_segments).replace('"', "'") + return Marker(marker_str) diff --git a/pipenv/vendor/requirementslib/models/pipfile.py b/pipenv/vendor/requirementslib/models/pipfile.py index e3d353d9..e55ad741 100644 --- a/pipenv/vendor/requirementslib/models/pipfile.py +++ b/pipenv/vendor/requirementslib/models/pipfile.py @@ -1,21 +1,79 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import, unicode_literals, print_function +from __future__ import absolute_import, print_function, unicode_literals -import attr import copy import os +import sys -import tomlkit - -from vistir.compat import Path, FileNotFoundError - -from .requirements import Requirement -from .project import ProjectFile -from .utils import optional_instance_of -from ..exceptions import RequirementError -from ..utils import is_vcs, is_editable, merge_items +import attr +import plette.models.base import plette.pipfiles +import tomlkit +from vistir.compat import FileNotFoundError, Path + +from .project import ProjectFile +from .requirements import Requirement +from .utils import get_url_name, optional_instance_of, tomlkit_value_to_python +from ..environment import MYPY_RUNNING +from ..exceptions import RequirementError +from ..utils import is_editable, is_vcs, merge_items + +if MYPY_RUNNING: + from typing import Union, Any, Dict, Iterable, Mapping, List, Text + + package_type = Dict[Text, Dict[Text, Union[List[Text], Text]]] + source_type = Dict[Text, Union[Text, bool]] + sources_type = Iterable[source_type] + meta_type = Dict[Text, Union[int, Dict[Text, Text], sources_type]] + lockfile_type = Dict[Text, Union[package_type, meta_type]] + + +# Let's start by patching plette to make sure we can validate data without being broken +try: + import cerberus +except ImportError: + cerberus = None + +VALIDATORS = plette.models.base.VALIDATORS + + +def patch_plette(): + # type: () -> None + + global VALIDATORS + + def validate(cls, data): + # type: (Any, Dict[Text, Any]) -> None + if not cerberus: # Skip validation if Cerberus is not available. + return + schema = cls.__SCHEMA__ + key = id(schema) + try: + v = VALIDATORS[key] + except KeyError: + v = VALIDATORS[key] = cerberus.Validator(schema, allow_unknown=True) + if v.validate(dict(data), normalize=False): + return + raise plette.models.base.ValidationError(data, v) + + names = ["plette.models.base", plette.models.base.__name__] + names = [name for name in names if name in sys.modules] + for name in names: + if name in sys.modules: + module = sys.modules[name] + else: + module = plette.models.base + original_fn = getattr(module, "validate") + for key in ["__qualname__", "__name__", "__module__"]: + original_val = getattr(original_fn, key, None) + if original_val is not None: + setattr(validate, key, original_val) + setattr(module, "validate", validate) + sys.modules[name] = module + + +patch_plette() is_pipfile = optional_instance_of(plette.pipfiles.Pipfile) @@ -24,11 +82,14 @@ is_projectfile = optional_instance_of(ProjectFile) def reorder_source_keys(data): - for i, entry in enumerate(data["source"]): - table = tomlkit.table() - table["name"] = entry["name"] - table["url"] = entry["url"] - table["verify_ssl"] = entry["verify_ssl"] + # type: ignore + sources = data["source"] # type: sources_type + for i, entry in enumerate(sources): + table = tomlkit.table() # type: Mapping + source_entry = PipfileLoader.populate_source(entry.copy()) + table["name"] = source_entry["name"] + table["url"] = source_entry["url"] + table["verify_ssl"] = source_entry["verify_ssl"] data["source"][i] = table return data @@ -36,6 +97,7 @@ def reorder_source_keys(data): class PipfileLoader(plette.pipfiles.Pipfile): @classmethod def validate(cls, data): + # type: (Dict[Text, Any]) -> None for key, klass in plette.pipfiles.PIPFILE_SECTIONS.items(): if key not in data or key == "source": continue @@ -44,8 +106,21 @@ class PipfileLoader(plette.pipfiles.Pipfile): except Exception: pass + @classmethod + def populate_source(cls, source): + """Derive missing values of source from the existing fields.""" + # Only URL pararemter is mandatory, let the KeyError be thrown. + if "name" not in source: + source["name"] = get_url_name(source["url"]) + if "verify_ssl" not in source: + source["verify_ssl"] = "https://" in source["url"] + if not isinstance(source["verify_ssl"], bool): + source["verify_ssl"] = source["verify_ssl"].lower() == "true" + return source + @classmethod def load(cls, f, encoding=None): + # type: (Any, Text) -> PipfileLoader content = f.read() if encoding is not None: content = content.decode(encoding) @@ -69,6 +144,7 @@ class PipfileLoader(plette.pipfiles.Pipfile): return instance def __getattribute__(self, key): + # type: (Text) -> Any if key == "source": return self._data[key] return super(PipfileLoader, self).__getattribute__(key) @@ -78,46 +154,58 @@ class PipfileLoader(plette.pipfiles.Pipfile): class Pipfile(object): path = attr.ib(validator=is_path, type=Path) projectfile = attr.ib(validator=is_projectfile, type=ProjectFile) - _pipfile = attr.ib(type=plette.pipfiles.Pipfile) - _pyproject = attr.ib(default=attr.Factory(tomlkit.document), type=tomlkit.toml_document.TOMLDocument) + _pipfile = attr.ib(type=PipfileLoader) + _pyproject = attr.ib( + default=attr.Factory(tomlkit.document), type=tomlkit.toml_document.TOMLDocument + ) build_system = attr.ib(default=attr.Factory(dict), type=dict) - requirements = attr.ib(default=attr.Factory(list), type=list) - dev_requirements = attr.ib(default=attr.Factory(list), type=list) + _requirements = attr.ib(default=attr.Factory(list), type=list) + _dev_requirements = attr.ib(default=attr.Factory(list), type=list) @path.default def _get_path(self): + # type: () -> Path return Path(os.curdir).absolute() @projectfile.default def _get_projectfile(self): + # type: () -> ProjectFile return self.load_projectfile(os.curdir, create=False) @_pipfile.default def _get_pipfile(self): + # type: () -> Union[plette.pipfiles.Pipfile, PipfileLoader] return self.projectfile.model @property def pipfile(self): + # type: () -> Union[PipfileLoader, plette.pipfiles.Pipfile] return self._pipfile def get_deps(self, dev=False, only=True): - deps = {} + # type: (bool, bool) -> Dict[Text, Dict[Text, Union[List[Text], Text]]] + deps = {} # type: Dict[Text, Dict[Text, Union[List[Text], Text]]] if dev: deps.update(self.pipfile._data["dev-packages"]) if only: return deps - return merge_items([deps, self.pipfile._data["packages"]]) + return tomlkit_value_to_python( + merge_items([deps, self.pipfile._data["packages"]]) + ) def get(self, k): + # type: (Text) -> Any return self.__getitem__(k) def __contains__(self, k): + # type: (Text) -> bool check_pipfile = k in self.extended_keys or self.pipfile.__contains__(k) if check_pipfile: return True - return super(Pipfile, self).__contains__(k) + return False def __getitem__(self, k, *args, **kwargs): + # type: ignore retval = None pipfile = self._pipfile section = None @@ -128,6 +216,7 @@ class Pipfile(object): if "-" in k: section, _, pkg_type = k.rpartition("-") vals = getattr(pipfile.get(section, {}), "_data", {}) + vals = tomlkit_value_to_python(vals) if pkg_type == "vcs": retval = {k: v for k, v in vals.items() if is_vcs(v)} elif pkg_type == "editable": @@ -139,6 +228,7 @@ class Pipfile(object): return retval def __getattr__(self, k, *args, **kwargs): + # type: ignore retval = None pipfile = super(Pipfile, self).__getattribute__("_pipfile") try: @@ -151,32 +241,37 @@ class Pipfile(object): @property def requires_python(self): - return self._pipfile.requires.requires_python + # type: () -> bool + return getattr( + self._pipfile.requires, + "python_version", + getattr(self._pipfile.requires, "python_full_version", None), + ) @property def allow_prereleases(self): + # type: () -> bool return self._pipfile.get("pipenv", {}).get("allow_prereleases", False) @classmethod def read_projectfile(cls, path): + # type: (Text) -> ProjectFile """Read the specified project file and provide an interface for writing/updating. - :param str path: Path to the target file. + :param Text path: Path to the target file. :return: A project file with the model and location for interaction :rtype: :class:`~requirementslib.models.project.ProjectFile` """ - pf = ProjectFile.read( - path, - PipfileLoader, - invalid_ok=True - ) + pf = ProjectFile.read(path, PipfileLoader, invalid_ok=True) return pf @classmethod def load_projectfile(cls, path, create=False): - """Given a path, load or create the necessary pipfile. + # type: (Text, bool) -> ProjectFile + """ + Given a path, load or create the necessary pipfile. - :param str path: Path to the project root or pipfile + :param Text path: Path to the project root or pipfile :param bool create: Whether to create the pipfile if not found, defaults to True :raises OSError: Thrown if the project root directory doesn't exist :raises FileNotFoundError: Thrown if the pipfile doesn't exist and ``create=False`` @@ -198,9 +293,11 @@ class Pipfile(object): @classmethod def load(cls, path, create=False): - """Given a path, load or create the necessary pipfile. + # type: (Text, bool) -> Pipfile + """ + Given a path, load or create the necessary pipfile. - :param str path: Path to the project root or pipfile + :param Text path: Path to the project root or pipfile :param bool create: Whether to create the pipfile if not found, defaults to True :raises OSError: Thrown if the project root directory doesn't exist :raises FileNotFoundError: Thrown if the pipfile doesn't exist and ``create=False`` @@ -210,38 +307,54 @@ class Pipfile(object): projectfile = cls.load_projectfile(path, create=create) pipfile = projectfile.model - dev_requirements = [ - Requirement.from_pipfile(k, getattr(v, "_data", v)) for k, v in pipfile.get("dev-packages", {}).items() - ] - requirements = [ - Requirement.from_pipfile(k, getattr(v, "_data", v)) for k, v in pipfile.get("packages", {}).items() - ] creation_args = { "projectfile": projectfile, "pipfile": pipfile, - "dev_requirements": dev_requirements, - "requirements": requirements, - "path": Path(projectfile.location) + "path": Path(projectfile.location), } return cls(**creation_args) def write(self): + # type: () -> None self.projectfile.model = copy.deepcopy(self._pipfile) self.projectfile.write() @property - def dev_packages(self, as_requirements=True): - if as_requirements: - return self.dev_requirements - return self._pipfile.get('dev-packages', {}) + def dev_packages(self): + # type: () -> List[Requirement] + return self.dev_requirements @property - def packages(self, as_requirements=True): - if as_requirements: - return self.requirements - return self._pipfile.get('packages', {}) + def packages(self): + # type: () -> List[Requirement] + return self.requirements + + @property + def dev_requirements(self): + # type: () -> List[Requirement] + if not self._dev_requirements: + packages = tomlkit_value_to_python(self.pipfile.get("dev-packages", {})) + self._dev_requirements = [ + Requirement.from_pipfile(k, v) + for k, v in packages.items() + if v is not None + ] + return self._dev_requirements + + @property + def requirements(self): + # type: () -> List[Requirement] + if not self._requirements: + packages = tomlkit_value_to_python(self.pipfile.get("packages", {})) + self._requirements = [ + Requirement.from_pipfile(k, v) + for k, v in packages.items() + if v is not None + ] + return self._requirements def _read_pyproject(self): + # type: () -> None pyproject = self.path.parent.joinpath("pyproject.toml") if pyproject.exists(): self._pyproject = tomlkit.load(pyproject) @@ -249,15 +362,17 @@ class Pipfile(object): if not os.path.exists(self.path_to("setup.py")): if not build_system or not build_system.get("requires"): build_system = { - "requires": ["setuptools>=38.2.5", "wheel"], - "build-backend": "setuptools.build_meta", + "requires": ["setuptools>=40.8", "wheel"], + "build-backend": "setuptools.build_meta:__legacy__", } self._build_system = build_system @property def build_requires(self): + # type: () -> List[Text] return self.build_system.get("requires", []) @property def build_backend(self): + # type: () -> Text return self.build_system.get("build-backend", None) diff --git a/pipenv/vendor/requirementslib/models/project.py b/pipenv/vendor/requirementslib/models/project.py index f6e037d6..7c1b0e81 100644 --- a/pipenv/vendor/requirementslib/models/project.py +++ b/pipenv/vendor/requirementslib/models/project.py @@ -1,6 +1,6 @@ # -*- coding=utf-8 -*- -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals import collections import io @@ -13,14 +13,10 @@ import plette import plette.models import six import tomlkit +from vistir.compat import FileNotFoundError - -SectionDifference = collections.namedtuple("SectionDifference", [ - "inthis", "inthat", -]) -FileDifference = collections.namedtuple("FileDifference", [ - "default", "develop", -]) +SectionDifference = collections.namedtuple("SectionDifference", ["inthis", "inthat"]) +FileDifference = collections.namedtuple("FileDifference", ["default", "develop"]) def _are_pipfile_entries_equal(a, b): @@ -52,12 +48,15 @@ def preferred_newlines(f): class ProjectFile(object): """A file in the Pipfile project. """ + location = attr.ib() line_ending = attr.ib() model = attr.ib() @classmethod def read(cls, location, model_cls, invalid_ok=False): + if not os.path.exists(location) and not invalid_ok: + raise FileNotFoundError(location) try: with io.open(location, encoding="utf-8") as f: model = model_cls.load(f) @@ -89,14 +88,9 @@ class Project(object): def __attrs_post_init__(self): self.root = root = os.path.abspath(self.root) - self._p = ProjectFile.read( - os.path.join(root, "Pipfile"), - plette.Pipfile, - ) + self._p = ProjectFile.read(os.path.join(root, "Pipfile"), plette.Pipfile) self._l = ProjectFile.read( - os.path.join(root, "Pipfile.lock"), - plette.Lockfile, - invalid_ok=True, + os.path.join(root, "Pipfile.lock"), plette.Lockfile, invalid_ok=True ) @property @@ -138,14 +132,17 @@ class Project(object): self._get_pipfile_section(develop=True, insert=False), ] return any( - (packaging.utils.canonicalize_name(name) == - packaging.utils.canonicalize_name(key)) + ( + packaging.utils.canonicalize_name(name) + == packaging.utils.canonicalize_name(key) + ) for section in sections for name in section ) def add_line_to_pipfile(self, line, develop): from requirementslib import Requirement + requirement = Requirement.from_line(line) section = self._get_pipfile_section(develop=develop) key = requirement.normalized_name @@ -164,13 +161,9 @@ class Project(object): keys = {packaging.utils.canonicalize_name(key) for key in keys} sections = [] if default: - sections.append(self._get_pipfile_section( - develop=False, insert=False, - )) + sections.append(self._get_pipfile_section(develop=False, insert=False)) if develop: - sections.append(self._get_pipfile_section( - develop=True, insert=False, - )) + sections.append(self._get_pipfile_section(develop=True, insert=False)) for section in sections: removals = set() for name in section: diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index d034a12d..dc766619 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -1,84 +1,1289 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import +from __future__ import absolute_import, print_function import collections -import hashlib +import copy import os - +import sys from contextlib import contextmanager +from distutils.sysconfig import get_python_lib +from functools import partial import attr import pip_shims - +import six +import vistir +from cached_property import cached_property from first import first from packaging.markers import Marker from packaging.requirements import Requirement as PackagingRequirement -from packaging.specifiers import Specifier, SpecifierSet, LegacySpecifier, InvalidSpecifier +from packaging.specifiers import ( + InvalidSpecifier, + LegacySpecifier, + Specifier, + SpecifierSet, +) from packaging.utils import canonicalize_name from six.moves.urllib import parse as urllib_parse from six.moves.urllib.parse import unquote -from vistir.compat import FileNotFoundError, Path +from vistir.compat import FileNotFoundError, Mapping, Path, lru_cache +from vistir.contextmanagers import temp_path from vistir.misc import dedup from vistir.path import ( create_tracked_tempdir, get_converted_relative_path, is_file_url, is_valid_url, + mkdir_p, + normalize_path, ) -from ..exceptions import RequirementError -from ..utils import ( - VCS_LIST, - is_installable_file, - is_vcs, - ensure_setup_py, - add_ssh_scheme_to_git_uri, - strip_ssh_from_git_uri, +from .markers import ( + cleanup_pyspecs, + contains_pyversion, + format_pyversion, + get_contained_pyversions, + normalize_marker_str, ) -from .setup_info import SetupInfo +from .setup_info import ( + SetupInfo, + _prepare_wheel_building_kwargs, + ast_parse_setup_py, + get_metadata, + parse_setup_cfg, +) +from .url import URI from .utils import ( + DIRECT_URL_RE, HASH_STRING, - build_vcs_link, + URL_RE, + build_vcs_uri, + convert_direct_url_to_url, + create_link, extras_to_string, filter_none, format_requirement, + get_default_pyproject_backend, + get_pyproject, get_version, init_requirement, is_pinned_requirement, make_install_requirement, + normalize_name, parse_extras, specs_to_string, split_markers_from_line, + split_ref_from_uri, split_vcs_method_from_uri, validate_path, validate_specifiers, validate_vcs, - normalize_name, - create_link, - get_pyproject +) +from ..environment import MYPY_RUNNING +from ..exceptions import RequirementError +from ..utils import ( + VCS_LIST, + add_ssh_scheme_to_git_uri, + get_setup_paths, + is_installable_dir, + is_installable_file, + is_vcs, + strip_ssh_from_git_uri, ) +if MYPY_RUNNING: + from typing import ( + Optional, + TypeVar, + List, + Dict, + Union, + Any, + Tuple, + Sequence, + Set, + AnyStr, + Text, + Generator, + FrozenSet, + ) + from pip_shims.shims import ( + Link, + InstallRequirement, + PackageFinder, + InstallationCandidate, + ) -@attr.s(slots=True) + RequirementType = TypeVar( + "RequirementType", covariant=True, bound=PackagingRequirement + ) + F = TypeVar("F", "FileRequirement", "VCSRequirement", covariant=True) + from six.moves.urllib.parse import SplitResult + from .vcs import VCSRepository + from .dependencies import AbstractDependency + + NON_STRING_ITERABLE = Union[List, Set, Tuple] + STRING_TYPE = Union[str, bytes, Text] + S = TypeVar("S", bytes, str, Text) + BASE_TYPES = Union[bool, STRING_TYPE, Tuple[STRING_TYPE, ...]] + CUSTOM_TYPES = Union[VCSRepository, RequirementType, SetupInfo, "Line"] + CREATION_ARG_TYPES = Union[BASE_TYPES, Link, CUSTOM_TYPES] + PIPFILE_ENTRY_TYPE = Union[STRING_TYPE, bool, Tuple[STRING_TYPE], List[STRING_TYPE]] + PIPFILE_TYPE = Union[STRING_TYPE, Dict[STRING_TYPE, PIPFILE_ENTRY_TYPE]] + TPIPFILE = Dict[STRING_TYPE, PIPFILE_ENTRY_TYPE] + + +SPECIFIERS_BY_LENGTH = sorted(list(Specifier._operators.keys()), key=len, reverse=True) + + +run = partial(vistir.misc.run, combine_stderr=False, return_object=True, nospin=True) + + +class Line(object): + def __init__(self, line, extras=None): + # type: (AnyStr, Optional[Union[List[S], Set[S], Tuple[S, ...]]]) -> None + self.editable = False # type: bool + if line.startswith("-e "): + line = line[len("-e ") :] + self.editable = True + self.extras = () # type: Tuple[STRING_TYPE, ...] + if extras is not None: + self.extras = tuple(sorted(set(extras))) + self.line = line # type: STRING_TYPE + self.hashes = [] # type: List[STRING_TYPE] + self.markers = None # type: Optional[STRING_TYPE] + self.vcs = None # type: Optional[STRING_TYPE] + self.path = None # type: Optional[STRING_TYPE] + self.relpath = None # type: Optional[STRING_TYPE] + self.uri = None # type: Optional[STRING_TYPE] + self._link = None # type: Optional[Link] + self.is_local = False # type: bool + self._name = None # type: Optional[STRING_TYPE] + self._specifier = None # type: Optional[STRING_TYPE] + self.parsed_marker = None # type: Optional[Marker] + self.preferred_scheme = None # type: Optional[STRING_TYPE] + self._requirement = None # type: Optional[PackagingRequirement] + self.is_direct_url = False # type: bool + self._parsed_url = None # type: Optional[urllib_parse.ParseResult] + self._setup_cfg = None # type: Optional[STRING_TYPE] + self._setup_py = None # type: Optional[STRING_TYPE] + self._pyproject_toml = None # type: Optional[STRING_TYPE] + self._pyproject_requires = None # type: Optional[Tuple[STRING_TYPE, ...]] + self._pyproject_backend = None # type: Optional[STRING_TYPE] + self._wheel_kwargs = None # type: Optional[Dict[STRING_TYPE, STRING_TYPE]] + self._vcsrepo = None # type: Optional[VCSRepository] + self._setup_info = None # type: Optional[SetupInfo] + self._ref = None # type: Optional[STRING_TYPE] + self._ireq = None # type: Optional[InstallRequirement] + self._src_root = None # type: Optional[STRING_TYPE] + self.dist = None # type: Any + super(Line, self).__init__() + self.parse() + + def __hash__(self): + return hash( + ( + self.editable, + self.line, + self.markers, + tuple(self.extras), + tuple(self.hashes), + self.vcs, + self.uri, + self.path, + self.name, + self._requirement, + ) + ) + + def __repr__(self): + try: + return ( + "".format( + self=self + ) + ) + except Exception: + return "".format(self.__dict__.values()) + + def __str__(self): + # type: () -> str + if self.markers: + return "{0}; {1}".format(self.get_line(), self.markers) + return self.get_line() + + def get_line( + self, with_prefix=False, with_markers=False, with_hashes=True, as_list=False + ): + # type: (bool, bool, bool, bool) -> Union[STRING_TYPE, List[STRING_TYPE]] + line = self.line + extras_str = extras_to_string(self.extras) + with_hashes = False if self.editable or self.is_vcs else with_hashes + hash_list = ["--hash={0}".format(h) for h in self.hashes] + if self.is_named: + line = self.name_and_specifier + elif self.is_direct_url: + line = self.link.url + elif extras_str: + if self.is_vcs: + line = self.link.url + if "git+file:/" in line and "git+file:///" not in line: + line = line.replace("git+file:/", "git+file:///") + elif extras_str not in line: + line = "{0}{1}".format(line, extras_str) + # XXX: For using markers on vcs or url requirements, they can be used + # as normal (i.e. no space between the requirement and the semicolon) + # and no additional quoting as long as they are not editable requirements + # HOWEVER, for editable requirements, the requirement+marker must be quoted + # We do this here for the line-formatted versions, but leave it up to the + # `Script.parse()` functionality in pipenv, for instance, to handle that + # in a cross-platform manner for the `as_list` approach since that is how + # we anticipate this will be used if passing directly to the command line + # for pip. + if with_markers and self.markers: + line = "{0}; {1}".format(line, self.markers) + if with_prefix and self.editable and not as_list: + line = '"{0}"'.format(line) + if as_list: + result_list = [] + if with_prefix and self.editable: + result_list.append("-e") + result_list.append(line) + if with_hashes: + result_list.extend(self.hashes) + return result_list + if with_prefix and self.editable: + line = "-e {0}".format(line) + if with_hashes and hash_list: + line = "{0} {1}".format(line, " ".join(hash_list)) + return line + + @property + def name_and_specifier(self): + name_str, spec_str = "", "" + if self.name: + name_str = "{0}".format(self.name.lower()) + extras_str = extras_to_string(self.extras) + if extras_str: + name_str = "{0}{1}".format(name_str, extras_str) + if self.specifier: + spec_str = "{0}".format(self.specifier) + return "{0}{1}".format(name_str, spec_str) + + @classmethod + def split_hashes(cls, line): + # type: (S) -> Tuple[S, List[S]] + if "--hash" not in line: + return line, [] + split_line = line.split() + line_parts = [] # type: List[S] + hashes = [] # type: List[S] + for part in split_line: + if part.startswith("--hash"): + param, _, value = part.partition("=") + hashes.append(value) + else: + line_parts.append(part) + line = " ".join(line_parts) + return line, hashes + + @property + def line_with_prefix(self): + # type: () -> STRING_TYPE + return self.get_line(with_prefix=True, with_hashes=False) + + @property + def line_for_ireq(self): + # type: () -> STRING_TYPE + line = "" # type: STRING_TYPE + if self.is_file or self.is_url and not self.is_vcs: + scheme = self.preferred_scheme if self.preferred_scheme is not None else "uri" + local_line = next( + iter( + [ + os.path.dirname(os.path.abspath(f)) + for f in [self.setup_py, self.setup_cfg, self.pyproject_toml] + if f is not None + ] + ), + None, + ) + if local_line and self.extras: + local_line = "{0}{1}".format(local_line, extras_to_string(self.extras)) + line = local_line if local_line is not None else self.line + if scheme == "path": + if not line and self.base_path is not None: + line = os.path.abspath(self.base_path) + else: + if DIRECT_URL_RE.match(self.line): + uri = URI.parse(self.line) + line = uri.full_url + self._requirement = init_requirement(self.line) + line = convert_direct_url_to_url(self.line) + else: + if self.link: + line = self.link.url + else: + try: + uri = URI.parse(line) + except ValueError: + line = line + else: + line = uri.base_url + self._link = uri.as_link + + if self.editable: + if not line: + if self.is_path or self.is_file: + if not self.path: + line = pip_shims.shims.url_to_path(self.url) + else: + line = self.path + if self.extras: + line = "{0}{1}".format(line, extras_to_string(self.extras)) + else: + line = self.link.url + elif self.is_vcs and not self.editable: + line = add_ssh_scheme_to_git_uri(self.line) + if not line: + line = self.line + return line + + @property + def base_path(self): + # type: () -> Optional[S] + if not self.link and not self.path: + self.parse_link() + if not self.path: + pass + path = normalize_path(self.path) + if os.path.exists(path) and os.path.isdir(path): + path = path + elif os.path.exists(path) and os.path.isfile(path): + path = os.path.dirname(path) + else: + path = None + return path + + @property + def setup_py(self): + # type: () -> Optional[STRING_TYPE] + if self._setup_py is None: + self.populate_setup_paths() + return self._setup_py + + @property + def setup_cfg(self): + # type: () -> Optional[STRING_TYPE] + if self._setup_cfg is None: + self.populate_setup_paths() + return self._setup_cfg + + @property + def pyproject_toml(self): + # type: () -> Optional[STRING_TYPE] + if self._pyproject_toml is None: + self.populate_setup_paths() + return self._pyproject_toml + + @property + def specifier(self): + # type: () -> Optional[STRING_TYPE] + options = [self._specifier] + for req in (self.ireq, self.requirement): + if req is not None and getattr(req, "specifier", None): + options.append(req.specifier) + specifier = next( + iter(spec for spec in options if spec is not None), None + ) # type: Optional[Union[Specifier, SpecifierSet]] + spec_string = None # type: Optional[STRING_TYPE] + if specifier is not None: + spec_string = specs_to_string(specifier) + elif ( + specifier is None + and not self.is_named + and (self._setup_info is not None and self._setup_info.version) + ): + spec_string = "=={0}".format(self._setup_info.version) + if spec_string: + self._specifier = spec_string + return self._specifier + + @specifier.setter + def specifier(self, spec): + # type: (str) -> None + if not spec.startswith("=="): + spec = "=={0}".format(spec) + self._specifier = spec + self.specifiers = SpecifierSet(spec) + + @property + def specifiers(self): + # type: () -> Optional[SpecifierSet] + ireq_needs_specifier = False + req_needs_specifier = False + if self.ireq is None or self.ireq.req is None or not self.ireq.req.specifier: + ireq_needs_specifier = True + if self.requirement is None or not self.requirement.specifier: + req_needs_specifier = True + if any([ireq_needs_specifier, req_needs_specifier]): + # TODO: Should we include versions for VCS dependencies? IS there a reason not + # to? For now we are using hashes as the equivalent to pin + # note: we need versions for direct dependencies at the very least + if ( + self.is_file + or self.is_url + or self.is_path + or (self.is_vcs and not self.editable) + ): + if self.specifier is not None: + specifier = self.specifier + if not isinstance(specifier, SpecifierSet): + specifier = SpecifierSet(specifier) + self.specifiers = specifier + return specifier + if self.ireq is not None and self.ireq.req is not None: + return self.ireq.req.specifier + elif self.requirement is not None: + return self.requirement.specifier + return None + + @specifiers.setter + def specifiers(self, specifiers): + # type: (Union[Text, str, SpecifierSet]) -> None + if not isinstance(specifiers, SpecifierSet): + if isinstance(specifiers, six.string_types): + specifiers = SpecifierSet(specifiers) + else: + raise TypeError("Must pass a string or a SpecifierSet") + specs = self.get_requirement_specs(specifiers) + if self.ireq is not None and self._ireq and self._ireq.req is not None: + self._ireq.req.specifier = specifiers + self._ireq.req.specs = specs + if self.requirement is not None: + self.requirement.specifier = specifiers + self.requirement.specs = specs + + @classmethod + def get_requirement_specs(cls, specifierset): + # type: (SpecifierSet) -> List[Tuple[AnyStr, AnyStr]] + specs = [] + spec = next(iter(specifierset._specs), None) + if spec: + specs.append(spec._spec) + return specs + + @property + def requirement(self): + # type: () -> Optional[RequirementType] + if self._requirement is None: + self.parse_requirement() + if self._requirement is None and self._name is not None: + self._requirement = init_requirement(canonicalize_name(self.name)) + if self.is_file or self.is_url and self._requirement is not None: + self._requirement.url = self.url + if ( + self._requirement + and self._requirement.specifier + and not self._requirement.specs + ): + specs = self.get_requirement_specs(self._requirement.specifier) + self._requirement.specs = specs + return self._requirement + + def populate_setup_paths(self): + # type: () -> None + if not self.link and not self.path: + self.parse_link() + if not self.path: + return + base_path = self.base_path + if base_path is None: + return + setup_paths = get_setup_paths( + base_path, subdirectory=self.subdirectory + ) # type: Dict[STRING_TYPE, Optional[STRING_TYPE]] + self._setup_py = setup_paths.get("setup_py") + self._setup_cfg = setup_paths.get("setup_cfg") + self._pyproject_toml = setup_paths.get("pyproject_toml") + + @property + def pyproject_requires(self): + # type: () -> Optional[Tuple[STRING_TYPE, ...]] + if self._pyproject_requires is None and self.pyproject_toml is not None: + if self.path is not None: + pyproject_requires, pyproject_backend = None, None + pyproject_results = get_pyproject(self.path) # type: ignore + if pyproject_results: + pyproject_requires, pyproject_backend = pyproject_results + if pyproject_requires: + self._pyproject_requires = tuple(pyproject_requires) + self._pyproject_backend = pyproject_backend + return self._pyproject_requires + + @property + def pyproject_backend(self): + # type: () -> Optional[STRING_TYPE] + if self._pyproject_requires is None and self.pyproject_toml is not None: + pyproject_requires = None # type: Optional[Sequence[STRING_TYPE]] + pyproject_backend = None # type: Optional[STRING_TYPE] + pyproject_results = get_pyproject(self.path) # type: ignore + if pyproject_results: + pyproject_requires, pyproject_backend = pyproject_results + if not pyproject_backend and self.setup_cfg is not None: + setup_dict = SetupInfo.get_setup_cfg(self.setup_cfg) + pyproject_backend = get_default_pyproject_backend() + pyproject_requires = setup_dict.get( + "build_requires", ["setuptools", "wheel"] + ) # type: ignore + if pyproject_requires: + self._pyproject_requires = tuple(pyproject_requires) + if pyproject_backend: + self._pyproject_backend = pyproject_backend + return self._pyproject_backend + + def parse_hashes(self): + # type: () -> None + """ + Parse hashes from *self.line* and set them on the current object. + :returns: Nothing + :rtype: None + """ + line, hashes = self.split_hashes(self.line) + self.hashes = hashes + self.line = line + return self + + def parse_extras(self): + # type: () -> None + """ + Parse extras from *self.line* and set them on the current object + :returns: Nothing + :rtype: None + """ + extras = None + if "@" in self.line or self.is_vcs or self.is_url: + line = "{0}".format(self.line) + uri = URI.parse(line) + name = uri.name + if name: + self._name = name + if uri.host and uri.path and uri.scheme: + self.line = uri.to_string( + escape_password=False, direct=False, strip_ssh=uri.is_implicit_ssh + ) + else: + self.line, extras = pip_shims.shims._strip_extras(self.line) + else: + self.line, extras = pip_shims.shims._strip_extras(self.line) + extras_set = set() # type: Set[STRING_TYPE] + if extras is not None: + extras_set = set(parse_extras(extras)) + if self._name: + self._name, name_extras = pip_shims.shims._strip_extras(self._name) + if name_extras: + name_extras = set(parse_extras(name_extras)) + extras_set |= name_extras + if extras_set is not None: + self.extras = tuple(sorted(extras_set)) + return self + + def get_url(self): + # type: () -> STRING_TYPE + """Sets ``self.name`` if given a **PEP-508** style URL""" + line = self.line + try: + parsed = URI.parse(line) + line = parsed.to_string(escape_password=False, direct=False, strip_ref=True) + except ValueError: + pass + else: + self._parsed_url = parsed + return line + if self.vcs is not None and self.line.startswith("{0}+".format(self.vcs)): + _, _, _parseable = self.line.partition("+") + parsed = urllib_parse.urlparse(add_ssh_scheme_to_git_uri(_parseable)) + line, _ = split_ref_from_uri(line) + else: + parsed = urllib_parse.urlparse(add_ssh_scheme_to_git_uri(line)) + if "@" in self.line and parsed.scheme == "": + name, _, url = self.line.partition("@") + if self._name is None: + url = url.strip() + self._name = name.strip() + if is_valid_url(url): + self.is_direct_url = True + line = url.strip() + parsed = urllib_parse.urlparse(line) + url_path = parsed.path + if "@" in url_path: + url_path, _, _ = url_path.rpartition("@") + parsed = parsed._replace(path=url_path) + self._parsed_url = parsed + return line + + @property + def name(self): + # type: () -> Optional[STRING_TYPE] + if self._name is None: + self.parse_name() + if self._name is None and not self.is_named and not self.is_wheel: + if self.setup_info: + self._name = self.setup_info.name + elif self.is_wheel: + self._name = self._parse_wheel() + if not self._name: + self._name = self.ireq.name + return self._name + + @name.setter + def name(self, name): + # type: (STRING_TYPE) -> None + self._name = name + if self._setup_info: + self._setup_info.name = name + if self.requirement and self._requirement: + self._requirement.name = name + if self.ireq and self._ireq and self._ireq.req: + self._ireq.req.name = name + + @property + def url(self): + # type: () -> Optional[STRING_TYPE] + if self.uri is not None: + url = add_ssh_scheme_to_git_uri(self.uri) + else: + url = getattr(self.link, "url_without_fragment", None) + if url is not None: + url = add_ssh_scheme_to_git_uri(unquote(url)) + if url is not None and self._parsed_url is None: + if self.vcs is not None: + _, _, _parseable = url.partition("+") + self._parsed_url = urllib_parse.urlparse(_parseable) + if self.is_vcs: + # strip the ref from the url + url, _ = split_ref_from_uri(url) + return url + + @property + def link(self): + # type: () -> Link + if self._link is None: + self.parse_link() + return self._link + + @property + def subdirectory(self): + # type: () -> Optional[STRING_TYPE] + if self.link is not None: + return self.link.subdirectory_fragment + return "" + + @property + def is_wheel(self): + # type: () -> bool + if self.link is None: + return False + return self.link.is_wheel + + @property + def is_artifact(self): + # type: () -> bool + if self.link is None: + return False + return self.link.is_artifact + + @property + def is_vcs(self): + # type: () -> bool + # Installable local files and installable non-vcs urls are handled + # as files, generally speaking + if is_vcs(self.line) or is_vcs(self.get_url()): + return True + return False + + @property + def is_url(self): + # type: () -> bool + url = self.get_url() + if is_valid_url(url) or is_file_url(url): + return True + return False + + @property + def is_path(self): + # type: () -> bool + if ( + self.path + and ( + self.path.startswith(".") + or os.path.isabs(self.path) + or os.path.exists(self.path) + ) + and is_installable_file(self.path) + ): + return True + elif (os.path.exists(self.line) and is_installable_file(self.line)) or ( + os.path.exists(self.get_url()) and is_installable_file(self.get_url()) + ): + return True + return False + + @property + def is_file_url(self): + # type: () -> bool + url = self.get_url() + parsed_url_scheme = self._parsed_url.scheme if self._parsed_url else "" + if url and is_file_url(self.get_url()) or parsed_url_scheme == "file": + return True + return False + + @property + def is_file(self): + # type: () -> bool + if ( + self.is_path + or (is_file_url(self.get_url()) and is_installable_file(self.get_url())) + or ( + self._parsed_url + and self._parsed_url.scheme == "file" + and is_installable_file(urllib_parse.urlunparse(self._parsed_url)) + ) + ): + return True + return False + + @property + def is_named(self): + # type: () -> bool + return not (self.is_file_url or self.is_url or self.is_file or self.is_vcs) + + @property + def ref(self): + # type: () -> Optional[STRING_TYPE] + if self._ref is None and self.relpath is not None: + self.relpath, self._ref = split_ref_from_uri(self.relpath) + return self._ref + + @property + def ireq(self): + # type: () -> Optional[pip_shims.InstallRequirement] + if self._ireq is None: + self.parse_ireq() + return self._ireq + + @property + def is_installable(self): + # type: () -> bool + possible_paths = (self.line, self.get_url(), self.path, self.base_path) + return any(is_installable_file(p) for p in possible_paths if p is not None) + + @property + def wheel_kwargs(self): + if not self._wheel_kwargs: + self._wheel_kwargs = _prepare_wheel_building_kwargs(self.ireq) + return self._wheel_kwargs + + def get_setup_info(self): + # type: () -> SetupInfo + setup_info = SetupInfo.from_ireq(self.ireq) + if not setup_info.name: + setup_info.get_info() + return setup_info + + @property + def setup_info(self): + # type: () -> Optional[SetupInfo] + if self._setup_info is None and not self.is_named and not self.is_wheel: + if self._setup_info: + if not self._setup_info.name: + self._setup_info.get_info() + else: + # make two attempts at this before failing to allow for stale data + try: + self.setup_info = self.get_setup_info() + except FileNotFoundError: + try: + self.setup_info = self.get_setup_info() + except FileNotFoundError: + raise + return self._setup_info + + @setup_info.setter + def setup_info(self, setup_info): + # type: (SetupInfo) -> None + self._setup_info = setup_info + if setup_info.version: + self.specifier = setup_info.version + if setup_info.name and not self.name: + self.name = setup_info.name + + def _get_vcsrepo(self): + # type: () -> Optional[VCSRepository] + from .vcs import VCSRepository + + checkout_directory = self.wheel_kwargs["src_dir"] # type: ignore + if self.name is not None: + checkout_directory = os.path.join( + checkout_directory, self.name + ) # type: ignore + vcsrepo = VCSRepository( + url=self.link.url, + name=self.name, + ref=self.ref if self.ref else None, + checkout_directory=checkout_directory, + vcs_type=self.vcs, + subdirectory=self.subdirectory, + ) + if not (self.link.scheme.startswith("file") and self.editable): + vcsrepo.obtain() + return vcsrepo + + @property + def vcsrepo(self): + # type: () -> Optional[VCSRepository] + if self._vcsrepo is None and self.is_vcs: + self._vcsrepo = self._get_vcsrepo() + return self._vcsrepo + + @cached_property + def metadata(self): + # type: () -> Dict[Any, Any] + if self.is_local and self.path and is_installable_dir(self.path): + return get_metadata(self.path) + return {} + + @cached_property + def parsed_setup_cfg(self): + # type: () -> Dict[Any, Any] + if self.is_local and self.path and is_installable_dir(self.path): + if self.setup_cfg: + return parse_setup_cfg(self.setup_cfg) + return {} + + @cached_property + def parsed_setup_py(self): + # type: () -> Dict[Any, Any] + if self.is_local and self.path and is_installable_dir(self.path): + if self.setup_py: + return ast_parse_setup_py(self.setup_py) + return {} + + @vcsrepo.setter + def vcsrepo(self, repo): + # type (VCSRepository) -> None + self._vcsrepo = repo + ireq = self.ireq + wheel_kwargs = self.wheel_kwargs.copy() + wheel_kwargs["src_dir"] = repo.checkout_directory + ireq.source_dir = wheel_kwargs["src_dir"] + ireq.build_location(wheel_kwargs["build_dir"]) + ireq._temp_build_dir.path = wheel_kwargs["build_dir"] + with temp_path(): + sys.path = [repo.checkout_directory, "", ".", get_python_lib(plat_specific=0)] + setupinfo = SetupInfo.create( + repo.checkout_directory, + ireq=ireq, + subdirectory=self.subdirectory, + kwargs=wheel_kwargs, + ) + self._setup_info = setupinfo + self._setup_info.reload() + + def get_ireq(self): + # type: () -> InstallRequirement + line = self.line_for_ireq + if self.editable: + ireq = pip_shims.shims.install_req_from_editable(line) + else: + ireq = pip_shims.shims.install_req_from_line(line) + if self.is_named: + ireq = pip_shims.shims.install_req_from_line(self.line) + if self.is_file or self.is_url: + ireq.link = self.link + if self.extras and not ireq.extras: + ireq.extras = set(self.extras) + if self.parsed_marker is not None and not ireq.markers: + ireq.markers = self.parsed_marker + if not ireq.req and self._requirement is not None: + ireq.req = copy.deepcopy(self._requirement) + return ireq + + def parse_ireq(self): + # type: () -> None + if self._ireq is None: + self._ireq = self.get_ireq() + if self._ireq is not None: + if self.requirement is not None and self._ireq.req is None: + self._ireq.req = self.requirement + + def _parse_wheel(self): + # type: () -> Optional[STRING_TYPE] + if not self.is_wheel: + pass + from pip_shims.shims import Wheel + + _wheel = Wheel(self.link.filename) + name = _wheel.name + version = _wheel.version + self._specifier = "=={0}".format(version) + return name + + def _parse_name_from_link(self): + # type: () -> Optional[STRING_TYPE] + if self.link is None: + return None + if getattr(self.link, "egg_fragment", None): + return self.link.egg_fragment + elif self.is_wheel: + return self._parse_wheel() + return None + + def _parse_name_from_line(self): + # type: () -> Optional[STRING_TYPE] + if not self.is_named: + pass + try: + self._requirement = init_requirement(self.line) + except Exception: + raise RequirementError( + "Failed parsing requirement from {0!r}".format(self.line) + ) + name = self._requirement.name + if not self._specifier and self._requirement and self._requirement.specifier: + self._specifier = specs_to_string(self._requirement.specifier) + if self._requirement.extras and not self.extras: + self.extras = self._requirement.extras + if not name: + name = self.line + specifier_match = next( + iter(spec for spec in SPECIFIERS_BY_LENGTH if spec in self.line), None + ) + specifier = None # type: Optional[STRING_TYPE] + if specifier_match: + specifier = "{0!s}".format(specifier_match) + if specifier is not None and specifier in name: + version = None # type: Optional[STRING_TYPE] + name, specifier, version = name.partition(specifier) + self._specifier = "{0}{1}".format(specifier, version) + return name + + def _parse_name_from_path(self): + # type: () -> Optional[S] + if self.path and self.is_local and is_installable_dir(self.path): + metadata = get_metadata(self.path) + if metadata: + name = metadata.get("name", "") + if name and name != "wheel": + return name + parsed_setup_cfg = self.parsed_setup_cfg + if parsed_setup_cfg: + name = parsed_setup_cfg.get("name", "") + if name: + return name + + parsed_setup_py = self.parsed_setup_py + if parsed_setup_py: + name = parsed_setup_py.get("name", "") + if name: + return name + return None + + def parse_name(self): + # type: () -> "Line" + if self._name is None: + name = None + if self.link is not None: + name = self._parse_name_from_link() + if name is None and ( + (self.is_url or self.is_artifact or self.is_vcs) and self._parsed_url + ): + if self._parsed_url.fragment: + _, _, name = self._parsed_url.fragment.partition("egg=") + if "&" in name: + # subdirectory fragments might also be in here + name, _, _ = name.partition("&") + if name is None and self.is_named: + name = self._parse_name_from_line() + elif name is None and self.is_file or self.is_url or self.is_path: + if self.is_local: + name = self._parse_name_from_path() + if name is not None: + name, extras = pip_shims.shims._strip_extras(name) + if extras is not None and not self.extras: + self.extras = tuple(sorted(set(parse_extras(extras)))) + self._name = name + return self + + def _parse_requirement_from_vcs(self): + # type: () -> Optional[PackagingRequirement] + url = self.url if self.url else self.link.url + if url: + url = unquote(url) + if ( + url + and self.uri != url + and "git+ssh://" in url + and (self.uri is not None and "git+git@" in self.uri) + and self._requirement is not None + ): + self._requirement.line = self.uri + self._requirement.url = self.url + vcs_uri = build_vcs_uri( # type: ignore + vcs=self.vcs, + uri=self.url, + ref=self.ref, + subdirectory=self.subdirectory, + extras=self.extras, + name=self.name, + ) + if vcs_uri: + self._requirement.link = create_link(vcs_uri) + elif self.link: + self._requirement.link = self.link + # else: + # req.link = self.link + if self.ref and self._requirement is not None: + if self._vcsrepo is not None: + self._requirement.revision = self._vcsrepo.get_commit_hash() + else: + self._requirement.revision = self.ref + return self._requirement + + def parse_requirement(self): + # type: () -> "Line" + if self._name is None: + self.parse_name() + if not self._name and not self.is_vcs and not self.is_named: + if self.setup_info and self.setup_info.name: + self._name = self.setup_info.name + name, extras, url = self.requirement_info + if name: + self._requirement = init_requirement(name) # type: PackagingRequirement + if extras: + self._requirement.extras = set(extras) + if url: + self._requirement.url = url + if self.is_direct_url: + url = self.link.url + if self.link: + self._requirement.link = self.link + self._requirement.editable = self.editable + if self.path and self.link and self.link.scheme.startswith("file"): + self._requirement.local_file = True + self._requirement.path = self.path + if self.is_vcs: + self._requirement.vcs = self.vcs + self._requirement.line = self.link.url + self._parse_requirement_from_vcs() + else: + self._requirement.line = self.line + if self.parsed_marker is not None: + self._requirement.marker = self.parsed_marker + if self.specifiers: + self._requirement.specifier = self.specifiers + specs = [] + spec = next(iter(s for s in self.specifiers._specs), None) + if spec: + specs.append(spec._spec) + self._requirement.spec = spec + else: + if self.is_vcs: + raise ValueError( + "pipenv requires an #egg fragment for version controlled " + "dependencies. Please install remote dependency " + "in the form {0}#egg=.".format(url) + ) + return self + + def parse_link(self): + # type: () -> "Line" + parsed_url = None # type: Optional[URI] + if not is_valid_url(self.line) and ( + self.line.startswith("./") + or (os.path.exists(self.line) or os.path.isabs(self.line)) + ): + url = pip_shims.shims.path_to_url(os.path.abspath(self.line)) + parsed_url = URI.parse(url) + elif is_valid_url(self.line) or is_vcs(self.line) or is_file_url(self.line): + parsed_url = URI.parse(self.line) + if parsed_url is not None: + line = parsed_url.to_string( + escape_password=False, direct=False, strip_ref=True, strip_ssh=False + ) + if parsed_url.is_vcs: + self.vcs, _ = parsed_url.scheme.split("+") + if parsed_url.is_file_url: + self.is_local = True + parsed_link = parsed_url.as_link + self._ref = parsed_url.ref + self.uri = parsed_url.bare_url + if parsed_url.name: + self._name = parsed_url.name + if parsed_url.extras: + self.extras = tuple(sorted(set(parsed_url.extras))) + self._link = parsed_link + vcs, prefer, relpath, path, uri, link = FileRequirement.get_link_from_line( + self.line + ) + ref = None + if link is not None and "@" in unquote(link.path) and uri is not None: + uri, _, ref = unquote(uri).rpartition("@") + if relpath is not None and "@" in relpath: + relpath, _, ref = relpath.rpartition("@") + if path is not None and "@" in path: + path, _ = split_ref_from_uri(path) + link_url = link.url_without_fragment + if "@" in link_url: + link_url, _ = split_ref_from_uri(link_url) + self.preferred_scheme = prefer + self.relpath = relpath + self.path = path + # self.uri = uri + if prefer in ("path", "relpath") or uri.startswith("file"): + self.is_local = True + if parsed_url.is_vcs or parsed_url.is_direct_url and parsed_link: + self._link = parsed_link + else: + self._link = link + return self + + def parse_markers(self): + # type: () -> None + if self.markers: + marker_str = self.markers.replace('"', "'") + markers = PackagingRequirement("fakepkg; {0}".format(marker_str)).marker + self.parsed_marker = markers + + @property + def requirement_info(self): + # type: () -> Tuple[Optional[S], Tuple[Optional[S], ...], Optional[S]] + """ + Generates a 3-tuple of the requisite *name*, *extras* and *url* to generate a + :class:`~packaging.requirements.Requirement` out of. + + :return: A Tuple of an optional name, a Tuple of extras, and an optional URL. + :rtype: Tuple[Optional[S], Tuple[Optional[S], ...], Optional[S]] + """ + + # Direct URLs can be converted to packaging requirements directly, but + # only if they are `file://` (with only two slashes) + name = None # type: Optional[S] + extras = () # type: Tuple[Optional[S], ...] + url = None # type: Optional[STRING_TYPE] + # if self.is_direct_url: + if self._name: + name = canonicalize_name(self._name) + if self.is_file or self.is_url or self.is_path or self.is_file_url or self.is_vcs: + url = "" + if self.is_vcs: + url = self.url if self.url else self.uri + if self.is_direct_url: + url = self.link.url_without_fragment + else: + if self.link: + url = self.link.url_without_fragment + elif self.url: + url = self.url + if self.ref: + url = "{0}@{1}".format(url, self.ref) + else: + url = self.uri + if self.link and name is None: + self._name = self.link.egg_fragment + if self._name: + name = canonicalize_name(self._name) + return name, extras, url # type: ignore + + @property + def line_is_installable(self): + # type: () -> bool + """ + This is a safeguard against decoy requirements when a user installs a package + whose name coincides with the name of a folder in the cwd, e.g. install *alembic* + when there is a folder called *alembic* in the working directory. + + In this case we first need to check that the given requirement is a valid + URL, VCS requirement, or installable filesystem path before deciding to treat it + as a file requirement over a named requirement. + """ + line = self.line + if is_file_url(line): + link = create_link(line) + line = link.url_without_fragment + line, _ = split_ref_from_uri(line) + if ( + is_vcs(line) + or ( + is_valid_url(line) + and (not is_file_url(line) or is_installable_file(line)) + ) + or is_installable_file(line) + ): + return True + return False + + def parse(self): + # type: () -> None + self.line = self.line.strip() + if self.line.startswith('"'): + self.line = self.line.strip('"') + self.line, self.markers = split_markers_from_line(self.parse_hashes().line) + if self.markers: + self.markers = self.markers.replace('"', "'") + self.parse_extras() + self.line = self.line.strip('"').strip("'").strip() + if self.line.startswith("git+file:/") and not self.line.startswith( + "git+file:///" + ): + self.line = self.line.replace("git+file:/", "git+file:///") + self.parse_markers() + if self.is_file_url: + if self.line_is_installable: + self.populate_setup_paths() + else: + raise RequirementError( + "Supplied requirement is not installable: {0!r}".format(self.line) + ) + self.parse_link() + # self.parse_requirement() + # self.parse_ireq() + + +@attr.s(slots=True, hash=True) class NamedRequirement(object): - name = attr.ib() - version = attr.ib(validator=attr.validators.optional(validate_specifiers)) - req = attr.ib() - extras = attr.ib(default=attr.Factory(list)) - editable = attr.ib(default=False) + name = attr.ib() # type: STRING_TYPE + version = attr.ib() # type: Optional[STRING_TYPE] + req = attr.ib() # type: PackagingRequirement + extras = attr.ib(default=attr.Factory(list)) # type: Tuple[STRING_TYPE, ...] + editable = attr.ib(default=False) # type: bool + _parsed_line = attr.ib(default=None) # type: Optional[Line] @req.default def get_requirement(self): + # type: () -> RequirementType req = init_requirement( "{0}{1}".format(canonicalize_name(self.name), self.version) ) return req + @property + def parsed_line(self): + # type: () -> Optional[Line] + if self._parsed_line is None: + self._parsed_line = Line(self.line_part) + return self._parsed_line + @classmethod - def from_line(cls, line): + def from_line(cls, line, parsed_line=None): + # type: (AnyStr, Optional[Line]) -> NamedRequirement req = init_requirement(line) - specifiers = None + specifiers = None # type: Optional[STRING_TYPE] if req.specifier: specifiers = specs_to_string(req.specifier) req.line = line @@ -89,39 +1294,57 @@ class NamedRequirement(object): if not name: name = getattr(req, "key", line) req.name = name - extras = None + creation_kwargs = { + "name": name, + "version": specifiers, + "req": req, + "parsed_line": parsed_line, + "extras": None, + } + extras = None # type: Optional[Tuple[STRING_TYPE, ...]] if req.extras: - extras = list(req.extras) - return cls(name=name, version=specifiers, req=req, extras=extras) + extras = tuple(req.extras) + creation_kwargs["extras"] = extras + return cls(**creation_kwargs) @classmethod def from_pipfile(cls, name, pipfile): - creation_args = {} + # type: (S, TPIPFILE) -> NamedRequirement + creation_args = {} # type: TPIPFILE if hasattr(pipfile, "keys"): attr_fields = [field.name for field in attr.fields(cls)] - creation_args = {k: v for k, v in pipfile.items() if k in attr_fields} + creation_args = { + k: v for k, v in pipfile.items() if k in attr_fields + } # type: ignore creation_args["name"] = name - version = get_version(pipfile) + version = get_version(pipfile) # type: Optional[STRING_TYPE] extras = creation_args.get("extras", None) - creation_args["version"] = version + creation_args["version"] = version # type: ignore req = init_requirement("{0}{1}".format(name, version)) - if extras: - req.extras += tuple(extras) + if req and extras and req.extras and isinstance(req.extras, tuple): + if isinstance(extras, six.string_types): + req.extras = (extras) + tuple(["{0}".format(xtra) for xtra in req.extras]) + elif isinstance(extras, (tuple, list)): + req.extras += tuple(extras) creation_args["req"] = req - return cls(**creation_args) + return cls(**creation_args) # type: ignore @property def line_part(self): + # type: () -> STRING_TYPE # FIXME: This should actually be canonicalized but for now we have to # simply lowercase it and replace underscores, since full canonicalization # also replaces dots and that doesn't actually work when querying the index - return "{0}".format(normalize_name(self.name)) + return normalize_name(self.name) @property def pipfile_part(self): - pipfile_dict = attr.asdict(self, filter=filter_none).copy() + # type: () -> Dict[STRING_TYPE, Any] + pipfile_dict = attr.asdict(self, filter=filter_none).copy() # type: ignore if "version" not in pipfile_dict: pipfile_dict["version"] = "*" + if "_parsed_line" in pipfile_dict: + pipfile_dict.pop("_parsed_line") name = pipfile_dict.pop("name") return {name: pipfile_dict} @@ -131,40 +1354,46 @@ LinkInfo = collections.namedtuple( ) -@attr.s(slots=True) +@attr.s(slots=True, cmp=True, hash=True) class FileRequirement(object): """File requirements for tar.gz installable files or wheels or setup.py containing directories.""" #: Path to the relevant `setup.py` location - setup_path = attr.ib(default=None) + setup_path = attr.ib(default=None, cmp=True) # type: Optional[STRING_TYPE] #: path to hit - without any of the VCS prefixes (like git+ / http+ / etc) - path = attr.ib(default=None, validator=attr.validators.optional(validate_path)) + path = attr.ib(default=None, cmp=True) # type: Optional[STRING_TYPE] #: Whether the package is editable - editable = attr.ib(default=False) + editable = attr.ib(default=False, cmp=True) # type: bool #: Extras if applicable - extras = attr.ib(default=attr.Factory(list)) - _uri_scheme = attr.ib(default=None) + extras = attr.ib( + default=attr.Factory(tuple), cmp=True + ) # type: Tuple[STRING_TYPE, ...] + _uri_scheme = attr.ib(default=None, cmp=True) # type: Optional[STRING_TYPE] #: URI of the package - uri = attr.ib() + uri = attr.ib(cmp=True) # type: Optional[STRING_TYPE] #: Link object representing the package to clone - link = attr.ib() + link = attr.ib(cmp=True) # type: Optional[Link] #: PyProject Requirements - pyproject_requires = attr.ib(default=attr.Factory(list)) + pyproject_requires = attr.ib( + factory=tuple, cmp=True + ) # type: Optional[Tuple[STRING_TYPE, ...]] #: PyProject Build System - pyproject_backend = attr.ib(default=None) + pyproject_backend = attr.ib(default=None, cmp=True) # type: Optional[STRING_TYPE] #: PyProject Path - pyproject_path = attr.ib(default=None) - _has_hashed_name = attr.ib(default=False) - #: Package name - name = attr.ib() - #: A :class:`~pkg_resources.Requirement` isntance - req = attr.ib() + pyproject_path = attr.ib(default=None, cmp=True) # type: Optional[STRING_TYPE] #: Setup metadata e.g. dependencies - setup_info = attr.ib(default=None) + _setup_info = attr.ib(default=None, cmp=True) # type: Optional[SetupInfo] + _has_hashed_name = attr.ib(default=False, cmp=True) # type: bool + _parsed_line = attr.ib(default=None, cmp=False, hash=True) # type: Optional[Line] + #: Package name + name = attr.ib(cmp=True) # type: Optional[STRING_TYPE] + #: A :class:`~pkg_resources.Requirement` instance + req = attr.ib(cmp=True) # type: Optional[PackagingRequirement] @classmethod def get_link_from_line(cls, line): + # type: (STRING_TYPE) -> LinkInfo """Parse link information from given requirement line. Return a 6-tuple: @@ -198,15 +1427,16 @@ class FileRequirement(object): # Git allows `git@github.com...` lines that are not really URIs. # Add "ssh://" so we can parse correctly, and restore afterwards. - fixed_line = add_ssh_scheme_to_git_uri(line) - added_ssh_scheme = fixed_line != line + fixed_line = add_ssh_scheme_to_git_uri(line) # type: STRING_TYPE + added_ssh_scheme = fixed_line != line # type: bool # We can assume a lot of things if this is a local filesystem path. if "://" not in fixed_line: - p = Path(fixed_line).absolute() - path = p.as_posix() - uri = p.as_uri() - link = create_link(uri) + p = Path(fixed_line).absolute() # type: Path + path = p.as_posix() # type: Optional[STRING_TYPE] + uri = p.as_uri() # type: STRING_TYPE + link = create_link(uri) # type: Link + relpath = None # type: Optional[STRING_TYPE] try: relpath = get_converted_relative_path(path) except ValueError: @@ -215,19 +1445,17 @@ class FileRequirement(object): # This is an URI. We'll need to perform some elaborated parsing. - parsed_url = urllib_parse.urlsplit(fixed_line) - original_url = parsed_url._replace() - if added_ssh_scheme and ":" in parsed_url.netloc: - original_netloc, original_path_start = parsed_url.netloc.rsplit(":", 1) - uri_path = "/{0}{1}".format(original_path_start, parsed_url.path) - parsed_url = original_url._replace(netloc=original_netloc, path=uri_path) + parsed_url = urllib_parse.urlsplit(fixed_line) # type: SplitResult + original_url = parsed_url._replace() # type: SplitResult # Split the VCS part out if needed. - original_scheme = parsed_url.scheme + original_scheme = parsed_url.scheme # type: STRING_TYPE + vcs_type = None # type: Optional[STRING_TYPE] if "+" in original_scheme: - vcs_type, scheme = original_scheme.split("+", 1) - parsed_url = parsed_url._replace(scheme=scheme) - prefer = "uri" + scheme = None # type: Optional[STRING_TYPE] + vcs_type, _, scheme = original_scheme.partition("+") + parsed_url = parsed_url._replace(scheme=scheme) # type: ignore + prefer = "uri" # type: STRING_TYPE else: vcs_type = None prefer = "file" @@ -249,102 +1477,134 @@ class FileRequirement(object): relpath = None # Cut the fragment, but otherwise this is fixed_line. uri = urllib_parse.urlunsplit( - parsed_url._replace(scheme=original_scheme, fragment="") + parsed_url._replace(scheme=original_scheme, fragment="") # type: ignore ) if added_ssh_scheme: original_uri = urllib_parse.urlunsplit( - original_url._replace(scheme=original_scheme, fragment="") + original_url._replace(scheme=original_scheme, fragment="") # type: ignore ) uri = strip_ssh_from_git_uri(original_uri) # Re-attach VCS prefix to build a Link. link = create_link( - urllib_parse.urlunsplit(parsed_url._replace(scheme=original_scheme)) + urllib_parse.urlunsplit( + parsed_url._replace(scheme=original_scheme) + ) # type: ignore ) return LinkInfo(vcs_type, prefer, relpath, path, uri, link) @property def setup_py_dir(self): + # type: () -> Optional[STRING_TYPE] if self.setup_path: return os.path.dirname(os.path.abspath(self.setup_path)) + return None @property def dependencies(self): - build_deps = [] - setup_deps = [] - deps = {} + # type: () -> Tuple[Dict[S, PackagingRequirement], List[Union[S, PackagingRequirement]], List[S]] + build_deps = [] # type: List[Union[S, PackagingRequirement]] + setup_deps = [] # type: List[S] + deps = {} # type: Dict[S, PackagingRequirement] if self.setup_info: setup_info = self.setup_info.as_dict() deps.update(setup_info.get("requires", {})) setup_deps.extend(setup_info.get("setup_requires", [])) build_deps.extend(setup_info.get("build_requires", [])) + if self.extras and self.setup_info.extras: + for dep in self.extras: + if dep not in self.setup_info.extras: + continue + extras_list = self.setup_info.extras.get(dep, []) # type: ignore + for req_instance in extras_list: # type: ignore + deps[req_instance.key] = req_instance if self.pyproject_requires: - build_deps.extend(self.pyproject_requires) + build_deps.extend(list(self.pyproject_requires)) + setup_deps = list(set(setup_deps)) + build_deps = list(set(build_deps)) return deps, setup_deps, build_deps + def __attrs_post_init__(self): + # type: () -> None + if self.name is None and self.parsed_line: + if self.parsed_line.setup_info: + self._setup_info = self.parsed_line.setup_info + if self.parsed_line.setup_info.name: + self.name = self.parsed_line.setup_info.name + if self.req is None and ( + self._parsed_line is not None and self._parsed_line.requirement is not None + ): + self.req = self._parsed_line.requirement + if ( + self._parsed_line + and self._parsed_line.ireq + and not self._parsed_line.ireq.req + ): + if self.req is not None and self._parsed_line._ireq is not None: + self._parsed_line._ireq.req = self.req + + @property + def setup_info(self): + # type: () -> Optional[SetupInfo] + from .setup_info import SetupInfo + + if self._setup_info is None and self.parsed_line: + if self.parsed_line and self._parsed_line and self.parsed_line.setup_info: + if ( + self._parsed_line._setup_info + and not self._parsed_line._setup_info.name + ): + self._parsed_line._setup_info.get_info() + self._setup_info = self.parsed_line._setup_info + elif self.parsed_line and ( + self.parsed_line.ireq and not self.parsed_line.is_wheel + ): + self._setup_info = SetupInfo.from_ireq(self.parsed_line.ireq) + else: + if self.link and not self.link.is_wheel: + self._setup_info = Line(self.line_part).setup_info + if self._setup_info: + self._setup_info.get_info() + return self._setup_info + + @setup_info.setter + def setup_info(self, setup_info): + # type: (SetupInfo) -> None + self._setup_info = setup_info + if self._parsed_line: + self._parsed_line._setup_info = setup_info + @uri.default def get_uri(self): + # type: () -> STRING_TYPE if self.path and not self.uri: self._uri_scheme = "path" return pip_shims.shims.path_to_url(os.path.abspath(self.path)) - elif self.req and getattr(self.req, "url"): + elif ( + getattr(self, "req", None) + and self.req is not None + and getattr(self.req, "url") + ): return self.req.url + elif self.link is not None: + return self.link.url_without_fragment + return "" @name.default def get_name(self): - loc = self.path or self.uri - if loc and not self._uri_scheme: - self._uri_scheme = "path" if self.path else "file" - name = None - if getattr(self, "req", None) and getattr(self.req, "name") and self.req.name is not None: - if self.is_direct_url: - return self.req.name - if self.link and self.link.egg_fragment and not self._has_hashed_name: + # type: () -> STRING_TYPE + if self.parsed_line and self.parsed_line.name: + return self.parsed_line.name + elif self.link and self.link.egg_fragment: return self.link.egg_fragment - elif self.link and self.link.is_wheel: - from pip_shims import Wheel - self._has_hashed_name = False - return Wheel(self.link.filename).name - elif self.link and ((self.link.scheme == "file" or self.editable) or ( - self.path and self.setup_path and os.path.isfile(str(self.setup_path)) - )): - if self.editable: - line = pip_shims.shims.path_to_url(self.setup_py_dir) - _ireq = pip_shims.shims.install_req_from_editable(line) - else: - _ireq = pip_shims.shims.install_req_from_line(Path(self.setup_py_dir).as_posix()) - from .setup_info import SetupInfo - subdir = getattr(self, "subdirectory", None) - setupinfo = SetupInfo.from_ireq(_ireq, subdir=subdir) - if setupinfo: - self.setup_info = setupinfo - setupinfo_dict = setupinfo.as_dict() - setup_name = setupinfo_dict.get("name", None) - if setup_name: - name = setup_name - self._has_hashed_name = False - build_requires = setupinfo_dict.get("build_requires") - build_backend = setupinfo_dict.get("build_backend") - if build_requires and not self.pyproject_requires: - self.pyproject_requires = build_requires - if build_backend and not self.pyproject_backend: - self.pyproject_backend = build_backend - hashed_loc = hashlib.sha256(loc.encode("utf-8")).hexdigest() - hashed_name = hashed_loc[-7:] - if not name or name.lower() == "unknown": - self._has_hashed_name = True - name = hashed_name - else: - self._has_hashed_name = False - name_in_link = getattr(self.link, "egg_fragment", "") if self.link else "" - if not self._has_hashed_name and name_in_link != name: - self.link = create_link("{0}#egg={1}".format(self.link.url, name)) - return name + elif self.setup_info and self.setup_info.name: + return self.setup_info.name @link.default def get_link(self): + # type: () -> pip_shims.shims.Link target = "{0}".format(self.uri) if hasattr(self, "name") and not self._has_hashed_name: target = "{0}#egg={1}".format(target, self.name) @@ -353,182 +1613,254 @@ class FileRequirement(object): @req.default def get_requirement(self): - req = init_requirement(normalize_name(self.name)) - req.editable = False - req.line = self.link.url_without_fragment - if self.path and self.link and self.link.scheme.startswith("file"): - req.local_file = True - req.path = self.path - if self.editable: - req.url = None + # type: () -> RequirementType + if self.name is None: + if self._parsed_line is not None and self._parsed_line.name is not None: + self.name = self._parsed_line.name else: - req.url = self.link.url_without_fragment - else: - req.local_file = False - req.path = None - req.url = self.link.url_without_fragment - if self.editable: - req.editable = True - req.link = self.link - return req + raise ValueError( + "Failed to generate a requirement: missing name for {0!r}".format( + self + ) + ) + if self._parsed_line: + try: + # initialize specifiers to make sure we capture them + self._parsed_line.specifiers + except Exception: + pass + req = copy.deepcopy(self._parsed_line.requirement) + if req: + return req + + @property + def parsed_line(self): + # type: () -> Optional[Line] + if self._parsed_line is None: + self._parsed_line = Line(self.line_part) + return self._parsed_line + + @property + def is_local(self): + # type: () -> bool + uri = getattr(self, "uri", None) + if uri is None: + if getattr(self, "path", None) and self.path is not None: + uri = pip_shims.shims.path_to_url(os.path.abspath(self.path)) + elif ( + getattr(self, "req", None) + and self.req is not None + and (getattr(self.req, "url") and self.req.url is not None) + ): + uri = self.req.url + if uri and is_file_url(uri): + return True + return False @property def is_remote_artifact(self): + # type: () -> bool + if self.link is None: + return False return ( - any( - self.link.scheme.startswith(scheme) - for scheme in ("http", "https", "ftp", "ftps", "uri") - ) - and (self.link.is_artifact or self.link.is_wheel) + self._parsed_line + and not self._parsed_line.is_local + and (self._parsed_line.is_artifact or self._parsed_line.is_wheel) and not self.editable ) @property def is_direct_url(self): + # type: () -> bool + if self._parsed_line is not None and self._parsed_line.is_direct_url: + return True return self.is_remote_artifact @property def formatted_path(self): + # type: () -> Optional[STRING_TYPE] if self.path: path = self.path if not isinstance(path, Path): path = Path(path) return path.as_posix() - return + return None @classmethod def create( - cls, path=None, uri=None, editable=False, extras=None, link=None, vcs_type=None, - name=None, req=None, line=None, uri_scheme=None, setup_path=None, relpath=None + cls, + path=None, # type: Optional[STRING_TYPE] + uri=None, # type: STRING_TYPE + editable=False, # type: bool + extras=None, # type: Optional[Tuple[STRING_TYPE, ...]] + link=None, # type: Link + vcs_type=None, # type: Optional[Any] + name=None, # type: Optional[STRING_TYPE] + req=None, # type: Optional[Any] + line=None, # type: Optional[STRING_TYPE] + uri_scheme=None, # type: STRING_TYPE + setup_path=None, # type: Optional[Any] + relpath=None, # type: Optional[Any] + parsed_line=None, # type: Optional[Line] ): + # type: (...) -> F + if parsed_line is None and line is not None: + parsed_line = Line(line) if relpath and not path: path = relpath - if not path and uri and link.scheme == "file": - path = os.path.abspath(pip_shims.shims.url_to_path(unquote(uri))) + if not path and uri and link is not None and link.scheme == "file": + path = os.path.abspath( + pip_shims.shims.url_to_path(unquote(uri)) + ) # type: ignore try: path = get_converted_relative_path(path) except ValueError: # Vistir raises a ValueError if it can't make a relpath path = path - if line and not (uri_scheme and uri and link): + if line is not None and not (uri_scheme and uri and link): vcs_type, uri_scheme, relpath, path, uri, link = cls.get_link_from_line(line) if not uri_scheme: uri_scheme = "path" if path else "file" if path and not uri: - uri = unquote(pip_shims.shims.path_to_url(os.path.abspath(path))) - if not link: - link = create_link(uri) - if not uri: + uri = unquote( + pip_shims.shims.path_to_url(os.path.abspath(path)) + ) # type: ignore + if not link and uri: + link = cls.get_link_from_line(uri).link + if not uri and link: uri = unquote(link.url_without_fragment) if not extras: - extras = [] + extras = () pyproject_path = None - if path is not None: - pyproject_requires = get_pyproject(os.path.abspath(path)) - pyproject_backend = None pyproject_requires = None - if pyproject_requires is not None: - pyproject_requires, pyproject_backend = pyproject_requires + pyproject_backend = None + pyproject_tuple = None # type: Optional[Tuple[STRING_TYPE]] + if path is not None: + pyproject_requires_and_backend = get_pyproject(path) + if pyproject_requires_and_backend is not None: + pyproject_requires, pyproject_backend = pyproject_requires_and_backend if path: - pyproject_path = Path(path).joinpath("pyproject.toml") - if not pyproject_path.exists(): - pyproject_path = None - if not setup_path and path is not None: - setup_path = Path(path).joinpath("setup.py") + setup_paths = get_setup_paths(path) + if isinstance(setup_paths, Mapping): + if "pyproject_toml" in setup_paths and setup_paths["pyproject_toml"]: + pyproject_path = Path(setup_paths["pyproject_toml"]) + if "setup_py" in setup_paths and setup_paths["setup_py"]: + setup_path = Path(setup_paths["setup_py"]).as_posix() if setup_path and isinstance(setup_path, Path): setup_path = setup_path.as_posix() creation_kwargs = { "editable": editable, "extras": extras, "pyproject_path": pyproject_path, - "setup_path": setup_path if setup_path else None, + "setup_path": setup_path, "uri_scheme": uri_scheme, "link": link, "uri": uri, - "pyproject_requires": pyproject_requires, - "pyproject_backend": pyproject_backend + "pyproject_requires": pyproject_tuple, + "pyproject_backend": pyproject_backend, + "path": path or relpath, + "parsed_line": parsed_line, } if vcs_type: - creation_kwargs["vcs_type"] = vcs_type - _line = None - if not name: - _line = unquote(link.url_without_fragment) if link.url else uri - if editable: - ireq = pip_shims.shims.install_req_from_editable(_line) - else: - _line = path if (uri_scheme and uri_scheme == "path") else _line - ireq = pip_shims.shims.install_req_from_line(_line) - setup_info = SetupInfo.from_ireq(ireq) - setupinfo_dict = setup_info.as_dict() - setup_name = setupinfo_dict.get("name", None) - if setup_name: - name = setup_name - build_requires = setupinfo_dict.get("build_requires", []) - build_backend = setupinfo_dict.get("build_backend", []) - if not creation_kwargs.get("pyproject_requires") and build_requires: - creation_kwargs["pyproject_requires"] = build_requires - if not creation_kwargs.get("pyproject_backend") and build_backend: - creation_kwargs["pyproject_backend"] = build_backend - creation_kwargs["setup_info"] = setup_info - if path or relpath: - creation_kwargs["path"] = relpath if relpath else path - if req: - creation_kwargs["req"] = req - if creation_kwargs.get("req") and line and not getattr(creation_kwargs["req"], "line", None): - creation_kwargs["req"].line = line + creation_kwargs["vcs"] = vcs_type if name: creation_kwargs["name"] = name - cls_inst = cls(**creation_kwargs) - if not _line: - if editable and uri_scheme == "path": - _line = relpath if relpath else path + _line = None # type: Optional[STRING_TYPE] + ireq = None # type: Optional[InstallRequirement] + setup_info = None # type: Optional[SetupInfo] + if parsed_line: + if parsed_line.name: + name = parsed_line.name + if parsed_line.setup_info: + name = parsed_line.setup_info.as_dict().get("name", name) + if not name or not parsed_line: + if link is not None and link.url_without_fragment is not None: + _line = unquote(link.url_without_fragment) + if name: + _line = "{0}#egg={1}".format(_line, name) + if _line and extras and extras_to_string(extras) not in _line: + _line = "{0}[{1}]".format( + _line, ",".join(sorted(set(extras))) + ) # type: ignore + elif isinstance(uri, six.string_types): + _line = unquote(uri) + elif line: + _line = unquote(line) + if editable: + if ( + _line + and extras + and extras_to_string(extras) not in _line + and ( + (link and link.scheme == "file") + or (uri and uri.startswith("file")) + or (not uri and not link) + ) + ): + _line = "{0}[{1}]".format( + _line, ",".join(sorted(set(extras))) + ) # type: ignore + if ireq is None: + ireq = pip_shims.shims.install_req_from_editable( + _line + ) # type: ignore else: - _line = unquote(cls_inst.link.url_without_fragment) or cls_inst.uri - _line = "{0}#egg={1}".format(line, cls_inst.name) if not cls_inst._has_hashed_name else _line - cls_inst.req.line = line if line else _line - return cls_inst + _line = path if (uri_scheme and uri_scheme == "path") else _line + if _line and extras and extras_to_string(extras) not in _line: + _line = "{0}[{1}]".format( + _line, ",".join(sorted(set(extras))) + ) # type: ignore + if ireq is None: + ireq = pip_shims.shims.install_req_from_line(_line) # type: ignore + if editable: + _line = "-e {0}".format(editable) + if _line: + parsed_line = Line(_line) + if ireq is None and parsed_line and parsed_line.ireq: + ireq = parsed_line.ireq + if extras and ireq is not None and not ireq.extras: + ireq.extras = set(extras) + if setup_info is None: + setup_info = SetupInfo.from_ireq(ireq) + setupinfo_dict = setup_info.as_dict() + setup_name = setupinfo_dict.get("name", None) + build_requires = () # type: Tuple[STRING_TYPE, ...] + build_backend = "" + if setup_name is not None: + name = setup_name + build_requires = setupinfo_dict.get("build_requires", build_requires) + build_backend = setupinfo_dict.get("build_backend", build_backend) + if "pyproject_requires" not in creation_kwargs and build_requires: + creation_kwargs["pyproject_requires"] = tuple(build_requires) + if "pyproject_backend" not in creation_kwargs and build_backend: + creation_kwargs["pyproject_backend"] = build_backend + if setup_info is None and parsed_line and parsed_line.setup_info: + setup_info = parsed_line.setup_info + creation_kwargs["setup_info"] = setup_info + if path or relpath: + creation_kwargs["path"] = relpath if relpath else path + if req is not None: + creation_kwargs["req"] = req + creation_req = creation_kwargs.get("req") + if creation_kwargs.get("req") is not None: + creation_req_line = getattr(creation_req, "line", None) + if creation_req_line is None and line is not None: + creation_kwargs["req"].line = line # type: ignore + if parsed_line and parsed_line.name: + if name and len(parsed_line.name) != 7 and len(name) == 7: + name = parsed_line.name + if name: + creation_kwargs["name"] = name + return cls(**creation_kwargs) # type: ignore @classmethod - def from_line(cls, line): - line = line.strip('"').strip("'") - link = None - path = None - editable = line.startswith("-e ") - line = line.split(" ", 1)[1] if editable else line - setup_path = None - name = None - req = None - if not any([is_installable_file(line), is_valid_url(line), is_file_url(line)]): - try: - req = init_requirement(line) - except Exception: - raise RequirementError( - "Supplied requirement is not installable: {0!r}".format(line) - ) - else: - name = getattr(req, "name", None) - line = getattr(req, "url", None) - vcs_type, prefer, relpath, path, uri, link = cls.get_link_from_line(line) - arg_dict = { - "path": relpath if relpath else path, - "uri": unquote(link.url_without_fragment), - "link": link, - "editable": editable, - "setup_path": setup_path, - "uri_scheme": prefer, - "line": line - } - if link and link.is_wheel: - from pip_shims import Wheel - - arg_dict["name"] = Wheel(link.filename).name - elif name: - arg_dict["name"] = name - elif link.egg_fragment: - arg_dict["name"] = link.egg_fragment - return cls.create(**arg_dict) + def from_line(cls, line, editable=None, extras=None, parsed_line=None): + # type: (AnyStr, Optional[bool], Optional[Tuple[AnyStr, ...]], Optional[Line]) -> F + parsed_line = Line(line) + file_req_from_parsed_line(parsed_line) @classmethod def from_pipfile(cls, name, pipfile): + # type: (STRING_TYPE, Dict[STRING_TYPE, Union[Tuple[STRING_TYPE, ...], STRING_TYPE, bool]]) -> F # Parse the values out. After this dance we should have two variables: # path - Local filesystem path. # uri - Absolute URI that is parsable with urlsplit. @@ -536,7 +1868,7 @@ class FileRequirement(object): uri = pipfile.get("uri") fil = pipfile.get("file") path = pipfile.get("path") - if path: + if path and isinstance(path, six.string_types): if isinstance(path, Path) and not path.is_absolute(): path = get_converted_relative_path(path.as_posix()) elif not os.path.isabs(path): @@ -560,124 +1892,197 @@ class FileRequirement(object): if not uri: uri = pip_shims.shims.path_to_url(path) - link = create_link(uri) + link_info = None # type: Optional[LinkInfo] + if uri and isinstance(uri, six.string_types): + link_info = cls.get_link_from_line(uri) + else: + raise ValueError( + "Failed parsing requirement from pipfile: {0!r}".format(pipfile) + ) + link = None # type: Optional[Link] + if link_info: + link = link_info.link + if link.url_without_fragment: + uri = unquote(link.url_without_fragment) + extras = () # type: Optional[Tuple[STRING_TYPE, ...]] + if "extras" in pipfile: + extras = tuple(pipfile["extras"]) # type: ignore + editable = pipfile["editable"] if "editable" in pipfile else False arg_dict = { "name": name, "path": path, - "uri": unquote(link.url_without_fragment), - "editable": pipfile.get("editable", False), + "uri": uri, + "editable": editable, "link": link, "uri_scheme": uri_scheme, + "extras": extras if extras else None, } - if link.scheme != "file" and not pipfile.get("editable", False): - arg_dict["line"] = "{0}@ {1}".format(name, link.url_without_fragment) - return cls.create(**arg_dict) + + line = "" # type: STRING_TYPE + extras_string = "" if not extras else extras_to_string(extras) + if editable and uri_scheme == "path": + line = "{0}{1}".format(path, extras_string) + else: + if name: + line_name = "{0}{1}".format(name, extras_string) + line = "{0}#egg={1}".format(unquote(link.url_without_fragment), line_name) + else: + if link: + line = unquote(link.url) + elif uri and isinstance(uri, six.string_types): + line = uri + else: + raise ValueError( + "Failed parsing requirement from pipfile: {0!r}".format(pipfile) + ) + line = "{0}{1}".format(line, extras_string) + if "subdirectory" in pipfile: + arg_dict["subdirectory"] = pipfile["subdirectory"] + line = "{0}&subdirectory={1}".format(line, pipfile["subdirectory"]) + if editable: + line = "-e {0}".format(line) + arg_dict["parsed_line"] = Line(line) + arg_dict["setup_info"] = arg_dict["parsed_line"].setup_info + return cls(**arg_dict) # type: ignore @property def line_part(self): + # type: () -> STRING_TYPE + link_url = None # type: Optional[STRING_TYPE] + seed = None # type: Optional[STRING_TYPE] + if self.link is not None: + link_url = unquote(self.link.url_without_fragment) if self._uri_scheme and self._uri_scheme == "path": # We may need any one of these for passing to pip - seed = self.path or unquote(self.link.url_without_fragment) or self.uri + seed = self.path or link_url or self.uri elif (self._uri_scheme and self._uri_scheme == "file") or ( (self.link.is_artifact or self.link.is_wheel) and self.link.url ): - seed = unquote(self.link.url_without_fragment) or self.uri + seed = link_url or self.uri # add egg fragments to remote artifacts (valid urls only) - if not self._has_hashed_name and self.is_remote_artifact: + if not self._has_hashed_name and self.is_remote_artifact and seed is not None: seed += "#egg={0}".format(self.name) editable = "-e " if self.editable else "" + if seed is None: + raise ValueError("Could not calculate url for {0!r}".format(self)) return "{0}{1}".format(editable, seed) @property def pipfile_part(self): + # type: () -> Dict[AnyStr, Dict[AnyStr, Any]] excludes = [ - "_base_line", "_has_hashed_name", "setup_path", "pyproject_path", - "pyproject_requires", "pyproject_backend", "setup_info" + "_base_line", + "_has_hashed_name", + "setup_path", + "pyproject_path", + "_uri_scheme", + "pyproject_requires", + "pyproject_backend", + "_setup_info", + "_parsed_line", ] - filter_func = lambda k, v: bool(v) is True and k.name not in excludes - pipfile_dict = attr.asdict(self, filter=filter_func).copy() - name = pipfile_dict.pop("name") + filter_func = lambda k, v: bool(v) is True and k.name not in excludes # noqa + pipfile_dict = attr.asdict(self, filter=filter_func).copy() # type: Dict + name = pipfile_dict.pop("name", None) + if name is None: + if self.name: + name = self.name + elif self.parsed_line and self.parsed_line.name: + name = self.name = self.parsed_line.name + elif self.setup_info and self.setup_info.name: + name = self.name = self.setup_info.name if "_uri_scheme" in pipfile_dict: pipfile_dict.pop("_uri_scheme") # For local paths and remote installable artifacts (zipfiles, etc) collision_keys = {"file", "uri", "path"} + collision_order = ["file", "uri", "path"] # type: List[STRING_TYPE] + collisions = [] # type: List[STRING_TYPE] + key_match = next(iter(k for k in collision_order if k in pipfile_dict.keys())) if self._uri_scheme: dict_key = self._uri_scheme - target_key = ( - dict_key - if dict_key in pipfile_dict - else next( - (k for k in ("file", "uri", "path") if k in pipfile_dict), None - ) - ) - if target_key: + target_key = dict_key if dict_key in pipfile_dict else key_match + if target_key is not None: winning_value = pipfile_dict.pop(target_key) - collisions = (k for k in collision_keys if k in pipfile_dict) + collisions = [k for k in collision_keys if k in pipfile_dict] for key in collisions: pipfile_dict.pop(key) pipfile_dict[dict_key] = winning_value elif ( self.is_remote_artifact - or self.link.is_artifact + or (self.link is not None and self.link.is_artifact) and (self._uri_scheme and self._uri_scheme == "file") ): dict_key = "file" # Look for uri first because file is a uri format and this is designed # to make sure we add file keys to the pipfile as a replacement of uri - target_key = next( - (k for k in ("file", "uri", "path") if k in pipfile_dict), None - ) - winning_value = pipfile_dict.pop(target_key) + if key_match is not None: + winning_value = pipfile_dict.pop(key_match) key_to_remove = (k for k in collision_keys if k in pipfile_dict) for key in key_to_remove: pipfile_dict.pop(key) pipfile_dict[dict_key] = winning_value else: - collisions = [key for key in ["path", "file", "uri"] if key in pipfile_dict] + collisions = [key for key in collision_order if key in pipfile_dict.keys()] if len(collisions) > 1: for k in collisions[1:]: pipfile_dict.pop(k) return {name: pipfile_dict} -@attr.s(slots=True) +@attr.s(slots=True, hash=True) class VCSRequirement(FileRequirement): #: Whether the repository is editable - editable = attr.ib(default=None) + editable = attr.ib(default=None) # type: Optional[bool] #: URI for the repository - uri = attr.ib(default=None) + uri = attr.ib(default=None) # type: Optional[STRING_TYPE] #: path to the repository, if it's local - path = attr.ib(default=None, validator=attr.validators.optional(validate_path)) + path = attr.ib( + default=None, validator=attr.validators.optional(validate_path) + ) # type: Optional[STRING_TYPE] #: vcs type, i.e. git/hg/svn - vcs = attr.ib(validator=attr.validators.optional(validate_vcs), default=None) + vcs = attr.ib( + validator=attr.validators.optional(validate_vcs), default=None + ) # type: Optional[STRING_TYPE] #: vcs reference name (branch / commit / tag) - ref = attr.ib(default=None) + ref = attr.ib(default=None) # type: Optional[STRING_TYPE] #: Subdirectory to use for installation if applicable - subdirectory = attr.ib(default=None) - _repo = attr.ib(default=None) - _base_line = attr.ib(default=None) - name = attr.ib() - link = attr.ib() - req = attr.ib() + subdirectory = attr.ib(default=None) # type: Optional[STRING_TYPE] + _repo = attr.ib(default=None) # type: Optional[VCSRepository] + _base_line = attr.ib(default=None) # type: Optional[STRING_TYPE] + name = attr.ib() # type: STRING_TYPE + link = attr.ib() # type: Optional[pip_shims.shims.Link] + req = attr.ib() # type: Optional[RequirementType] def __attrs_post_init__(self): + # type: () -> None if not self.uri: if self.path: self.uri = pip_shims.shims.path_to_url(self.path) - split = urllib_parse.urlsplit(self.uri) - scheme, rest = split[0], split[1:] - vcs_type = "" - if "+" in scheme: - vcs_type, scheme = scheme.split("+", 1) - vcs_type = "{0}+".format(vcs_type) - new_uri = urllib_parse.urlunsplit((scheme,) + rest[:-1] + ("",)) - new_uri = "{0}{1}".format(vcs_type, new_uri) - self.uri = new_uri + if self.uri is not None: + split = urllib_parse.urlsplit(self.uri) + scheme, rest = split[0], split[1:] + vcs_type = "" + if "+" in scheme: + vcs_type, scheme = scheme.split("+", 1) + vcs_type = "{0}+".format(vcs_type) + new_uri = urllib_parse.urlunsplit((scheme,) + rest[:-1] + ("",)) + new_uri = "{0}{1}".format(vcs_type, new_uri) + self.uri = new_uri + + @property + def url(self): + # type: () -> STRING_TYPE + if self.link and self.link.url: + return self.link.url + elif self.uri: + return self.uri + raise ValueError("No valid url found for requirement {0!r}".format(self)) @link.default def get_link(self): + # type: () -> pip_shims.shims.Link uri = self.uri if self.uri else pip_shims.shims.path_to_url(self.path) - return build_vcs_link( + vcs_uri = build_vcs_uri( self.vcs, add_ssh_scheme_to_git_uri(uri), name=self.name, @@ -685,26 +2090,65 @@ class VCSRequirement(FileRequirement): subdirectory=self.subdirectory, extras=self.extras, ) + return self.get_link_from_line(vcs_uri).link @name.default def get_name(self): - return ( - self.link.egg_fragment or self.req.name - if self.req - else super(VCSRequirement, self).get_name() - ) + # type: () -> STRING_TYPE + if self.link and self.link.egg_fragment: + return self.link.egg_fragment + if self.req and self.req.name: + return self.req.name + return super(VCSRequirement, self).get_name() @property def vcs_uri(self): + # type: () -> Optional[STRING_TYPE] uri = self.uri - if not any(uri.startswith("{0}+".format(vcs)) for vcs in VCS_LIST): - uri = "{0}+{1}".format(self.vcs, uri) + if uri and not any(uri.startswith("{0}+".format(vcs)) for vcs in VCS_LIST): + if self.vcs: + uri = "{0}+{1}".format(self.vcs, uri) return uri + @property + def setup_info(self): + if self._parsed_line and self._parsed_line.setup_info: + if not self._parsed_line.setup_info.name: + self._parsed_line._setup_info.get_info() + return self._parsed_line.setup_info + if self._repo: + from .setup_info import SetupInfo + + self._setup_info = SetupInfo.from_ireq( + Line(self._repo.checkout_directory).ireq + ) + self._setup_info.get_info() + return self._setup_info + ireq = self.parsed_line.ireq + from .setup_info import SetupInfo + + self._setup_info = SetupInfo.from_ireq(ireq) + return self._setup_info + + @setup_info.setter + def setup_info(self, setup_info): + self._setup_info = setup_info + if self._parsed_line: + self._parsed_line.setup_info = setup_info + @req.default def get_requirement(self): - name = self.name or self.link.egg_fragment - url = self.uri or self.link.url_without_fragment + # type: () -> PackagingRequirement + name = None # type: Optional[STRING_TYPE] + if self.name: + name = self.name + elif self.link and self.link.egg_fragment: + name = self.link.egg_fragment + url = None + if self.uri: + url = self.uri + elif self.link is not None: + url = self.link.url_without_fragment if not name: raise ValueError( "pipenv requires an #egg fragment for version controlled " @@ -713,9 +2157,25 @@ class VCSRequirement(FileRequirement): ) req = init_requirement(canonicalize_name(self.name)) req.editable = self.editable - if not getattr(req, "url") and self.uri: - req.url = self.uri - req.line = self.link.url + if not getattr(req, "url"): + if url is not None: + url = add_ssh_scheme_to_git_uri(url) + elif self.uri is not None: + link = self.get_link_from_line(self.uri).link + if link: + url = link.url_without_fragment + if ( + url + and url.startswith("git+file:/") + and not url.startswith("git+file:///") + ): + url = url.replace("git+file:/", "git+file:///") + if url: + req.url = url + line = url if url else self.vcs_uri + if self.editable: + line = "-e {0}".format(line) + req.line = line if self.ref: req.revision = self.ref if self.extras: @@ -726,27 +2186,38 @@ class VCSRequirement(FileRequirement): req.path = self.path req.link = self.link if ( - self.uri != unquote(self.link.url_without_fragment) + self.link + and self.link.url_without_fragment + and self.uri + and self.uri != unquote(self.link.url_without_fragment) and "git+ssh://" in self.link.url and "git+git@" in self.uri ): req.line = self.uri - req.url = self.uri + url = self.link.url_without_fragment + if ( + url + and url.startswith("git+file:/") + and not url.startswith("git+file:///") + ): + url = url.replace("git+file:/", "git+file:///") + req.url = url return req - @property - def is_local(self): - if is_file_url(self.uri): - return True - return False - @property def repo(self): + # type: () -> VCSRepository if self._repo is None: - self._repo = self.get_vcs_repo() + if self._parsed_line and self._parsed_line.vcsrepo: + self._repo = self._parsed_line.vcsrepo + else: + self._repo = self.get_vcs_repo() + if self._parsed_line: + self._parsed_line.vcsrepo = self._repo return self._repo def get_checkout_dir(self, src_dir=None): + # type: (Optional[S]) -> STRING_TYPE src_dir = os.environ.get("PIP_SRC", None) if not src_dir else src_dir checkout_dir = None if self.is_local: @@ -756,22 +2227,20 @@ class VCSRequirement(FileRequirement): if path and os.path.exists(path): checkout_dir = os.path.abspath(path) return checkout_dir + if src_dir is not None: + checkout_dir = os.path.join(os.path.abspath(src_dir), self.name) + mkdir_p(src_dir) + return checkout_dir return os.path.join(create_tracked_tempdir(prefix="requirementslib"), self.name) - def get_vcs_repo(self, src_dir=None): + def get_vcs_repo(self, src_dir=None, checkout_dir=None): + # type: (Optional[STRING_TYPE], STRING_TYPE) -> VCSRepository from .vcs import VCSRepository - checkout_dir = self.get_checkout_dir(src_dir=src_dir) - link = build_vcs_link( - self.vcs, - self.uri, - name=self.name, - ref=self.ref, - subdirectory=self.subdirectory, - extras=self.extras, - ) + if checkout_dir is None: + checkout_dir = self.get_checkout_dir(src_dir=src_dir) vcsrepo = VCSRepository( - url=link.url, + url=self.url, name=self.name, ref=self.ref if self.ref else None, checkout_directory=checkout_dir, @@ -783,7 +2252,9 @@ class VCSRequirement(FileRequirement): pyproject_info = None if self.subdirectory: self.setup_path = os.path.join(checkout_dir, self.subdirectory, "setup.py") - self.pyproject_path = os.path.join(checkout_dir, self.subdirectory, "pyproject.toml") + self.pyproject_path = os.path.join( + checkout_dir, self.subdirectory, "pyproject.toml" + ) pyproject_info = get_pyproject(os.path.join(checkout_dir, self.subdirectory)) else: self.setup_path = os.path.join(checkout_dir, "setup.py") @@ -791,16 +2262,18 @@ class VCSRequirement(FileRequirement): pyproject_info = get_pyproject(checkout_dir) if pyproject_info is not None: pyproject_requires, pyproject_backend = pyproject_info - self.pyproject_requires = pyproject_requires + self.pyproject_requires = tuple(pyproject_requires) self.pyproject_backend = pyproject_backend return vcsrepo def get_commit_hash(self): + # type: () -> STRING_TYPE hash_ = None hash_ = self.repo.get_commit_hash() return hash_ def update_repo(self, src_dir=None, ref=None): + # type: (Optional[STRING_TYPE], Optional[STRING_TYPE]) -> STRING_TYPE if ref: self.ref = ref else: @@ -810,29 +2283,57 @@ class VCSRequirement(FileRequirement): if not self.is_local and ref is not None: self.repo.checkout_ref(ref) repo_hash = self.repo.get_commit_hash() - self.req.revision = repo_hash + if self.req: + self.req.revision = repo_hash return repo_hash @contextmanager def locked_vcs_repo(self, src_dir=None): + # type: (Optional[AnyStr]) -> Generator[VCSRepository, None, None] if not src_dir: src_dir = create_tracked_tempdir(prefix="requirementslib-", suffix="-src") vcsrepo = self.get_vcs_repo(src_dir=src_dir) - self.req.revision = vcsrepo.get_commit_hash() + if not self.req: + if self.parsed_line is not None: + self.req = self.parsed_line.requirement + else: + self.req = self.get_requirement() + revision = self.req.revision = vcsrepo.get_commit_hash() # Remove potential ref in the end of uri after ref is parsed - if "@" in self.link.show_url and "@" in self.uri: - uri, ref = self.uri.rsplit("@", 1) - checkout = self.req.revision - if checkout and ref in checkout: + if self.link and "@" in self.link.show_url and self.uri and "@" in self.uri: + uri, ref = split_ref_from_uri(self.uri) + checkout = revision + if checkout and ref and ref in checkout: self.uri = uri - - yield vcsrepo + orig_repo = self._repo self._repo = vcsrepo + if self._parsed_line: + self._parsed_line.vcsrepo = vcsrepo + if self._setup_info: + self._setup_info = attr.evolve( + self._setup_info, + requirements=(), + _extras_requirements=(), + build_requires=(), + setup_requires=(), + version=None, + metadata=None, + ) + if self.parsed_line and self._parsed_line: + self._parsed_line.vcsrepo = vcsrepo + if self.req and not self.editable: + self.req.specifier = SpecifierSet("=={0}".format(self.setup_info.version)) + try: + yield self._repo + except Exception: + self._repo = orig_repo + raise @classmethod def from_pipfile(cls, name, pipfile): - creation_args = {} + # type: (STRING_TYPE, Dict[S, Union[Tuple[S, ...], S, bool]]) -> F + creation_args = {} # type: Dict[STRING_TYPE, CREATION_ARG_TYPES] pipfile_keys = [ k for k in ( @@ -848,153 +2349,221 @@ class VCSRequirement(FileRequirement): + VCS_LIST if k in pipfile ] + # extras = None # type: Optional[Tuple[STRING_TYPE, ...]] for key in pipfile_keys: - if key == "extras": - extras = pipfile.get(key, None) - if extras: - pipfile[key] = sorted(dedup([extra.lower() for extra in extras])) - if key in VCS_LIST: - creation_args["vcs"] = key - target = pipfile.get(key) - drive, path = os.path.splitdrive(target) - if ( - not drive - and not os.path.exists(target) - and ( - is_valid_url(target) - or is_file_url(target) - or target.startswith("git@") - ) - ): - creation_args["uri"] = target + if key == "extras" and key in pipfile: + extras = pipfile[key] + if isinstance(extras, (list, tuple)): + pipfile[key] = tuple(sorted({extra.lower() for extra in extras})) else: - creation_args["path"] = target - if os.path.isabs(target): - creation_args["uri"] = pip_shims.shims.path_to_url(target) - else: - creation_args[key] = pipfile.get(key) + pipfile[key] = extras + if key in VCS_LIST and key in pipfile_keys: + creation_args["vcs"] = key + target = pipfile[key] + if isinstance(target, six.string_types): + drive, path = os.path.splitdrive(target) + if ( + not drive + and not os.path.exists(target) + and ( + is_valid_url(target) + or is_file_url(target) + or target.startswith("git@") + ) + ): + creation_args["uri"] = target + else: + creation_args["path"] = target + if os.path.isabs(target): + creation_args["uri"] = pip_shims.shims.path_to_url(target) + elif key in pipfile_keys: + creation_args[key] = pipfile[key] creation_args["name"] = name - return cls(**creation_args) + cls_inst = cls(**creation_args) # type: ignore + return cls_inst @classmethod - def from_line(cls, line, editable=None, extras=None): - relpath = None - if line.startswith("-e "): - editable = True - line = line.split(" ", 1)[1] - vcs_type, prefer, relpath, path, uri, link = cls.get_link_from_line(line) - if not extras and link.egg_fragment: - name, extras = pip_shims.shims._strip_extras(link.egg_fragment) - if extras: - extras = parse_extras(extras) - else: - name = link.egg_fragment - subdirectory = link.subdirectory_fragment - ref = None - if "@" in link.path and "@" in uri: - uri, _, ref = uri.rpartition("@") - if relpath and "@" in relpath: - relpath, ref = relpath.rsplit("@", 1) - return cls( - name=name, - ref=ref, - vcs=vcs_type, - subdirectory=subdirectory, - link=link, - path=relpath or path, - editable=editable, - uri=uri, - extras=extras, - base_line=line, - ) + def from_line(cls, line, editable=None, extras=None, parsed_line=None): + # type: (AnyStr, Optional[bool], Optional[Tuple[AnyStr, ...]], Optional[Line]) -> F + parsed_line = Line(line) + return vcs_req_from_parsed_line(parsed_line) @property def line_part(self): + # type: () -> STRING_TYPE """requirements.txt compatible line part sans-extras""" + base = "" # type: STRING_TYPE if self.is_local: base_link = self.link if not self.link: base_link = self.get_link() - final_format = ( - "{{0}}#egg={0}".format(base_link.egg_fragment) - if base_link.egg_fragment - else "{0}" - ) + if base_link and base_link.egg_fragment: + final_format = "{{0}}#egg={0}".format(base_link.egg_fragment) + else: + final_format = "{0}" base = final_format.format(self.vcs_uri) - elif self._base_line: + elif self._parsed_line is not None and ( + self._parsed_line.is_direct_url and self._parsed_line.line_with_prefix + ): + return self._parsed_line.line_with_prefix + elif getattr(self, "_base_line", None) and ( + isinstance(self._base_line, six.string_types) + ): base = self._base_line else: - base = self.link.url - if base and self.extras and not extras_to_string(self.extras) in base: + base = getattr(self, "link", self.get_link()).url + if base and self.extras and extras_to_string(self.extras) not in base: if self.subdirectory: base = "{0}".format(self.get_link().url) else: base = "{0}{1}".format(base, extras_to_string(sorted(self.extras))) - if self.editable: + if "git+file:/" in base and "git+file:///" not in base: + base = base.replace("git+file:/", "git+file:///") + if self.editable and not base.startswith("-e "): base = "-e {0}".format(base) return base @staticmethod def _choose_vcs_source(pipfile): + # type: (Dict[S, Union[S, Any]]) -> Dict[S, Union[S, Any]] src_keys = [k for k in pipfile.keys() if k in ["path", "uri", "file"]] + vcs_type = "" # type: Optional[STRING_TYPE] + alt_type = "" # type: Optional[STRING_TYPE] + vcs_value = "" # type: STRING_TYPE if src_keys: chosen_key = first(src_keys) vcs_type = pipfile.pop("vcs") - _, pipfile_url = split_vcs_method_from_uri(pipfile.get(chosen_key)) - pipfile[vcs_type] = pipfile_url + if chosen_key in pipfile: + vcs_value = pipfile[chosen_key] + alt_type, pipfile_url = split_vcs_method_from_uri(vcs_value) + if vcs_type is None: + vcs_type = alt_type + if vcs_type and pipfile_url: + pipfile[vcs_type] = pipfile_url for removed in src_keys: pipfile.pop(removed) return pipfile @property def pipfile_part(self): + # type: () -> Dict[S, Dict[S, Union[List[S], S, bool, RequirementType, pip_shims.shims.Link]]] excludes = [ - "_repo", "_base_line", "setup_path", "_has_hashed_name", "pyproject_path", - "pyproject_requires", "pyproject_backend", "setup_info" + "_repo", + "_base_line", + "setup_path", + "_has_hashed_name", + "pyproject_path", + "pyproject_requires", + "pyproject_backend", + "_setup_info", + "_parsed_line", + "_uri_scheme", ] - filter_func = lambda k, v: bool(v) is True and k.name not in excludes + filter_func = lambda k, v: bool(v) is True and k.name not in excludes # noqa pipfile_dict = attr.asdict(self, filter=filter_func).copy() + name = pipfile_dict.pop("name", None) + if name is None: + if self.name: + name = self.name + elif self.parsed_line and self.parsed_line.name: + name = self.name = self.parsed_line.name + elif self.setup_info and self.setup_info.name: + name = self.name = self.setup_info.name if "vcs" in pipfile_dict: pipfile_dict = self._choose_vcs_source(pipfile_dict) - name, _ = pip_shims.shims._strip_extras(pipfile_dict.pop("name")) - return {name: pipfile_dict} + name, _ = pip_shims.shims._strip_extras(name) + return {name: pipfile_dict} # type: ignore -@attr.s +@attr.s(cmp=True, hash=True) class Requirement(object): - name = attr.ib() - vcs = attr.ib(default=None, validator=attr.validators.optional(validate_vcs)) - req = attr.ib(default=None) - markers = attr.ib(default=None) - specifiers = attr.ib(validator=attr.validators.optional(validate_specifiers)) - index = attr.ib(default=None) - editable = attr.ib(default=None) - hashes = attr.ib(default=attr.Factory(list), converter=list) - extras = attr.ib(default=attr.Factory(list)) - abstract_dep = attr.ib(default=None) - _ireq = None + _name = attr.ib(cmp=True) # type: STRING_TYPE + vcs = attr.ib( + default=None, validator=attr.validators.optional(validate_vcs), cmp=True + ) # type: Optional[STRING_TYPE] + req = attr.ib( + default=None, cmp=True + ) # type: Optional[Union[VCSRequirement, FileRequirement, NamedRequirement]] + markers = attr.ib(default=None, cmp=True) # type: Optional[STRING_TYPE] + _specifiers = attr.ib( + validator=attr.validators.optional(validate_specifiers), cmp=True + ) # type: Optional[STRING_TYPE] + index = attr.ib(default=None, cmp=True) # type: Optional[STRING_TYPE] + editable = attr.ib(default=None, cmp=True) # type: Optional[bool] + hashes = attr.ib( + factory=frozenset, converter=frozenset, cmp=True + ) # type: FrozenSet[STRING_TYPE] + extras = attr.ib(factory=tuple, cmp=True) # type: Tuple[STRING_TYPE, ...] + abstract_dep = attr.ib(default=None, cmp=False) # type: Optional[AbstractDependency] + _line_instance = attr.ib(default=None, cmp=False) # type: Optional[Line] + _ireq = attr.ib( + default=None, cmp=False + ) # type: Optional[pip_shims.InstallRequirement] - @name.default + def __hash__(self): + return hash(self.as_line()) + + @_name.default def get_name(self): - return self.req.name + # type: () -> Optional[STRING_TYPE] + if self.req is not None: + return self.req.name + return None + + @property + def name(self): + # type: () -> Optional[STRING_TYPE] + if self._name is not None: + return self._name + name = None + if self.req and self.req.name: + name = self.req.name + elif self.req and self.is_file_or_url and self.req.setup_info: + name = self.req.setup_info.name + self._name = name + return name @property def requirement(self): - return self.req.req + # type: () -> Optional[PackagingRequirement] + if self.req: + return self.req.req + return None + + def add_hashes(self, hashes): + # type: (Union[S, List[S], Set[S], Tuple[S, ...]]) -> Requirement + new_hashes = set() # type: Set[STRING_TYPE] + if self.hashes is not None: + new_hashes |= set(self.hashes) + if isinstance(hashes, six.string_types): + new_hashes.add(hashes) + else: + new_hashes |= set(hashes) + return attr.evolve(self, hashes=tuple(new_hashes)) def get_hashes_as_pip(self, as_list=False): - if self.hashes: - if as_list: - return [HASH_STRING.format(h) for h in self.hashes] - return "".join([HASH_STRING.format(h) for h in self.hashes]) - return "" if not as_list else [] + # type: (bool) -> Union[STRING_TYPE, List[STRING_TYPE]] + hashes = "" # type: Union[STRING_TYPE, List[STRING_TYPE]] + if as_list: + hashes = [] + if self.hashes: + hashes = [HASH_STRING.format(h) for h in self.hashes] + else: + hashes = "" + if self.hashes: + hashes = "".join([HASH_STRING.format(h) for h in self.hashes]) + return hashes @property def hashes_as_pip(self): - self.get_hashes_as_pip() + # type: () -> STRING_TYPE + hashes = self.get_hashes_as_pip() + assert isinstance(hashes, six.string_types) + return hashes @property def markers_as_pip(self): + # type: () -> S if self.markers: return " ; {0}".format(self.markers).replace('"', "'") @@ -1002,141 +2571,248 @@ class Requirement(object): @property def extras_as_pip(self): + # type: () -> STRING_TYPE if self.extras: return "[{0}]".format( - ",".join(sorted([extra.lower() for extra in self.extras])) + ",".join(sorted([extra.lower() for extra in self.extras])) # type: ignore ) return "" - @property + @cached_property def commit_hash(self): - if not self.is_vcs: + # type: () -> Optional[S] + if self.req is None or not isinstance(self.req, VCSRequirement): return None commit_hash = None - with self.req.locked_vcs_repo() as repo: - commit_hash = repo.get_commit_hash() + if self.req is not None: + with self.req.locked_vcs_repo() as repo: + commit_hash = repo.get_commit_hash() return commit_hash - @specifiers.default + @_specifiers.default def get_specifiers(self): - if self.req and self.req.req.specifier: + # type: () -> S + if self.req and self.req.req and self.req.req.specifier: return specs_to_string(self.req.req.specifier) - return + return "" + + def update_name_from_path(self, path): + metadata = get_metadata(path) + name = self.name + if metadata is not None: + metadata_name = metadata.get("name") + if metadata_name and metadata_name != "wheel": + name = metadata_name + if name is not None: + if self.req.name is None: + self.req.name = name + if self.req.req and self.req.req.name is None: + self.req.req.name = name + if self._line_instance._name is None: + self._line_instance.name = name + if self.req._parsed_line._name is None: + self.req._parsed_line.name = name + if self.req._setup_info and self.req._setup_info.name is None: + self.req._setup_info.name = name + + def get_line_instance(self): + # type: () -> Line + line_parts = [] + if self.req: + if self.req.line_part.startswith("-e "): + line_parts.extend(self.req.line_part.split(" ", 1)) + else: + line_parts.append(self.req.line_part) + if not self.is_vcs and not self.vcs and self.extras_as_pip: + line_parts.append(self.extras_as_pip) + if self._specifiers and not (self.is_file_or_url or self.is_vcs): + line_parts.append(self._specifiers) + if self.markers: + line_parts.append("; {0}".format(self.markers.replace('"', "'"))) + if self.hashes_as_pip and not (self.editable or self.vcs or self.is_vcs): + line_parts.append(self.hashes_as_pip) + if self.editable: + if line_parts[0] == "-e": + line = "".join(line_parts[1:]) + else: + line = "".join(line_parts) + if self.markers: + line = '"{0}"'.format(line) + line = "-e {0}".format(line) + else: + line = "".join(line_parts) + return Line(line) + + @property + def line_instance(self): + # type: () -> Optional[Line] + if self._line_instance is None: + self.line_instance = self.get_line_instance() + return self._line_instance + + @line_instance.setter + def line_instance(self, line_instance): + # type: (Line) -> None + if self.req: + self.req._parsed_line = line_instance + self._line_instance = line_instance + + @property + def specifiers(self): + # type: () -> Optional[STRING_TYPE] + if self._specifiers: + return self._specifiers + else: + specs = self.get_specifiers() + if specs: + self._specifiers = specs + return specs + if not self._specifiers and ( + self.req is not None + and isinstance(self.req, NamedRequirement) + and self.req.version + ): + self._specifiers = self.req.version + elif ( + not self.editable + and self.req + and (not isinstance(self.req, NamedRequirement) and self.req.setup_info) + ): + if ( + self.line_instance + and self.line_instance.setup_info + and self.line_instance.setup_info.version + ): + self._specifiers = "=={0}".format(self.req.setup_info.version) + elif not self._specifiers: + if self.req and self.req.parsed_line and self.req.parsed_line.specifiers: + self._specifiers = specs_to_string(self.req.parsed_line.specifiers) + elif self.line_instance and self.line_instance.specifiers: + self._specifiers = specs_to_string(self.line_instance.specifiers) + elif self.is_file_or_url or self.is_vcs: + try: + setupinfo_dict = self.run_requires() + except Exception: + setupinfo_dict = None + if setupinfo_dict is not None: + self._specifiers = "=={0}".format(setupinfo_dict.get("version")) + if self._specifiers: + specset = SpecifierSet(self._specifiers) + if self.line_instance and not self.line_instance.specifiers: + self.line_instance.specifiers = specset + if self.req: + if self.req._parsed_line and not self.req._parsed_line.specifiers: + self.req._parsed_line.specifiers = specset + elif not self.req._parsed_line and self.line_instance: + self.req._parsed_line = self.line_instance + if self.req and self.req.req and not self.req.req.specifier: + self.req.req.specifier = specset + return self._specifiers @property def is_vcs(self): + # type: () -> bool return isinstance(self.req, VCSRequirement) + @property + def build_backend(self): + # type: () -> Optional[STRING_TYPE] + if self.req is not None and ( + not isinstance(self.req, NamedRequirement) and self.req.is_local + ): + setup_info = self.run_requires() + build_backend = setup_info.get("build_backend") + return build_backend + return "setuptools.build_meta" + + @property + def uses_pep517(self): + # type: () -> bool + if self.build_backend: + return True + return False + @property def is_file_or_url(self): + # type: () -> bool return isinstance(self.req, FileRequirement) @property def is_named(self): + # type: () -> bool return isinstance(self.req, NamedRequirement) + @property + def is_wheel(self): + # type: () -> bool + if ( + self.req + and not isinstance(self.req, NamedRequirement) + and (self.req.link is not None and self.req.link.is_wheel) + ): + return True + return False + @property def normalized_name(self): + # type: () -> S return canonicalize_name(self.name) def copy(self): return attr.evolve(self) @classmethod + @lru_cache() def from_line(cls, line): + # type: (AnyStr) -> Requirement if isinstance(line, pip_shims.shims.InstallRequirement): line = format_requirement(line) - hashes = None - if "--hash=" in line: - hashes = line.split(" --hash=") - line, hashes = hashes[0], hashes[1:] - editable = line.startswith("-e ") - line = line.split(" ", 1)[1] if editable else line - line, markers = split_markers_from_line(line) - line, extras = pip_shims.shims._strip_extras(line) - if extras: - extras = parse_extras(extras) - line = line.strip('"').strip("'").strip() - line_with_prefix = "-e {0}".format(line) if editable else line - vcs = None - # Installable local files and installable non-vcs urls are handled - # as files, generally speaking - line_is_vcs = is_vcs(line) - # check for pep-508 compatible requirements - name, _, possible_url = line.partition("@") - if is_installable_file(line) or ( - (is_valid_url(possible_url) or is_file_url(line) or is_valid_url(line)) and - not (line_is_vcs or is_vcs(possible_url)) - ): - r = FileRequirement.from_line(line_with_prefix) - elif line_is_vcs: - r = VCSRequirement.from_line(line_with_prefix, extras=extras) - vcs = r.vcs + parsed_line = Line(line) + r = ( + None + ) # type: Optional[Union[VCSRequirement, FileRequirement, NamedRequirement]] + if ( + (parsed_line.is_file and parsed_line.is_installable) or parsed_line.is_url + ) and not parsed_line.is_vcs: + r = file_req_from_parsed_line(parsed_line) + elif parsed_line.is_vcs: + r = vcs_req_from_parsed_line(parsed_line) elif line == "." and not is_installable_file(line): raise RequirementError( "Error parsing requirement %s -- are you sure it is installable?" % line ) else: - specs = "!=<>~" - spec_matches = set(specs) & set(line) - version = None - name = line - if spec_matches: - spec_idx = min((line.index(match) for match in spec_matches)) - name = line[:spec_idx] - version = line[spec_idx:] - if not extras: - name, extras = pip_shims.shims._strip_extras(name) - if extras: - extras = parse_extras(extras) - if version: - name = "{0}{1}".format(name, version) - r = NamedRequirement.from_line(line) + r = named_req_from_parsed_line(parsed_line) req_markers = None - if markers: - req_markers = PackagingRequirement("fakepkg; {0}".format(markers)) - r.req.marker = getattr(req_markers, "marker", None) if req_markers else None - r.req.local_file = getattr(r.req, "local_file", False) - name = getattr(r.req, "name", None) - if not name: - name = getattr(r.req, "project_name", None) - r.req.name = name - if not name: - name = getattr(r.req, "key", None) - if name: - r.req.name = name + if parsed_line.markers: + req_markers = PackagingRequirement("fakepkg; {0}".format(parsed_line.markers)) + if r is not None and r.req is not None: + r.req.marker = getattr(req_markers, "marker", None) if req_markers else None + args = {} # type: Dict[STRING_TYPE, CREATION_ARG_TYPES] args = { "name": r.name, - "vcs": vcs, + "vcs": parsed_line.vcs, "req": r, - "markers": markers, - "editable": editable, + "markers": parsed_line.markers, + "editable": parsed_line.editable, + "line_instance": parsed_line, } - if extras: - extras = sorted(dedup([extra.lower() for extra in extras])) + if parsed_line.extras: + extras = () # type: Tuple[STRING_TYPE, ...] + extras = tuple(sorted(dedup([extra.lower() for extra in parsed_line.extras]))) args["extras"] = extras - r.req.extras = extras - r.extras = extras - elif r.extras: - args["extras"] = sorted(dedup([extra.lower() for extra in r.extras])) - if hashes: - args["hashes"] = hashes - cls_inst = cls(**args) - if not cls_inst.is_named and not cls_inst.editable and not name: - if cls_inst.is_vcs: - ireq = pip_shims.shims.install_req_from_req(cls_inst.as_line(include_hashes=False)) - info = SetupInfo.from_ireq(ireq) - if info is not None: - info_dict = info.as_dict() - cls_inst.req.setup_info = info - else: - info_dict = {} - else: - info_dict = cls_inst.run_requires() - found_name = info_dict.get("name", old_name) - if old_name != found_name: - cls_inst.req.req.line.replace(old_name, found_name) + if r is not None: + r.extras = extras + elif r is not None and r.extras is not None: + args["extras"] = tuple( + sorted(dedup([extra.lower() for extra in r.extras])) + ) # type: ignore + if r.req is not None: + r.req.extras = args["extras"] + if parsed_line.hashes: + args["hashes"] = tuple(parsed_line.hashes) # type: ignore + cls_inst = cls(**args) # type: ignore return cls_inst @classmethod @@ -1170,41 +2846,29 @@ class Requirement(object): if markers: markers = str(markers) req_markers = PackagingRequirement("fakepkg; {0}".format(markers)) - r.req.marker = getattr(req_markers, "marker", None) - r.req.specifier = SpecifierSet(_pipfile["version"]) + if r.req is not None: + r.req.marker = req_markers.marker extras = _pipfile.get("extras") - r.req.extras = ( - sorted(dedup([extra.lower() for extra in extras])) if extras else [] - ) + if r.req: + if r.req.specifier: + r.req.specifier = SpecifierSet(_pipfile["version"]) + r.req.extras = ( + tuple(sorted(dedup([extra.lower() for extra in extras]))) + if extras + else () + ) args = { "name": r.name, "vcs": vcs, "req": r, "markers": markers, - "extras": _pipfile.get("extras"), + "extras": tuple(_pipfile.get("extras", ())), "editable": _pipfile.get("editable", False), "index": _pipfile.get("index"), } if any(key in _pipfile for key in ["hash", "hashes"]): args["hashes"] = _pipfile.get("hashes", [pipfile.get("hash")]) cls_inst = cls(**args) - if cls_inst.is_named: - cls_inst.req.req.line = cls_inst.as_line() - old_name = cls_inst.req.req.name or cls_inst.req.name - if not cls_inst.is_named and not cls_inst.editable and not name: - if cls_inst.is_vcs: - ireq = pip_shims.shims.install_req_from_req(cls_inst.as_line(include_hashes=False)) - info = SetupInfo.from_ireq(ireq) - if info is not None: - info_dict = info.as_dict() - cls_inst.req.setup_info = info - else: - info_dict = {} - else: - info_dict = cls_inst.run_requires() - found_name = info_dict.get("name", old_name) - if old_name != found_name: - cls_inst.req.req.line.replace(old_name, found_name) return cls_inst def as_line( @@ -1224,27 +2888,14 @@ class Requirement(object): in the requirement line. """ - include_specifiers = True if self.specifiers else False - if self.is_vcs: - include_extras = False - if self.is_file_or_url or self.is_vcs: - include_specifiers = False - parts = [ - self.req.line_part, - self.extras_as_pip if include_extras else "", - self.specifiers if include_specifiers else "", - self.markers_as_pip if include_markers else "", - ] - if as_list: - # This is used for passing to a subprocess call - parts = ["".join(parts)] - if include_hashes: - hashes = self.get_hashes_as_pip(as_list=as_list) - if as_list: - parts.extend(hashes) - else: - parts.append(hashes) - if sources and not (self.requirement.local_file or self.vcs): + assert self.line_instance is not None + parts = self.line_instance.get_line( + with_prefix=True, + with_hashes=include_hashes, + with_markers=include_markers, + as_list=as_list, + ) + if sources and self.requirement and not (self.line_instance.is_local or self.vcs): from ..utils import prepare_pip_source_args if self.index: @@ -1254,20 +2905,19 @@ class Requirement(object): parts.extend(sources) else: index_string = " ".join(source_list) - parts.extend([" ", index_string]) - if as_list: - return parts - line = "".join(parts) - return line + parts = "{0} {1}".format(parts, index_string) + return parts def get_markers(self): + # type: () -> Marker markers = self.markers if markers: fake_pkg = PackagingRequirement("fakepkg; {0}".format(markers)) - markers = fake_pkg.markers + markers = fake_pkg.marker return markers def get_specifier(self): + # type: () -> Union[SpecifierSet, LegacySpecifier] try: return Specifier(self.specifiers) except InvalidSpecifier: @@ -1295,7 +2945,11 @@ class Requirement(object): @property def is_direct_url(self): - return self.is_file_or_url and self.req.is_direct_url + return ( + self.is_file_or_url + and self.req.is_direct_url + or (self.line_instance.is_direct_url or self.req.parsed_line.is_direct_url) + ) def as_pipfile(self): good_keys = ( @@ -1314,10 +2968,20 @@ class Requirement(object): name = self.name if "markers" in req_dict and req_dict["markers"]: req_dict["markers"] = req_dict["markers"].replace('"', "'") + if not self.req.name: + name_carriers = (self.req, self, self.line_instance, self.req.parsed_line) + name_options = [ + getattr(carrier, "name", None) + for carrier in name_carriers + if carrier is not None + ] + req_name = next(iter(n for n in name_options if n is not None), None) + self.req.name = req_name + req_name, dict_from_subreq = self.req.pipfile_part.popitem() base_dict = { k: v - for k, v in self.req.pipfile_part[name].items() - if k not in ["req", "link"] + for k, v in dict_from_subreq.items() + if k not in ["req", "link", "_setup_info"] } base_dict.update(req_dict) conflicting_keys = ("file", "path", "uri") @@ -1334,24 +2998,31 @@ class Requirement(object): except AttributeError: hashes.append(_hash) base_dict["hashes"] = sorted(hashes) + if "extras" in base_dict: + base_dict["extras"] = list(base_dict["extras"]) if len(base_dict.keys()) == 1 and "version" in base_dict: base_dict = base_dict.get("version") return {name: base_dict} def as_ireq(self): - ireq_line = self.as_line(include_hashes=False) - if self.editable or self.req.editable: - if ireq_line.startswith("-e "): - ireq_line = ireq_line[len("-e ") :] - with ensure_setup_py(self.req.setup_path): - ireq = pip_shims.shims.install_req_from_editable(ireq_line) - else: - ireq = pip_shims.shims.install_req_from_line(ireq_line) + if self.line_instance and self.line_instance.ireq: + return self.line_instance.ireq + elif getattr(self.req, "_parsed_line", None) and self.req._parsed_line.ireq: + return self.req._parsed_line.ireq + kwargs = {"include_hashes": False} + if (self.is_file_or_url and self.req.is_local) or self.is_vcs: + kwargs["include_markers"] = False + ireq_line = self.as_line(**kwargs) + ireq = Line(ireq_line).ireq if not getattr(ireq, "req", None): ireq.req = self.req.req + if (self.is_file_or_url and self.req.is_local) or self.is_vcs: + if getattr(ireq, "req", None) and getattr(ireq.req, "marker", None): + ireq.req.marker = None else: ireq.req.extras = self.req.req.extras - ireq.req.marker = self.req.req.marker + if not ((self.is_file_or_url and self.req.is_local) or self.is_vcs): + ireq.req.marker = self.req.req.marker return ireq @property @@ -1411,17 +3082,17 @@ class Requirement(object): else: ireq = sorted(self.find_all_matches(), key=lambda k: k.version) deps = get_dependencies(ireq.pop(), sources=sources) - return get_abstract_dependencies( - deps, sources=sources, parent=self.abstract_dep - ) + return get_abstract_dependencies(deps, sources=sources, parent=self.abstract_dep) def find_all_matches(self, sources=None, finder=None): + # type: (Optional[List[Dict[S, Union[S, bool]]]], Optional[PackageFinder]) -> List[InstallationCandidate] """Find all matching candidates for the current requirement. Consults a finder to find all matching candidates. :param sources: Pipfile-formatted sources, defaults to None :param sources: list[dict], optional + :param PackageFinder finder: A **PackageFinder** instance from pip's repository implementation :return: A list of Installation Candidates :rtype: list[ :class:`~pip._internal.index.InstallationCandidate` ] """ @@ -1435,17 +3106,21 @@ class Requirement(object): def run_requires(self, sources=None, finder=None): if self.req and self.req.setup_info is not None: info_dict = self.req.setup_info.as_dict() + elif self.line_instance and self.line_instance.setup_info is not None: + info_dict = self.line_instance.setup_info.as_dict() else: from .setup_info import SetupInfo + if not finder: from .dependencies import get_finder + finder = get_finder(sources=sources) info = SetupInfo.from_requirement(self, finder=finder) if info is None: return {} info_dict = info.get_info() if self.req and not self.req.setup_info: - self.req.setup_info = info + self.req._setup_info = info if self.req._has_hashed_name and info_dict.get("name"): self.req.name = self.name = info_dict["name"] if self.req.req.name != info_dict["name"]: @@ -1453,10 +3128,121 @@ class Requirement(object): return info_dict def merge_markers(self, markers): + # type: (Union[AnyStr, Marker]) -> None + if not markers: + return self if not isinstance(markers, Marker): markers = Marker(markers) - _markers = set(Marker(self.ireq.markers)) if self.ireq.markers else set(markers) - _markers.add(markers) - new_markers = Marker(" or ".join([str(m) for m in sorted(_markers)])) - self.markers = str(new_markers) - self.req.req.marker = new_markers + _markers = [] # type: List[Marker] + ireq = self.as_ireq() + if ireq and ireq.markers: + ireq_marker = ireq.markers + _markers.append(str(ireq_marker)) + _markers.append(str(markers)) + marker_str = " and ".join([normalize_marker_str(m) for m in _markers if m]) + new_marker = Marker(marker_str) + line = copy.deepcopy(self._line_instance) + line.markers = marker_str + line.parsed_marker = new_marker + if getattr(line, "_requirement", None) is not None: + line._requirement.marker = new_marker + if getattr(line, "_ireq", None) is not None and line._ireq.req: + line._ireq.req.marker = new_marker + new_ireq = getattr(self, "ireq", None) + if new_ireq and new_ireq.req: + new_ireq.req.marker = new_marker + req = self.req + if req.req: + req_requirement = req.req + req_requirement.marker = new_marker + req = attr.evolve(req, req=req_requirement, parsed_line=line) + return attr.evolve( + self, markers=str(new_marker), ireq=new_ireq, req=req, line_instance=line + ) + + +def file_req_from_parsed_line(parsed_line): + # type: (Line) -> FileRequirement + path = parsed_line.relpath if parsed_line.relpath else parsed_line.path + pyproject_requires = None # type: Optional[Tuple[STRING_TYPE, ...]] + if parsed_line.pyproject_requires is not None: + pyproject_requires = tuple(parsed_line.pyproject_requires) + req_dict = { + "setup_path": parsed_line.setup_py, + "path": path, + "editable": parsed_line.editable, + "extras": parsed_line.extras, + "uri_scheme": parsed_line.preferred_scheme, + "link": parsed_line.link, + "uri": parsed_line.uri, + "pyproject_requires": pyproject_requires, + "pyproject_backend": parsed_line.pyproject_backend, + "pyproject_path": Path(parsed_line.pyproject_toml) + if parsed_line.pyproject_toml + else None, + "parsed_line": parsed_line, + "req": parsed_line.requirement, + } + if parsed_line.name is not None: + req_dict["name"] = parsed_line.name + return FileRequirement(**req_dict) # type: ignore + + +def vcs_req_from_parsed_line(parsed_line): + # type: (Line) -> VCSRequirement + line = "{0}".format(parsed_line.line) + if parsed_line.editable: + line = "-e {0}".format(line) + if parsed_line.url is not None: + link = create_link( + build_vcs_uri( + vcs=parsed_line.vcs, + uri=parsed_line.url, + name=parsed_line.name, + ref=parsed_line.ref, + subdirectory=parsed_line.subdirectory, + extras=list(parsed_line.extras), + ) + ) + else: + link = parsed_line.link + pyproject_requires = () # type: Optional[Tuple[STRING_TYPE, ...]] + if parsed_line.pyproject_requires is not None: + pyproject_requires = tuple(parsed_line.pyproject_requires) + vcs_dict = { + "setup_path": parsed_line.setup_py, + "path": parsed_line.path, + "editable": parsed_line.editable, + "vcs": parsed_line.vcs, + "ref": parsed_line.ref, + "subdirectory": parsed_line.subdirectory, + "extras": parsed_line.extras, + "uri_scheme": parsed_line.preferred_scheme, + "link": link, + "uri": parsed_line.uri, + "pyproject_requires": pyproject_requires, + "pyproject_backend": parsed_line.pyproject_backend, + "pyproject_path": Path(parsed_line.pyproject_toml) + if parsed_line.pyproject_toml + else None, + "parsed_line": parsed_line, + "req": parsed_line.requirement, + "base_line": line, + } + if parsed_line.name: + vcs_dict["name"] = parsed_line.name + return VCSRequirement(**vcs_dict) # type: ignore + + +def named_req_from_parsed_line(parsed_line): + # type: (Line) -> NamedRequirement + if parsed_line.name is not None: + return NamedRequirement( + name=parsed_line.name, + version=parsed_line.specifier, + req=parsed_line.requirement, + extras=parsed_line.extras, + editable=parsed_line.editable, + parsed_line=parsed_line, + ) + return NamedRequirement.from_line(parsed_line.line) diff --git a/pipenv/vendor/requirementslib/models/setup_info.py b/pipenv/vendor/requirementslib/models/setup_info.py index c32c2790..454cbfcc 100644 --- a/pipenv/vendor/requirementslib/models/setup_info.py +++ b/pipenv/vendor/requirementslib/models/setup_info.py @@ -1,27 +1,51 @@ # -*- coding=utf-8 -*- +from __future__ import absolute_import, print_function + +import ast +import atexit import contextlib +import importlib import os +import shutil import sys +from functools import partial import attr -import packaging.version import packaging.specifiers import packaging.utils +import packaging.version +import pep517.envbuild +import pep517.wrappers +import pkg_resources.extern.packaging.requirements as pkg_resources_requirements +import six +from appdirs import user_cache_dir +from distlib.wheel import Wheel +from packaging.markers import Marker +from six.moves import configparser +from six.moves.urllib.parse import unquote, urlparse, urlunparse +from vistir.compat import FileNotFoundError, Iterable, Mapping, Path, lru_cache +from vistir.contextmanagers import cd, temp_path +from vistir.misc import run +from vistir.path import create_tracked_tempdir, ensure_mkdir_p, mkdir_p, rmtree + +from .utils import ( + get_default_pyproject_backend, + get_name_variants, + get_pyproject, + init_requirement, + read_source, + split_vcs_method_from_uri, + strip_extras_markers_from_requirement, +) +from ..environment import MYPY_RUNNING +from ..exceptions import RequirementError try: - from setuptools.dist import distutils + from setuptools.dist import distutils, Distribution except ImportError: import distutils + from distutils.core import Distribution -from appdirs import user_cache_dir -from six.moves import configparser -from six.moves.urllib.parse import unquote -from vistir.compat import Path -from vistir.contextmanagers import cd -from vistir.misc import run -from vistir.path import create_tracked_tempdir, ensure_mkdir_p, mkdir_p - -from .utils import init_requirement, get_pyproject try: from os import scandir @@ -29,6 +53,41 @@ except ImportError: from scandir import scandir +if MYPY_RUNNING: + from typing import ( + Any, + Callable, + Dict, + List, + Generator, + Optional, + Union, + Tuple, + TypeVar, + Text, + Set, + AnyStr, + Sequence, + ) + from pip_shims.shims import InstallRequirement, PackageFinder + from pkg_resources import ( + PathMetadata, + DistInfoDistribution, + EggInfoDistribution, + Requirement as PkgResourcesRequirement, + ) + from packaging.requirements import Requirement as PackagingRequirement + + TRequirement = TypeVar("TRequirement") + RequirementType = TypeVar( + "RequirementType", covariant=True, bound=PackagingRequirement + ) + MarkerType = TypeVar("MarkerType", covariant=True, bound=Marker) + STRING_TYPE = Union[str, bytes, Text] + S = TypeVar("S", bytes, str, Text) + AST_SEQ = TypeVar("AST_SEQ", ast.Tuple, ast.List) + + CACHE_DIR = os.environ.get("PIPENV_CACHE_DIR", user_cache_dir("pipenv")) # The following are necessary for people who like to use "if __name__" conditionals @@ -37,8 +96,229 @@ _setup_stop_after = None _setup_distribution = None +def pep517_subprocess_runner(cmd, cwd=None, extra_environ=None): + # type: (List[AnyStr], Optional[AnyStr], Optional[Mapping[S, S]]) -> None + """The default method of calling the wrapper subprocess.""" + env = os.environ.copy() + if extra_environ: + env.update(extra_environ) + + run( + cmd, + cwd=cwd, + env=env, + block=True, + combine_stderr=True, + return_object=False, + write_to_stdout=False, + nospin=True, + ) + + +class BuildEnv(pep517.envbuild.BuildEnvironment): + def pip_install(self, reqs): + cmd = [ + sys.executable, + "-m", + "pip", + "install", + "--ignore-installed", + "--prefix", + self.path, + ] + list(reqs) + run( + cmd, + block=True, + combine_stderr=True, + return_object=False, + write_to_stdout=False, + nospin=True, + ) + + +class HookCaller(pep517.wrappers.Pep517HookCaller): + def __init__(self, source_dir, build_backend, backend_path=None): + self.source_dir = os.path.abspath(source_dir) + self.build_backend = build_backend + self._subprocess_runner = pep517_subprocess_runner + if backend_path: + backend_path = [ + pep517.wrappers.norm_and_check(self.source_dir, p) for p in backend_path + ] + self.backend_path = backend_path + + +def parse_special_directives(setup_entry, package_dir=None): + # type: (S, Optional[STRING_TYPE]) -> S + rv = setup_entry + if not package_dir: + package_dir = os.getcwd() + if setup_entry.startswith("file:"): + _, path = setup_entry.split("file:") + path = path.strip() + if os.path.exists(path): + rv = read_source(path) + elif setup_entry.startswith("attr:"): + _, resource = setup_entry.split("attr:") + resource = resource.strip() + with temp_path(): + sys.path.insert(0, package_dir) + if "." in resource: + resource, _, attribute = resource.rpartition(".") + package, _, path = resource.partition(".") + base_path = os.path.join(package_dir, package) + if path: + path = os.path.join(base_path, os.path.join(*path.split("."))) + else: + path = base_path + if not os.path.exists(path) and os.path.exists("{0}.py".format(path)): + path = "{0}.py".format(path) + elif os.path.isdir(path): + path = os.path.join(path, "__init__.py") + rv = ast_parse_attribute_from_file(path, attribute) + if rv: + return str(rv) + module = importlib.import_module(resource) + rv = getattr(module, attribute) + if not isinstance(rv, six.string_types): + rv = str(rv) + return rv + + +def make_base_requirements(reqs): + # type: (Sequence[STRING_TYPE]) -> Set[BaseRequirement] + requirements = set() + if not isinstance(reqs, (list, tuple, set)): + reqs = [reqs] + for req in reqs: + if isinstance(req, BaseRequirement): + requirements.add(req) + elif isinstance(req, pkg_resources_requirements.Requirement): + requirements.add(BaseRequirement.from_req(req)) + elif req and not req.startswith("#"): + requirements.add(BaseRequirement.from_string(req)) + return requirements + + +def setuptools_parse_setup_cfg(path): + from setuptools.config import read_configuration + + parsed = read_configuration(path) + results = parsed.get("metadata", {}) + results.update({parsed.get("options", {})}) + results["install_requires"] = make_base_requirements( + results.get("install_requires", []) + ) + extras = {} + for extras_section, extras in results.get("extras_require", {}).items(): + new_reqs = tuple(make_base_requirements(extras)) + if new_reqs: + extras[extras_section] = new_reqs + results["extras_require"] = extras + results["setup_requires"] = make_base_requirements(results.get("setup_requires", [])) + return results + + +def get_package_dir_from_setupcfg(parser, base_dir=None): + # type: (configparser.ConfigParser, STRING_TYPE) -> Text + if base_dir is not None: + package_dir = base_dir + else: + package_dir = os.getcwd() + if parser.has_option("options", "packages.find"): + pkg_dir = parser.get("options", "packages.find") + if isinstance(package_dir, Mapping): + package_dir = os.path.join(package_dir, pkg_dir.get("where")) + elif parser.has_option("options", "packages"): + pkg_dir = parser.get("options", "packages") + if "find:" in pkg_dir: + _, pkg_dir = pkg_dir.split("find:") + pkg_dir = pkg_dir.strip() + package_dir = os.path.join(package_dir, pkg_dir) + elif os.path.exists(os.path.join(package_dir, "setup.py")): + setup_py = ast_parse_setup_py(os.path.join(package_dir, "setup.py")) + if "package_dir" in setup_py: + package_lookup = setup_py["package_dir"] + if not isinstance(package_lookup, Mapping): + return package_lookup + return package_lookup.get( + next(iter(list(package_lookup.keys()))), package_dir + ) + return package_dir + + +def get_name_and_version_from_setupcfg(parser, package_dir): + # type: (configparser.ConfigParser, STRING_TYPE) -> Tuple[Optional[S], Optional[S]] + name, version = None, None + if parser.has_option("metadata", "name"): + name = parse_special_directives(parser.get("metadata", "name"), package_dir) + if parser.has_option("metadata", "version"): + version = parse_special_directives(parser.get("metadata", "version"), package_dir) + return name, version + + +def get_extras_from_setupcfg(parser): + # type: (configparser.ConfigParser) -> Dict[STRING_TYPE, Tuple[BaseRequirement, ...]] + extras = {} # type: Dict[STRING_TYPE, Tuple[BaseRequirement, ...]] + if "options.extras_require" not in parser.sections(): + return extras + extras_require_section = parser.options("options.extras_require") + for section in extras_require_section: + if section in ["options", "metadata"]: + continue + section_contents = parser.get("options.extras_require", section) + section_list = section_contents.split("\n") + section_extras = tuple(make_base_requirements(section_list)) + if section_extras: + extras[section] = section_extras + return extras + + +def parse_setup_cfg(setup_cfg_path): + # type: (S) -> Dict[S, Union[S, None, Set[BaseRequirement], List[S], Dict[STRING_TYPE, Tuple[BaseRequirement]]]] + if not os.path.exists(setup_cfg_path): + raise FileNotFoundError(setup_cfg_path) + try: + return setuptools_parse_setup_cfg(setup_cfg_path) + except Exception: + pass + default_opts = { + "metadata": {"name": "", "version": ""}, + "options": { + "install_requires": "", + "python_requires": "", + "build_requires": "", + "setup_requires": "", + "extras": "", + "packages.find": {"where": "."}, + }, + } + parser = configparser.ConfigParser(default_opts) + parser.read(setup_cfg_path) + results = {} + package_dir = get_package_dir_from_setupcfg(parser, base_dir=os.getcwd()) + name, version = get_name_and_version_from_setupcfg(parser, package_dir) + results["name"] = name + results["version"] = version + install_requires = set() # type: Set[BaseRequirement] + if parser.has_option("options", "install_requires"): + install_requires = make_base_requirements( + parser.get("options", "install_requires").split("\n") + ) + results["install_requires"] = install_requires + if parser.has_option("options", "python_requires"): + results["python_requires"] = parse_special_directives( + parser.get("options", "python_requires"), package_dir + ) + if parser.has_option("options", "build_requires"): + results["build_requires"] = parser.get("options", "build_requires") + results["extras_require"] = get_extras_from_setupcfg(parser) + return results + + @contextlib.contextmanager def _suppress_distutils_logs(): + # type: () -> Generator[None, None, None] """Hack to hide noise generated by `setup.py develop`. There isn't a good way to suppress them now, so let's monky-patch. @@ -56,38 +336,87 @@ def _suppress_distutils_logs(): distutils.log.Log._log = f +def build_pep517(source_dir, build_dir, config_settings=None, dist_type="wheel"): + if config_settings is None: + config_settings = {} + requires, backend = get_pyproject(source_dir) + hookcaller = HookCaller(source_dir, backend) + if dist_type == "sdist": + get_requires_fn = hookcaller.get_requires_for_build_sdist + build_fn = hookcaller.build_sdist + else: + get_requires_fn = hookcaller.get_requires_for_build_wheel + build_fn = hookcaller.build_wheel + + with BuildEnv() as env: + env.pip_install(requires) + reqs = get_requires_fn(config_settings) + env.pip_install(reqs) + return build_fn(build_dir, config_settings) + + @ensure_mkdir_p(mode=0o775) -def _get_src_dir(): +def _get_src_dir(root): + # type: (AnyStr) -> AnyStr src = os.environ.get("PIP_SRC") if src: return src virtual_env = os.environ.get("VIRTUAL_ENV") - if virtual_env: + if virtual_env is not None: return os.path.join(virtual_env, "src") - return os.path.join(os.getcwd(), "src") # Match pip's behavior. + if root is not None: + # Intentionally don't match pip's behavior here -- this is a temporary copy + src_dir = create_tracked_tempdir(prefix="requirementslib-", suffix="-src") + else: + src_dir = os.path.join(root, "src") + return src_dir -def _prepare_wheel_building_kwargs(ireq): - download_dir = os.path.join(CACHE_DIR, "pkgs") +@lru_cache() +def ensure_reqs(reqs): + # type: (List[Union[S, PkgResourcesRequirement]]) -> List[PkgResourcesRequirement] + import pkg_resources + + if not isinstance(reqs, Iterable): + raise TypeError("Expecting an Iterable, got %r" % reqs) + new_reqs = [] + for req in reqs: + if not req: + continue + if isinstance(req, six.string_types): + req = pkg_resources.Requirement.parse("{0}".format(str(req))) + # req = strip_extras_markers_from_requirement(req) + new_reqs.append(req) + return new_reqs + + +def _prepare_wheel_building_kwargs( + ireq=None, # type: Optional[InstallRequirement] + src_root=None, # type: Optional[STRING_TYPE] + src_dir=None, # type: Optional[STRING_TYPE] + editable=False, # type: bool +): + # type: (...) -> Dict[STRING_TYPE, STRING_TYPE] + download_dir = os.path.join(CACHE_DIR, "pkgs") # type: STRING_TYPE mkdir_p(download_dir) - wheel_download_dir = os.path.join(CACHE_DIR, "wheels") + wheel_download_dir = os.path.join(CACHE_DIR, "wheels") # type: STRING_TYPE mkdir_p(wheel_download_dir) - if ireq.source_dir is not None: - src_dir = ireq.source_dir - elif ireq.editable: - src_dir = _get_src_dir() - else: - src_dir = create_tracked_tempdir(prefix="reqlib-src") + if src_dir is None: + if editable and src_root is not None: + src_dir = src_root + elif ireq is None and src_root is not None and not editable: + src_dir = _get_src_dir(root=src_root) # type: STRING_TYPE + elif ireq is not None and ireq.editable and src_root is not None: + src_dir = _get_src_dir(root=src_root) + else: + src_dir = create_tracked_tempdir(prefix="reqlib-src") - # This logic matches pip's behavior, although I don't fully understand the - # intention. I guess the idea is to build editables in-place, otherwise out - # of the source tree? - if ireq.editable: - build_dir = src_dir - else: - build_dir = create_tracked_tempdir(prefix="reqlib-build") + # Let's always resolve in isolation + if src_dir is None: + src_dir = create_tracked_tempdir(prefix="reqlib-src") + build_dir = create_tracked_tempdir(prefix="reqlib-build") return { "build_dir": build_dir, @@ -97,259 +426,956 @@ def _prepare_wheel_building_kwargs(ireq): } -def iter_egginfos(path, pkg_name=None): - for entry in scandir(path): - if entry.is_dir(): - if not entry.name.endswith("egg-info"): - for dir_entry in iter_egginfos(entry.path, pkg_name=pkg_name): - yield dir_entry - elif pkg_name is None or entry.name.startswith(pkg_name.replace("-", "_")): - yield entry +class ScandirCloser(object): + def __init__(self, path): + self.iterator = scandir(path) + + def __next__(self): + return next(iter(self.iterator)) + + def __iter__(self): + return self + + def next(self): + return self.__next__() + + def close(self): + if getattr(self.iterator, "close", None): + self.iterator.close() + else: + pass + + +def iter_metadata(path, pkg_name=None, metadata_type="egg-info"): + # type: (AnyStr, Optional[AnyStr], AnyStr) -> Generator + if pkg_name is not None: + pkg_variants = get_name_variants(pkg_name) + non_matching_dirs = [] + with contextlib.closing(ScandirCloser(path)) as path_iterator: + for entry in path_iterator: + if entry.is_dir(): + entry_name, ext = os.path.splitext(entry.name) + if ext.endswith(metadata_type): + if pkg_name is None or entry_name.lower() in pkg_variants: + yield entry + elif not entry.name.endswith(metadata_type): + non_matching_dirs.append(entry) + for entry in non_matching_dirs: + for dir_entry in iter_metadata( + entry.path, pkg_name=pkg_name, metadata_type=metadata_type + ): + yield dir_entry def find_egginfo(target, pkg_name=None): - egg_dirs = (egg_dir for egg_dir in iter_egginfos(target, pkg_name=pkg_name)) + # type: (AnyStr, Optional[AnyStr]) -> Generator + egg_dirs = ( + egg_dir + for egg_dir in iter_metadata(target, pkg_name=pkg_name) + if egg_dir is not None + ) if pkg_name: - yield next(iter(egg_dirs), None) + yield next(iter(eggdir for eggdir in egg_dirs if eggdir is not None), None) else: for egg_dir in egg_dirs: yield egg_dir -def get_metadata(path, pkg_name=None): +def find_distinfo(target, pkg_name=None): + # type: (AnyStr, Optional[AnyStr]) -> Generator + dist_dirs = ( + dist_dir + for dist_dir in iter_metadata( + target, pkg_name=pkg_name, metadata_type="dist-info" + ) + if dist_dir is not None + ) if pkg_name: - pkg_name = packaging.utils.canonicalize_name(pkg_name) + yield next(iter(dist for dist in dist_dirs if dist is not None), None) + else: + for dist_dir in dist_dirs: + yield dist_dir + + +def get_distinfo_dist(path, pkg_name=None): + # type: (S, Optional[S]) -> Optional[DistInfoDistribution] + import pkg_resources + + dist_dir = next(iter(find_distinfo(path, pkg_name=pkg_name)), None) + if dist_dir is not None: + metadata_dir = dist_dir.path + base_dir = os.path.dirname(metadata_dir) + dist = next(iter(pkg_resources.find_distributions(base_dir)), None) + if dist is not None: + return dist + return None + + +def get_egginfo_dist(path, pkg_name=None): + # type: (S, Optional[S]) -> Optional[EggInfoDistribution] + import pkg_resources + egg_dir = next(iter(find_egginfo(path, pkg_name=pkg_name)), None) if egg_dir is not None: - import pkg_resources + metadata_dir = egg_dir.path + base_dir = os.path.dirname(metadata_dir) + path_metadata = pkg_resources.PathMetadata(base_dir, metadata_dir) + dist_iter = pkg_resources.distributions_from_metadata(path_metadata.egg_info) + dist = next(iter(dist_iter), None) + if dist is not None: + return dist + return None - egg_dir = os.path.abspath(egg_dir.path) - base_dir = os.path.dirname(egg_dir) - path_metadata = pkg_resources.PathMetadata(base_dir, egg_dir) - dist = next( - iter(pkg_resources.distributions_from_metadata(path_metadata.egg_info)), - None, - ) - if dist: - try: - requires = dist.requires() - except exception: - requires = [] - try: - dep_map = dist._build_dep_map() - except Exception: - dep_map = {} - deps = [] - extras = {} - for k in dep_map.keys(): - if k is None: - deps.extend(dep_map.get(k)) - continue + +def get_metadata(path, pkg_name=None, metadata_type=None): + # type: (S, Optional[S], Optional[S]) -> Dict[S, Union[S, List[RequirementType], Dict[S, RequirementType]]] + wheel_allowed = metadata_type == "wheel" or metadata_type is None + egg_allowed = metadata_type == "egg" or metadata_type is None + dist = None # type: Optional[Union[DistInfoDistribution, EggInfoDistribution]] + if wheel_allowed: + dist = get_distinfo_dist(path, pkg_name=pkg_name) + if egg_allowed and dist is None: + dist = get_egginfo_dist(path, pkg_name=pkg_name) + if dist is not None: + return get_metadata_from_dist(dist) + return {} + + +@lru_cache() +def get_extra_name_from_marker(marker): + # type: (MarkerType) -> Optional[S] + if not marker: + raise ValueError("Invalid value for marker: {0!r}".format(marker)) + if not getattr(marker, "_markers", None): + raise TypeError("Expecting a marker instance, received {0!r}".format(marker)) + for elem in marker._markers: + if isinstance(elem, tuple) and elem[0].value == "extra": + return elem[2].value + return None + + +def get_metadata_from_wheel(wheel_path): + # type: (S) -> Dict[Any, Any] + if not isinstance(wheel_path, six.string_types): + raise TypeError("Expected string instance, received {0!r}".format(wheel_path)) + try: + dist = Wheel(wheel_path) + except Exception: + pass + metadata = dist.metadata + name = metadata.name + version = metadata.version + requires = [] + extras_keys = getattr(metadata, "extras", []) # type: List[STRING_TYPE] + extras = { + k: [] for k in extras_keys + } # type: Dict[STRING_TYPE, List[RequirementType]] + for req in getattr(metadata, "run_requires", []): + parsed_req = init_requirement(req) + parsed_marker = parsed_req.marker + if parsed_marker: + extra = get_extra_name_from_marker(parsed_marker) + if extra is None: + requires.append(parsed_req) + continue + if extra not in extras: + extras[extra] = [] + parsed_req = strip_extras_markers_from_requirement(parsed_req) + extras[extra].append(parsed_req) + else: + requires.append(parsed_req) + return {"name": name, "version": version, "requires": requires, "extras": extras} + + +def get_metadata_from_dist(dist): + # type: (Union[PathMetadata, EggInfoDistribution, DistInfoDistribution]) -> Dict[S, Union[S, List[RequirementType], Dict[S, RequirementType]]] + try: + requires = dist.requires() + except Exception: + requires = [] + try: + dep_map = dist._build_dep_map() + except Exception: + dep_map = {} + deps = [] # type: List[PkgResourcesRequirement] + extras = {} + for k in dep_map.keys(): + if k is None: + deps.extend(dep_map.get(k)) + continue + else: + extra = None + _deps = dep_map.get(k) + if k.startswith(":python_version"): + marker = k.replace(":", "; ") + else: + if ":python_version" in k: + extra, _, marker = k.partition(":") + marker = "; {0}".format(marker) else: - extra = None - _deps = dep_map.get(k) - if k.startswith(":python_version"): - marker = k.replace(":", "; ") - else: - marker = "" - extra = "{0}".format(k) - _deps = [ - pkg_resources.Requirement.parse("{0}{1}".format(str(req), marker)) - for req in _deps - ] - if extra: - extras[extra] = _deps - else: - deps.extend(_deps) - return { - "name": dist.project_name, - "version": dist.version, - "requires": requires, - "extras": extras - } + marker = "" + extra = "{0}".format(k) + _deps = ensure_reqs( + tuple(["{0}{1}".format(str(req), marker) for req in _deps]) + ) + if extra: + extras[extra] = _deps + else: + deps.extend(_deps) + requires.extend(deps) + return { + "name": dist.project_name, + "version": dist.version, + "requires": requires, + "extras": extras, + } -@attr.s(slots=True) +class Analyzer(ast.NodeVisitor): + def __init__(self): + self.name_types = [] + self.function_map = {} # type: Dict[Any, Any] + self.functions = [] + self.strings = [] + self.assignments = {} + super(Analyzer, self).__init__() + + def generic_visit(self, node): + if isinstance(node, ast.Call): + self.functions.append(node) + self.function_map.update(ast_unparse(node, initial_mapping=True)) + if isinstance(node, ast.Name): + self.name_types.append(node) + if isinstance(node, ast.Str): + self.strings.append(node) + if isinstance(node, ast.Assign): + self.assignments.update(ast_unparse(node, initial_mapping=True)) + super(Analyzer, self).generic_visit(node) + + def match_assignment_str(self, match): + return next( + iter(k for k in self.assignments if getattr(k, "id", "") == match), None + ) + + def match_assignment_name(self, match): + return next( + iter(k for k in self.assignments if getattr(k, "id", "") == match.id), None + ) + + +def ast_unparse(item, initial_mapping=False, analyzer=None, recurse=True): # noqa:C901 + # type: (Any, bool, Optional[Analyzer], bool) -> Union[List[Any], Dict[Any, Any], Tuple[Any, ...], STRING_TYPE] + unparse = partial( + ast_unparse, initial_mapping=initial_mapping, analyzer=analyzer, recurse=recurse + ) + if isinstance(item, ast.Dict): + unparsed = dict(zip(unparse(item.keys), unparse(item.values))) + elif isinstance(item, ast.List): + unparsed = [unparse(el) for el in item.elts] + elif isinstance(item, ast.Tuple): + unparsed = tuple([unparse(el) for el in item.elts]) + elif isinstance(item, ast.Str): + unparsed = item.s + elif isinstance(item, ast.Subscript): + unparsed = unparse(item.value) + elif isinstance(item, ast.Name): + if not initial_mapping: + unparsed = item.id + if analyzer and recurse: + if item in analyzer.assignments: + items = unparse(analyzer.assignments[item]) + unparsed = items.get(item.id, item.id) + else: + assignment = analyzer.match_assignment_name(item) + if assignment is not None: + items = unparse(analyzer.assignments[assignment]) + unparsed = items.get(item.id, item.id) + else: + unparsed = item + elif six.PY3 and isinstance(item, ast.NameConstant): + unparsed = item.value + elif isinstance(item, ast.Attribute): + attr_name = getattr(item, "value", None) + attr_attr = getattr(item, "attr", None) + name = None + if initial_mapping: + unparsed = item + elif attr_name and not recurse: + name = attr_name + else: + name = unparse(attr_name) if attr_name is not None else attr_attr + if name and attr_attr: + if not initial_mapping and isinstance(name, six.string_types): + unparsed = ".".join([item for item in (name, attr_attr) if item]) + else: + unparsed = item + elif attr_attr and not name and not initial_mapping: + unparsed = attr_attr + else: + unparsed = name if not unparsed else unparsed + elif isinstance(item, ast.Call): + unparsed = {} + if isinstance(item.func, ast.Name): + func_name = unparse(item.func) + elif isinstance(item.func, ast.Attribute): + func_name = unparse(item.func) + if func_name: + unparsed[func_name] = {} + for keyword in item.keywords: + unparsed[func_name].update(unparse(keyword)) + elif isinstance(item, ast.keyword): + unparsed = {unparse(item.arg): unparse(item.value)} + elif isinstance(item, ast.Assign): + # XXX: DO NOT UNPARSE THIS + # XXX: If we unparse this it becomes impossible to map it back + # XXX: To the original node in the AST so we can find the + # XXX: Original reference + if not initial_mapping: + target = unparse(next(iter(item.targets)), recurse=False) + val = unparse(item.value, recurse=False) + if isinstance(target, (tuple, set, list)): + unparsed = dict(zip(target, val)) + else: + unparsed = {target: val} + else: + unparsed = {next(iter(item.targets)): item} + elif isinstance(item, Mapping): + unparsed = {} + for k, v in item.items(): + try: + unparsed[unparse(k)] = unparse(v) + except TypeError: + unparsed[k] = unparse(v) + elif isinstance(item, (list, tuple)): + unparsed = type(item)([unparse(el) for el in item]) + elif isinstance(item, six.string_types): + unparsed = item + else: + return item + return unparsed + + +def ast_parse_attribute_from_file(path, attribute): + # type: (S) -> Any + analyzer = ast_parse_file(path) + target_value = None + for k, v in analyzer.assignments.items(): + name = "" + if isinstance(k, ast.Name): + name = k.id + elif isinstance(k, ast.Attribute): + fn = ast_unparse(k) + if isinstance(fn, six.string_types): + _, _, name = fn.rpartition(".") + if name == attribute: + target_value = ast_unparse(v, analyzer=analyzer) + break + if isinstance(target_value, Mapping) and attribute in target_value: + return target_value[attribute] + return target_value + + +def ast_parse_file(path): + # type: (S) -> Analyzer + tree = ast.parse(read_source(path)) + ast_analyzer = Analyzer() + ast_analyzer.visit(tree) + return ast_analyzer + + +def ast_parse_setup_py(path): + # type: (S) -> Dict[Any, Any] + ast_analyzer = ast_parse_file(path) + setup = {} # type: Dict[Any, Any] + for k, v in ast_analyzer.function_map.items(): + fn_name = "" + if isinstance(k, ast.Name): + fn_name = k.id + elif isinstance(k, ast.Attribute): + fn = ast_unparse(k) + if isinstance(fn, six.string_types): + _, _, fn_name = fn.rpartition(".") + if fn_name == "setup": + setup = v + cleaned_setup = ast_unparse(setup, analyzer=ast_analyzer) + return cleaned_setup + + +def run_setup(script_path, egg_base=None): + # type: (str, Optional[str]) -> Distribution + """Run a `setup.py` script with a target **egg_base** if provided. + + :param S script_path: The path to the `setup.py` script to run + :param Optional[S] egg_base: The metadata directory to build in + :raises FileNotFoundError: If the provided `script_path` does not exist + :return: The metadata dictionary + :rtype: Dict[Any, Any] + """ + + if not os.path.exists(script_path): + raise FileNotFoundError(script_path) + target_cwd = os.path.dirname(os.path.abspath(script_path)) + if egg_base is None: + egg_base = os.path.join(target_cwd, "reqlib-metadata") + with temp_path(), cd(target_cwd), _suppress_distutils_logs(): + # This is for you, Hynek + # see https://github.com/hynek/environ_config/blob/69b1c8a/setup.py + args = ["egg_info"] + if egg_base: + args += ["--egg-base", egg_base] + script_name = os.path.basename(script_path) + g = {"__file__": script_name, "__name__": "__main__"} + sys.path.insert(0, target_cwd) + local_dict = {} + if sys.version_info < (3, 5): + save_argv = sys.argv + else: + save_argv = sys.argv.copy() + try: + global _setup_distribution, _setup_stop_after + _setup_stop_after = "run" + sys.argv[0] = script_name + sys.argv[1:] = args + with open(script_name, "rb") as f: + contents = f.read() + if six.PY3: + contents.replace(br"\r\n", br"\n") + else: + contents.replace(r"\r\n", r"\n") + if sys.version_info < (3, 5): + exec(contents, g, local_dict) + else: + exec(contents, g) + # We couldn't import everything needed to run setup + except Exception: + python = os.environ.get("PIP_PYTHON_PATH", sys.executable) + out, _ = run( + [python, "setup.py"] + args, + cwd=target_cwd, + block=True, + combine_stderr=False, + return_object=False, + nospin=True, + ) + finally: + _setup_stop_after = None + sys.argv = save_argv + _setup_distribution = get_metadata(egg_base, metadata_type="egg") + dist = _setup_distribution + return dist + + +@attr.s(slots=True, frozen=True) +class BaseRequirement(object): + name = attr.ib(default="", cmp=True) # type: STRING_TYPE + requirement = attr.ib( + default=None, cmp=True + ) # type: Optional[PkgResourcesRequirement] + + def __str__(self): + # type: () -> S + return "{0}".format(str(self.requirement)) + + def as_dict(self): + # type: () -> Dict[STRING_TYPE, Optional[PkgResourcesRequirement]] + return {self.name: self.requirement} + + def as_tuple(self): + # type: () -> Tuple[STRING_TYPE, Optional[PkgResourcesRequirement]] + return (self.name, self.requirement) + + @classmethod + @lru_cache() + def from_string(cls, line): + # type: (S) -> BaseRequirement + line = line.strip() + req = init_requirement(line) + return cls.from_req(req) + + @classmethod + @lru_cache() + def from_req(cls, req): + # type: (PkgResourcesRequirement) -> BaseRequirement + name = None + key = getattr(req, "key", None) + name = getattr(req, "name", None) + project_name = getattr(req, "project_name", None) + if key is not None: + name = key + if name is None: + name = project_name + return cls(name=name, requirement=req) + + +@attr.s(slots=True, frozen=True) +class Extra(object): + name = attr.ib(default=None, cmp=True) # type: STRING_TYPE + requirements = attr.ib(factory=frozenset, cmp=True, type=frozenset) + + def __str__(self): + # type: () -> S + return "{0}: {{{1}}}".format( + self.name, ", ".join([r.name for r in self.requirements]) + ) + + def add(self, req): + # type: (BaseRequirement) -> "Extra" + if req not in self.requirements: + current_set = set(self.requirements) + current_set.add(req) + return attr.evolve(self, requirements=frozenset(current_set)) + return self + + def as_dict(self): + # type: () -> Dict[STRING_TYPE, Tuple[RequirementType, ...]] + return {self.name: tuple([r.requirement for r in self.requirements])} + + +@attr.s(slots=True, cmp=True, hash=True) class SetupInfo(object): - name = attr.ib(type=str, default=None) - base_dir = attr.ib(type=Path, default=None) - version = attr.ib(type=packaging.version.Version, default=None) - requires = attr.ib(type=dict, default=attr.Factory(dict)) - build_requires = attr.ib(type=list, default=attr.Factory(list)) - build_backend = attr.ib(type=list, default=attr.Factory(list)) - setup_requires = attr.ib(type=dict, default=attr.Factory(list)) - python_requires = attr.ib(type=packaging.specifiers.SpecifierSet, default=None) - extras = attr.ib(type=dict, default=attr.Factory(dict)) - setup_cfg = attr.ib(type=Path, default=None) - setup_py = attr.ib(type=Path, default=None) - pyproject = attr.ib(type=Path, default=None) - ireq = attr.ib(default=None) - extra_kwargs = attr.ib(default=attr.Factory(dict), type=dict) + name = attr.ib(default=None, cmp=True) # type: STRING_TYPE + base_dir = attr.ib(default=None, cmp=True, hash=False) # type: STRING_TYPE + _version = attr.ib(default=None, cmp=True) # type: STRING_TYPE + _requirements = attr.ib( + type=frozenset, factory=frozenset, cmp=True, hash=True + ) # type: Optional[frozenset] + build_requires = attr.ib(default=None, cmp=True) # type: Optional[Tuple] + build_backend = attr.ib(cmp=True) # type: STRING_TYPE + setup_requires = attr.ib(default=None, cmp=True) # type: Optional[Tuple] + python_requires = attr.ib( + default=None, cmp=True + ) # type: Optional[packaging.specifiers.SpecifierSet] + _extras_requirements = attr.ib(default=None, cmp=True) # type: Optional[Tuple] + setup_cfg = attr.ib(type=Path, default=None, cmp=True, hash=False) + setup_py = attr.ib(type=Path, default=None, cmp=True, hash=False) + pyproject = attr.ib(type=Path, default=None, cmp=True, hash=False) + ireq = attr.ib( + default=None, cmp=True, hash=False + ) # type: Optional[InstallRequirement] + extra_kwargs = attr.ib(default=attr.Factory(dict), type=dict, cmp=False, hash=False) + metadata = attr.ib(default=None) # type: Optional[Tuple[STRING_TYPE]] + + @build_backend.default + def get_build_backend(self): + # type: () -> STRING_TYPE + return get_default_pyproject_backend() + + @property + def requires(self): + # type: () -> Dict[S, RequirementType] + if self._requirements is None: + self._requirements = frozenset() + self.get_info() + return {req.name: req.requirement for req in self._requirements} + + @property + def extras(self): + # type: () -> Dict[S, Optional[Any]] + if self._extras_requirements is None: + self._extras_requirements = () + self.get_info() + extras_dict = {} + extras = set(self._extras_requirements) + for section, deps in extras: + if isinstance(deps, BaseRequirement): + extras_dict[section] = deps.requirement + elif isinstance(deps, (list, tuple)): + extras_dict[section] = [d.requirement for d in deps] + return extras_dict + + @property + def version(self): + # type: () -> Optional[str] + if not self._version: + info = self.get_info() + self._version = info.get("version", None) + return self._version + + @classmethod + def get_setup_cfg(cls, setup_cfg_path): + # type: (S) -> Dict[S, Union[S, None, Set[BaseRequirement], List[S], Tuple[S, Tuple[BaseRequirement]]]] + return parse_setup_cfg(setup_cfg_path) + + @property + def egg_base(self): + # type: () -> S + base = None # type: Optional[STRING_TYPE] + if self.setup_py.exists(): + base = self.setup_py.parent + elif self.pyproject.exists(): + base = self.pyproject.parent + elif self.setup_cfg.exists(): + base = self.setup_cfg.parent + if base is None: + base = Path(self.base_dir) + if base is None: + base = Path(self.extra_kwargs["src_dir"]) + egg_base = base.joinpath("reqlib-metadata") + if not egg_base.exists(): + atexit.register(rmtree, egg_base.as_posix()) + egg_base.mkdir(parents=True, exist_ok=True) + return egg_base.as_posix() + + def update_from_dict(self, metadata): + name = metadata.get("name", self.name) + if isinstance(name, six.string_types): + self.name = self.name if self.name else name + version = metadata.get("version", None) + if version: + try: + packaging.version.parse(version) + except TypeError: + version = self.version if self.version else None + else: + version = version + if version: + self._version = version + build_requires = metadata.get("build_requires", []) + if self.build_requires is None: + self.build_requires = () + self.build_requires = tuple(set(self.build_requires) | set(build_requires)) + self._requirements = ( + frozenset() if self._requirements is None else self._requirements + ) + requirements = set(self._requirements) + install_requires = make_base_requirements(metadata.get("install_requires", [])) + requirements |= install_requires + setup_requires = make_base_requirements(metadata.get("setup_requires", [])) + if self.setup_requires is None: + self.setup_requires = () + self.setup_requires = tuple(set(self.setup_requires) | setup_requires) + if self.ireq.editable: + requirements |= setup_requires + # TODO: Should this be a specifierset? + self.python_requires = metadata.get("python_requires", self.python_requires) + extras_require = metadata.get("extras_require", {}) + extras_tuples = [] + for section in set(list(extras_require.keys())) - set(list(self.extras.keys())): + extras = extras_require[section] + extras_set = make_base_requirements(extras) + if self.ireq and self.ireq.extras and section in self.ireq.extras: + requirements |= extras_set + extras_tuples.append((section, tuple(extras_set))) + if self._extras_requirements is None: + self._extras_requirements = () + self._extras_requirements += tuple(extras_tuples) + build_backend = metadata.get("build_backend", "setuptools.build_meta:__legacy__") + if not self.build_backend: + self.build_backend = build_backend + self._requirements = frozenset(requirements) + + def get_extras_from_ireq(self): + # type: () -> None + if self.ireq and self.ireq.extras: + for extra in self.ireq.extras: + if extra in self.extras: + extras = make_base_requirements(self.extras[extra]) + self._requirements = frozenset(set(self._requirements) | extras) + else: + extras = tuple(make_base_requirements(extra)) + self._extras_requirements += (extra, extras) def parse_setup_cfg(self): + # type: () -> Dict[STRING_TYPE, Any] if self.setup_cfg is not None and self.setup_cfg.exists(): - default_opts = { - "metadata": {"name": "", "version": ""}, - "options": { - "install_requires": "", - "python_requires": "", - "build_requires": "", - "setup_requires": "", - "extras": "", - }, - } - parser = configparser.ConfigParser(default_opts) - parser.read(self.setup_cfg.as_posix()) - if parser.has_option("metadata", "name"): - name = parser.get("metadata", "name") - if not self.name and name is not None: - self.name = name - if parser.has_option("metadata", "version"): - version = parser.get("metadata", "version") - if not self.version and version is not None: - self.version = version - if parser.has_option("options", "install_requires"): - self.requires.update( - { - dep.strip(): init_requirement(dep.strip()) - for dep in parser.get("options", "install_requires").split("\n") - if dep - } - ) - if parser.has_option("options", "python_requires"): - python_requires = parser.get("options", "python_requires") - if python_requires and not self.python_requires: - self.python_requires = python_requires - if parser.has_option("options", "extras_require"): - self.extras.update( - { - section: [ - dep.strip() - for dep in parser.get( - "options.extras_require", section - ).split("\n") - if dep - ] - for section in parser.options("options.extras_require") - } - ) + parsed = self.get_setup_cfg(self.setup_cfg.as_posix()) + if not parsed: + return {} + return parsed + return {} + + def parse_setup_py(self): + # type: () -> Dict[STRING_TYPE, Any] + if self.setup_py is not None and self.setup_py.exists(): + parsed = ast_parse_setup_py(self.setup_py.as_posix()) + if not parsed: + return {} + return parsed + return {} def run_setup(self): + # type: () -> "SetupInfo" if self.setup_py is not None and self.setup_py.exists(): + dist = run_setup(self.setup_py.as_posix(), egg_base=self.egg_base) target_cwd = self.setup_py.parent.as_posix() - with cd(target_cwd), _suppress_distutils_logs(): - # This is for you, Hynek - # see https://github.com/hynek/environ_config/blob/69b1c8a/setup.py - script_name = self.setup_py.as_posix() - args = ["egg_info"] - g = {"__file__": script_name, "__name__": "__main__"} - local_dict = {} - if sys.version_info < (3, 5): - save_argv = sys.argv - else: - save_argv = sys.argv.copy() - try: - global _setup_distribution, _setup_stop_after - _setup_stop_after = "run" - sys.argv[0] = script_name - sys.argv[1:] = args - with open(script_name, 'rb') as f: - if sys.version_info < (3, 5): - exec(f.read(), g, local_dict) - else: - exec(f.read(), g) - # We couldn't import everything needed to run setup - except NameError: - python = os.environ.get('PIP_PYTHON_PATH', sys.executable) - out, _ = run([python, "setup.py"] + args, cwd=target_cwd, block=True, - combine_stderr=False, return_object=False, nospin=True) - finally: - _setup_stop_after = None - sys.argv = save_argv - dist = _setup_distribution + with temp_path(), cd(target_cwd): if not dist: - self.get_egg_metadata() - return + metadata = self.get_egg_metadata() + if metadata: + return self.populate_metadata(metadata) + if isinstance(dist, Mapping): + self.populate_metadata(dist) + return name = dist.get_name() if name: self.name = name - if dist.python_requires and not self.python_requires: - self.python_requires = packaging.specifiers.SpecifierSet( - dist.python_requires - ) - if dist.extras_require and not self.extras: - self.extras = dist.extras_require - install_requires = dist.get_requires() - if not install_requires: - install_requires = dist.install_requires - if install_requires and not self.requires: - requirements = [init_requirement(req) for req in install_requires] - self.requires.update({req.key: req for req in requirements}) - if dist.setup_requires and not self.setup_requires: - self.setup_requires = dist.setup_requires - if not self.version: - self.version = dist.get_version() - - def get_egg_metadata(self): - if self.setup_py is not None and self.setup_py.exists(): - metadata = get_metadata(self.setup_py.parent.as_posix(), pkg_name=self.name) - if metadata: - if not self.name: - self.name = metadata.get("name", self.name) - if not self.version: - self.version = metadata.get("version", self.version) - self.requires.update( - {req.key: req for req in metadata.get("requires", {})} + update_dict = {} + if dist.python_requires: + update_dict["python_requires"] = dist.python_requires + update_dict["extras_require"] = {} + if dist.extras_require: + for extra, extra_requires in dist.extras_require: + extras_tuple = make_base_requirements(extra_requires) + update_dict["extras_require"][extra] = extras_tuple + update_dict["install_requires"] = make_base_requirements( + dist.get_requires() ) - if getattr(self.ireq, "extras", None): - for extra in self.ireq.extras: - self.requires.update( - { - req.key: req for req - in metadata.get("extras", {}).get(extra) - if req is not None - } - ) + if dist.setup_requires: + update_dict["setup_requires"] = make_base_requirements( + dist.setup_requires + ) + version = dist.get_version() + if version: + update_dict["version"] = version + return self.update_from_dict(update_dict) + + @property + def pep517_config(self): + config = {} + config.setdefault("--global-option", []) + return config + + def build_wheel(self): + # type: () -> S + if not self.pyproject.exists(): + build_requires = ", ".join(['"{0}"'.format(r) for r in self.build_requires]) + self.pyproject.write_text( + u""" +[build-system] +requires = [{0}] +build-backend = "{1}" + """.format( + build_requires, self.build_backend + ).strip() + ) + return build_pep517( + self.base_dir, + self.extra_kwargs["build_dir"], + config_settings=self.pep517_config, + dist_type="wheel", + ) + + # noinspection PyPackageRequirements + def build_sdist(self): + # type: () -> S + if not self.pyproject.exists(): + if not self.build_requires: + build_requires = '"setuptools", "wheel"' + else: + build_requires = ", ".join( + ['"{0}"'.format(r) for r in self.build_requires] + ) + self.pyproject.write_text( + u""" +[build-system] +requires = [{0}] +build-backend = "{1}" + """.format( + build_requires, self.build_backend + ).strip() + ) + return build_pep517( + self.base_dir, + self.extra_kwargs["build_dir"], + config_settings=self.pep517_config, + dist_type="sdist", + ) + + def build(self): + # type: () -> "SetupInfo" + dist_path = None + metadata = None + try: + dist_path = self.build_wheel() + metadata = self.get_metadata_from_wheel( + os.path.join(self.extra_kwargs["build_dir"], dist_path) + ) + except Exception: + try: + dist_path = self.build_sdist() + metadata = self.get_egg_metadata(metadata_type="egg") + if metadata: + self.populate_metadata(metadata) + except Exception: + pass + if metadata: + self.populate_metadata(metadata) + if not self.metadata or not self.name: + metadata = self.get_egg_metadata() + if metadata: + self.populate_metadata(metadata) + if not self.metadata or not self.name: + return self.run_setup() + return self + + def reload(self): + # type: () -> Dict[S, Any] + """Wipe existing distribution info metadata for rebuilding. + + Erases metadata from **self.egg_base** and unsets **self.requirements** + and **self.extras**. + """ + for metadata_dir in os.listdir(self.egg_base): + shutil.rmtree(metadata_dir, ignore_errors=True) + self.metadata = None + self._requirements = frozenset() + self._extras_requirements = () + self.get_info() + + def get_metadata_from_wheel(self, wheel_path): + # type: (S) -> Dict[Any, Any] + """Given a path to a wheel, return the metadata from that wheel. + + :return: A dictionary of metadata from the provided wheel + :rtype: Dict[Any, Any] + """ + + metadata_dict = get_metadata_from_wheel(wheel_path) + return metadata_dict + + def get_egg_metadata(self, metadata_dir=None, metadata_type=None): + # type: (Optional[AnyStr], Optional[AnyStr]) -> Dict[Any, Any] + """Given a metadata directory, return the corresponding metadata dictionary. + + :param Optional[str] metadata_dir: Root metadata path, default: `os.getcwd()` + :param Optional[str] metadata_type: Type of metadata to search for, default None + :return: A metadata dictionary built from the metadata in the given location + :rtype: Dict[Any, Any] + """ + + package_indicators = [self.pyproject, self.setup_py, self.setup_cfg] + metadata_dirs = [] # type: List[STRING_TYPE] + if any([fn is not None and fn.exists() for fn in package_indicators]): + metadata_dirs = [ + self.extra_kwargs["build_dir"], + self.egg_base, + self.extra_kwargs["src_dir"], + ] + if metadata_dir is not None: + metadata_dirs = [metadata_dir] + metadata_dirs + metadata = [ + get_metadata(d, pkg_name=self.name, metadata_type=metadata_type) + for d in metadata_dirs + if os.path.exists(d) + ] + metadata = next(iter(d for d in metadata if d), None) + return metadata + + def populate_metadata(self, metadata): + # type: (Dict[Any, Any]) -> "SetupInfo" + """Populates the metadata dictionary from the supplied metadata. + + :return: The current instance. + :rtype: `SetupInfo` + """ + + _metadata = () + for k, v in metadata.items(): + if k == "extras" and isinstance(v, dict): + extras = () + for extra, reqs in v.items(): + extras += ((extra, tuple(reqs)),) + _metadata += extras + elif isinstance(v, (list, tuple)): + _metadata += (k, tuple(v)) + else: + _metadata += (k, v) + self.metadata = _metadata + cleaned = metadata.copy() + cleaned.update({"install_requires": metadata.get("requires", [])}) + if cleaned: + self.update_from_dict(cleaned.copy()) + else: + self.update_from_dict(metadata) + return self def run_pyproject(self): + # type: () -> "SetupInfo" + """Populates the **pyproject.toml** metadata if available. + + :return: The current instance + :rtype: `SetupInfo` + """ + if self.pyproject and self.pyproject.exists(): result = get_pyproject(self.pyproject.parent) if result is not None: requires, backend = result + if self.build_requires is None: + self.build_requires = () if backend: self.build_backend = backend - if requires and not self.build_requires: - self.build_requires = requires + else: + self.build_backend = get_default_pyproject_backend() + if requires: + self.build_requires = tuple(set(requires) | set(self.build_requires)) + else: + self.build_requires = ("setuptools", "wheel") + return self + + def get_initial_info(self): + # type: () -> Dict[S, Any] + parse_setupcfg = False + parse_setuppy = False + if self.setup_cfg and self.setup_cfg.exists(): + parse_setupcfg = True + if self.setup_py and self.setup_py.exists(): + parse_setuppy = True + if parse_setuppy or parse_setupcfg: + with cd(self.base_dir): + if parse_setuppy: + self.update_from_dict(self.parse_setup_py()) + if parse_setupcfg: + self.update_from_dict(self.parse_setup_cfg()) + if self.name is not None and any( + [ + self.requires, + self.setup_requires, + self._extras_requirements, + self.build_backend, + ] + ): + return self.as_dict() + return self.get_info() def get_info(self): - if self.setup_cfg and self.setup_cfg.exists(): - self.parse_setup_cfg() - if self.setup_py and self.setup_py.exists(): + # type: () -> Dict[S, Any] + with cd(self.base_dir): + self.run_pyproject() + self.build() + + if self.setup_py and self.setup_py.exists() and self.metadata is None: if not self.requires or not self.name: try: - self.run_setup() + with cd(self.base_dir): + self.run_setup() except Exception: - self.get_egg_metadata() - if not self.requires or not self.name: - self.get_egg_metadata() + with cd(self.base_dir): + metadata = self.get_egg_metadata() + if metadata: + self.populate_metadata(metadata) + if self.metadata is None or not self.name: + with cd(self.base_dir): + metadata = self.get_egg_metadata() + if metadata: + self.populate_metadata(metadata) - if self.pyproject and self.pyproject.exists(): - self.run_pyproject() return self.as_dict() def as_dict(self): + # type: () -> Dict[STRING_TYPE, Any] prop_dict = { "name": self.name, - "version": self.version, + "version": self.version if self._version else None, "base_dir": self.base_dir, "ireq": self.ireq, "build_backend": self.build_backend, "build_requires": self.build_requires, - "requires": self.requires, + "requires": self.requires if self._requirements else None, "setup_requires": self.setup_requires, "python_requires": self.python_requires, - "extras": self.extras, + "extras": self.extras if self._extras_requirements else None, "extra_kwargs": self.extra_kwargs, "setup_cfg": self.setup_cfg, "setup_py": self.setup_py, @@ -359,23 +1385,40 @@ class SetupInfo(object): @classmethod def from_requirement(cls, requirement, finder=None): + # type: (TRequirement, Optional[PackageFinder]) -> Optional[SetupInfo] ireq = requirement.as_ireq() subdir = getattr(requirement.req, "subdirectory", None) return cls.from_ireq(ireq, subdir=subdir, finder=finder) @classmethod + @lru_cache() def from_ireq(cls, ireq, subdir=None, finder=None): + # type: (InstallRequirement, Optional[AnyStr], Optional[PackageFinder]) -> Optional[SetupInfo] import pip_shims.shims + if not ireq.link: + return None if ireq.link.is_wheel: - return + return None if not finder: from .dependencies import get_finder finder = get_finder() + _, uri = split_vcs_method_from_uri(unquote(ireq.link.url_without_fragment)) + parsed = urlparse(uri) + if "file" in parsed.scheme: + url_path = parsed.path + if "@" in url_path: + url_path, _, _ = url_path.rpartition("@") + parsed = parsed._replace(path=url_path) + uri = urlunparse(parsed) + path = None + if ireq.link.scheme == "file" or uri.startswith("file://"): + if "file:/" in uri and "file:///" not in uri: + uri = uri.replace("file:/", "file:///") + path = pip_shims.shims.url_to_path(uri) kwargs = _prepare_wheel_building_kwargs(ireq) - ireq.populate_link(finder, False, False) - ireq.ensure_has_source_dir(kwargs["build_dir"]) + ireq.source_dir = kwargs["src_dir"] if not ( ireq.editable and pip_shims.shims.is_file_url(ireq.link) @@ -387,38 +1430,32 @@ class SetupInfo(object): else: only_download = False download_dir = kwargs["download_dir"] - ireq_src_dir = None - if ireq.link.scheme == "file": - path = pip_shims.shims.url_to_path(unquote(ireq.link.url_without_fragment)) - if pip_shims.shims.is_installable_dir(path): - ireq_src_dir = path - if not ireq.editable or not (pip_shims.is_file_url(ireq.link) and ireq_src_dir): - pip_shims.shims.unpack_url( - ireq.link, - ireq.source_dir, - download_dir, - only_download=only_download, - session=finder.session, - hashes=ireq.hashes(False), - progress_bar="off", + elif path is not None and os.path.isdir(path): + raise RequirementError( + "The file URL points to a directory not installable: {}".format(ireq.link) ) - if ireq.editable: - created = cls.create( - ireq.source_dir, subdirectory=subdir, ireq=ireq, kwargs=kwargs - ) - else: - build_dir = ireq.build_location(kwargs["build_dir"]) - ireq._temp_build_dir.path = kwargs["build_dir"] - created = cls.create( - build_dir, subdirectory=subdir, ireq=ireq, kwargs=kwargs - ) - created.get_info() + ireq.build_location(kwargs["build_dir"]) + src_dir = ireq.ensure_has_source_dir(kwargs["src_dir"]) + ireq._temp_build_dir.path = kwargs["build_dir"] + + ireq.populate_link(finder, False, False) + pip_shims.shims.unpack_url( + ireq.link, + src_dir, + download_dir, + only_download=only_download, + session=finder.session, + hashes=ireq.hashes(False), + progress_bar="off", + ) + created = cls.create(src_dir, subdirectory=subdir, ireq=ireq, kwargs=kwargs) return created @classmethod def create(cls, base_dir, subdirectory=None, ireq=None, kwargs=None): + # type: (AnyStr, Optional[AnyStr], Optional[InstallRequirement], Optional[Dict[AnyStr, AnyStr]]) -> Optional[SetupInfo] if not base_dir or base_dir is None: - return + return None creation_kwargs = {"extra_kwargs": kwargs} if not isinstance(base_dir, Path): @@ -435,4 +1472,6 @@ class SetupInfo(object): creation_kwargs["setup_cfg"] = setup_cfg if ireq: creation_kwargs["ireq"] = ireq - return cls(**creation_kwargs) + created = cls(**creation_kwargs) + created.get_initial_info() + return created diff --git a/pipenv/vendor/requirementslib/models/url.py b/pipenv/vendor/requirementslib/models/url.py new file mode 100644 index 00000000..889a4bdd --- /dev/null +++ b/pipenv/vendor/requirementslib/models/url.py @@ -0,0 +1,404 @@ +# -*- coding=utf-8 -*- +from __future__ import absolute_import, print_function + +import attr +import pip_shims.shims +from orderedmultidict import omdict +from six.moves.urllib.parse import quote_plus, unquote_plus +from urllib3 import util as urllib3_util +from urllib3.util import parse_url as urllib3_parse +from urllib3.util.url import Url + +from ..environment import MYPY_RUNNING + +if MYPY_RUNNING: + from typing import List, Tuple, Text, Union, TypeVar, Optional + from pip_shims.shims import Link + from vistir.compat import Path + + _T = TypeVar("_T") + STRING_TYPE = Union[bytes, str, Text] + S = TypeVar("S", bytes, str, Text) + + +def _get_parsed_url(url): + # type: (S) -> Url + """ + This is a stand-in function for `urllib3.util.parse_url` + + The orignal function doesn't handle special characters very well, this simply splits + out the authentication section, creates the parsed url, then puts the authentication + section back in, bypassing validation. + + :return: The new, parsed URL object + :rtype: :class:`~urllib3.util.url.Url` + """ + + try: + parsed = urllib3_parse(url) + except ValueError: + scheme, _, url = url.partition("://") + auth, _, url = url.rpartition("@") + url = "{scheme}://{url}".format(scheme=scheme, url=url) + parsed = urllib3_parse(url)._replace(auth=auth) + return parsed + + +def remove_password_from_url(url): + # type: (S) -> S + """ + Given a url, remove the password and insert 4 dashes + + :param url: The url to replace the authentication in + :type url: S + :return: The new URL without authentication + :rtype: S + """ + + parsed = _get_parsed_url(url) + if parsed.auth: + auth, _, _ = parsed.auth.partition(":") + return parsed._replace(auth="{auth}:----".format(auth=auth)).url + return parsed.url + + +@attr.s +class URI(object): + #: The target hostname, e.g. `amazon.com` + host = attr.ib(type=str) + #: The URI Scheme, e.g. `salesforce` + scheme = attr.ib(default="https", type=str) + #: The numeric port of the url if specified + port = attr.ib(default=None, type=int) + #: The url path, e.g. `/path/to/endpoint` + path = attr.ib(default="", type=str) + #: Query parameters, e.g. `?variable=value...` + query = attr.ib(default="", type=str) + #: URL Fragments, e.g. `#fragment=value` + fragment = attr.ib(default="", type=str) + #: Subdirectory fragment, e.g. `&subdirectory=blah...` + subdirectory = attr.ib(default="", type=str) + #: VCS ref this URI points at, if available + ref = attr.ib(default="", type=str) + #: The username if provided, parsed from `user:password@hostname` + username = attr.ib(default="", type=str) + #: Password parsed from `user:password@hostname` + password = attr.ib(default="", type=str, repr=False) + #: An orderedmultidict representing query fragments + query_dict = attr.ib(factory=omdict, type=omdict) + #: The name of the specified package in case it is a VCS URI with an egg fragment + name = attr.ib(default="", type=str) + #: Any extras requested from the requirement + extras = attr.ib(factory=tuple, type=tuple) + #: Whether the url was parsed as a direct pep508-style URL + is_direct_url = attr.ib(default=False, type=bool) + #: Whether the url was an implicit `git+ssh` url (passed as `git+git@`) + is_implicit_ssh = attr.ib(default=False, type=bool) + _auth = attr.ib(default=None, type=str, repr=False) + _fragment_dict = attr.ib(factory=dict, type=dict) + + def _parse_query(self): + # type: () -> URI + query = self.query if self.query is not None else "" + query_dict = omdict() + queries = query.split("&") + query_items = [] + for q in queries: + key, _, val = q.partition("=") + val = unquote_plus(val.replace("+", " ")) + query_items.append((key, val)) + query_dict.load(query_items) + return attr.evolve(self, query_dict=query_dict, query=query) + + def _parse_fragment(self): + # type: () -> URI + subdirectory = self.subdirectory if self.subdirectory else "" + fragment = self.fragment if self.fragment else "" + if self.fragment is None: + return self + fragments = self.fragment.split("&") + fragment_items = {} + name = self.name if self.name else "" + extras = self.extras + for q in fragments: + key, _, val = q.partition("=") + val = unquote_plus(val.replace("+", " ")) + fragment_items[key] = val + if key == "egg": + from .utils import parse_extras + + name, stripped_extras = pip_shims.shims._strip_extras(val) + if stripped_extras: + extras = tuple(parse_extras(stripped_extras)) + elif key == "subdirectory": + subdirectory = val + return attr.evolve( + self, + fragment_dict=fragment_items, + subdirectory=subdirectory, + fragment=fragment, + extras=extras, + name=name, + ) + + def _parse_auth(self): + # type: () -> URI + if self._auth: + username, _, password = self._auth.partition(":") + password = quote_plus(password) + return attr.evolve(self, username=username, password=password) + return self + + def get_password(self, unquote=False, include_token=True): + # type: (bool, bool) -> str + password = self.password + if password and unquote: + password = unquote_plus(password) + else: + password = "" + return password + + @staticmethod + def parse_subdirectory(url_part): + # type: (str) -> Tuple[str, Optional[str]] + subdir = None + if "&subdirectory" in url_part: + url_part, _, subdir = url_part.rpartition("&") + subdir = "&{0}".format(subdir.strip()) + return url_part.strip(), subdir + + @classmethod + def parse(cls, url): + # type: (S) -> URI + from .utils import DIRECT_URL_RE, split_ref_from_uri + + is_direct_url = False + name_with_extras = None + is_implicit_ssh = url.strip().startswith("git+git@") + if is_implicit_ssh: + from ..utils import add_ssh_scheme_to_git_uri + + url = add_ssh_scheme_to_git_uri(url) + direct_match = DIRECT_URL_RE.match(url) + if direct_match is not None: + is_direct_url = True + name_with_extras, _, url = url.partition("@") + name_with_extras = name_with_extras.strip() + url, ref = split_ref_from_uri(url.strip()) + if "file:/" in url and "file:///" not in url: + url = url.replace("file:/", "file:///") + parsed = _get_parsed_url(url) + if not (parsed.scheme and parsed.host and parsed.path): + # check if this is a file uri + if not ( + parsed.scheme + and parsed.path + and (parsed.scheme == "file" or parsed.scheme.endswith("+file")) + ): + raise ValueError("Failed parsing URL {0!r} - Not a valid url".format(url)) + parsed_dict = dict(parsed._asdict()).copy() + parsed_dict["is_direct_url"] = is_direct_url + parsed_dict["is_implicit_ssh"] = is_implicit_ssh + if name_with_extras: + fragment = "" + if parsed_dict["fragment"] is not None: + fragment = "{0}".format(parsed_dict["fragment"]) + if fragment.startswith("egg="): + name, extras = pip_shims.shims._strip_extras(name_with_extras) + fragment_name, fragment_extras = pip_shims.shims._strip_extras(fragment) + if fragment_extras and not extras: + name_with_extras = "{0}{1}".format(name, fragment_extras) + fragment = "" + elif "&subdirectory" in parsed_dict["path"]: + path, fragment = cls.parse_subdirectory(parsed_dict["path"]) + parsed_dict["path"] = path + elif ref is not None and "&subdirectory" in ref: + ref, fragment = cls.parse_subdirectory(ref) + parsed_dict["fragment"] = "egg={0}{1}".format(name_with_extras, fragment) + if ref is not None: + parsed_dict["ref"] = ref.strip() + return cls(**parsed_dict)._parse_auth()._parse_query()._parse_fragment() + + def to_string( + self, + escape_password=True, # type: bool + unquote=True, # type: bool + direct=None, # type: Optional[bool] + strip_ssh=False, # type: bool + strip_ref=False, # type: bool + strip_name=False, # type: bool + strip_subdir=False, # type: bool + ): + # type: (...) -> str + """ + Converts the current URI to a string, unquoting or escaping the password as needed + + :param escape_password: Whether to replace password with ``----``, default True + :param escape_password: bool, optional + :param unquote: Whether to unquote url-escapes in the password, default False + :param unquote: bool, optional + :param bool direct: Whether to format as a direct URL + :param bool strip_ssh: Whether to strip the SSH scheme from the url (git only) + :param bool strip_ref: Whether to drop the VCS ref (if present) + :param bool strip_name: Whether to drop the name and extras (if present) + :param bool strip_subdir: Whether to drop the subdirectory (if present) + :return: The reconstructed string representing the URI + :rtype: str + """ + + if direct is None: + direct = self.is_direct_url + if escape_password: + password = "----" if (self.password or self.username) else "" + else: + password = self.get_password(unquote=unquote) + auth = "" + if self.username: + if password: + auth = "{self.username}:{password}@".format(password=password, self=self) + else: + auth = "{self.username}@".format(self=self) + query = "" + if self.query: + query = "{query}?{self.query}".format(query=query, self=self) + if not direct: + if self.name and not strip_name: + fragment = "#egg={self.name_with_extras}".format(self=self) + elif not strip_name and ( + self.extras and self.scheme and self.scheme.startswith("file") + ): + from .utils import extras_to_string + + fragment = extras_to_string(self.extras) + else: + fragment = "" + query = "{query}{fragment}".format(query=query, fragment=fragment) + if self.subdirectory and not strip_subdir: + query = "{query}&subdirectory={self.subdirectory}".format( + query=query, self=self + ) + host_port_path = self.get_host_port_path(strip_ref=strip_ref) + url = "{self.scheme}://{auth}{host_port_path}{query}".format( + self=self, auth=auth, host_port_path=host_port_path, query=query + ) + if strip_ssh: + from ..utils import strip_ssh_from_git_uri + + url = strip_ssh_from_git_uri(url) + if self.name and direct and not strip_name: + return "{self.name_with_extras}@ {url}".format(self=self, url=url) + return url + + def get_host_port_path(self, strip_ref=False): + # type: (bool) -> str + host = self.host if self.host else "" + if self.port: + host = "{host}:{self.port!s}".format(host=host, self=self) + path = "{self.path}".format(self=self) + if self.ref and not strip_ref: + path = "{path}@{self.ref}".format(path=path, self=self) + return "{host}{path}".format(host=host, path=path) + + @property + def name_with_extras(self): + # type: () -> str + from .utils import extras_to_string + + if not self.name: + return "" + extras = extras_to_string(self.extras) + return "{self.name}{extras}".format(self=self, extras=extras) + + @property + def as_link(self): + # type: () -> Link + link = pip_shims.shims.Link( + self.to_string(escape_password=False, strip_ssh=False, direct=False) + ) + return link + + @property + def bare_url(self): + # type: () -> str + return self.to_string( + escape_password=False, + strip_ssh=self.is_implicit_ssh, + direct=False, + strip_name=True, + strip_ref=True, + strip_subdir=True, + ) + + @property + def url_without_fragment_or_ref(self): + # type: () -> str + return self.to_string( + escape_password=False, + strip_ssh=self.is_implicit_ssh, + direct=False, + strip_name=True, + strip_ref=True, + ) + + @property + def url_without_fragment(self): + # type: () -> str + return self.to_string( + escape_password=False, + strip_ssh=self.is_implicit_ssh, + direct=False, + strip_name=True, + ) + + @property + def url_without_ref(self): + # type: () -> str + return self.to_string( + escape_password=False, + strip_ssh=self.is_implicit_ssh, + direct=False, + strip_ref=True, + ) + + @property + def base_url(self): + # type: () -> str + return self.to_string( + escape_password=False, strip_ssh=self.is_implicit_ssh, direct=False + ) + + @property + def full_url(self): + # type: () -> str + return self.to_string(escape_password=False, strip_ssh=False, direct=False) + + @property + def safe_string(self): + # type: () -> str + return self.to_string(escape_password=True, unquote=True) + + @property + def unsafe_string(self): + # type: () -> str + return self.to_string(escape_password=False, unquote=True) + + @property + def uri_escape(self): + # type: () -> str + return self.to_string(escape_password=False, unquote=False) + + @property + def is_vcs(self): + # type: () -> bool + from ..utils import VCS_SCHEMES + + return self.scheme in VCS_SCHEMES + + @property + def is_file_url(self): + # type: () -> bool + return all([self.scheme, self.scheme == "file"]) + + def __str__(self): + # type: () -> str + return self.to_string(escape_password=True, unquote=True) diff --git a/pipenv/vendor/requirementslib/models/utils.py b/pipenv/vendor/requirementslib/models/utils.py index 0fac2aa3..4b497954 100644 --- a/pipenv/vendor/requirementslib/models/utils.py +++ b/pipenv/vendor/requirementslib/models/utils.py @@ -1,47 +1,198 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import +from __future__ import absolute_import, print_function import io import os +import re +import string import sys - from collections import defaultdict from itertools import chain, groupby from operator import attrgetter import six import tomlkit - from attr import validators from first import first from packaging.markers import InvalidMarker, Marker, Op, Value, Variable from packaging.specifiers import InvalidSpecifier, Specifier, SpecifierSet +from packaging.version import parse as parse_version +from plette.models import Package, PackageCollection +from six.moves.urllib import parse as urllib_parse +from tomlkit.container import Container +from tomlkit.items import AoT, Array, Bool, InlineTable, Item, String, Table +from urllib3 import util as urllib3_util +from vistir.compat import lru_cache from vistir.misc import dedup +from vistir.path import is_valid_url + +from ..environment import MYPY_RUNNING +from ..utils import SCHEME_LIST, VCS_LIST, is_star + +if MYPY_RUNNING: + from typing import ( + Union, + Optional, + List, + Set, + Any, + TypeVar, + Tuple, + Sequence, + Dict, + Text, + AnyStr, + Match, + Iterable, # noqa + ) + from attr import _ValidatorType # noqa + from packaging.requirements import Requirement as PackagingRequirement + from pkg_resources import Requirement as PkgResourcesRequirement + from pkg_resources.extern.packaging.markers import ( + Op as PkgResourcesOp, + Variable as PkgResourcesVariable, + Value as PkgResourcesValue, + Marker as PkgResourcesMarker, + ) + from pip_shims.shims import Link + from vistir.compat import Path + + _T = TypeVar("_T") + TMarker = Union[Marker, PkgResourcesMarker] + TVariable = TypeVar("TVariable", PkgResourcesVariable, Variable) + TValue = TypeVar("TValue", PkgResourcesValue, Value) + TOp = TypeVar("TOp", PkgResourcesOp, Op) + MarkerTuple = Tuple[TVariable, TOp, TValue] + TRequirement = Union[PackagingRequirement, PkgResourcesRequirement] + STRING_TYPE = Union[bytes, str, Text] + TOML_DICT_TYPES = Union[Container, Package, PackageCollection, Table, InlineTable] + S = TypeVar("S", bytes, str, Text) -from ..utils import SCHEME_LIST, VCS_LIST, is_star, add_ssh_scheme_to_git_uri - +TOML_DICT_OBJECTS = (Container, Package, Table, InlineTable, PackageCollection) +TOML_DICT_NAMES = [o.__class__.__name__ for o in TOML_DICT_OBJECTS] HASH_STRING = " --hash={0}" +ALPHA_NUMERIC = r"[{0}{1}]".format(string.ascii_letters, string.digits) +PUNCTUATION = r"[\-_\.]" +ALPHANUM_PUNCTUATION = r"[{0}{1}\-_\.]".format(string.ascii_letters, string.digits) +NAME = r"{0}+{1}*{2}".format(ALPHANUM_PUNCTUATION, PUNCTUATION, ALPHA_NUMERIC) +REF = r"[{0}{1}\-\_\./]".format(string.ascii_letters, string.digits) +EXTRAS = r"(?P\[{0}(?:,{0})*\])".format(NAME) +NAME_WITH_EXTRAS = r"(?P{0}){1}?".format(NAME, EXTRAS) +NAME_RE = re.compile(NAME_WITH_EXTRAS) +SUBDIR_RE = r"(?:[&#]subdirectory=(?P.*))" +URL_NAME = r"(?:#egg={0})".format(NAME_WITH_EXTRAS) +REF_RE = r"(?:@(?P{0}+)?)".format(REF) +PATH_RE = r"(?P[:/])(?P[^ @]+){0}?".format(REF_RE) +PASS_RE = r"(?:(?<=:)(?P[^ ]+))" +AUTH_RE = r"(?:(?P[^ ]+)[:@]{0}?@)".format(PASS_RE) +HOST_RE = r"(?:{0}?(?P[^ ]+?\.?{1}+(?P:\d+)?))?".format( + AUTH_RE, ALPHA_NUMERIC +) +URL = r"(?P[^ ]+://){0}{1}".format(HOST_RE, PATH_RE) +URL_RE = re.compile(r"{0}(?:{1}?{2}?)?".format(URL, URL_NAME, SUBDIR_RE)) +DIRECT_URL_RE = re.compile(r"{0}\s?@\s?{1}".format(NAME_WITH_EXTRAS, URL)) + def filter_none(k, v): + # type: (AnyStr, Any) -> bool if v: return True return False def optional_instance_of(cls): + # type: (Any) -> _ValidatorType[Optional[_T]] return validators.optional(validators.instance_of(cls)) def create_link(link): - from pip_shims import Link + # type: (AnyStr) -> Link + + if not isinstance(link, six.string_types): + raise TypeError("must provide a string to instantiate a new link") + from pip_shims.shims import Link + return Link(link) +def tomlkit_value_to_python(toml_value): + # type: (Union[Array, AoT, TOML_DICT_TYPES, Item]) -> Union[List, Dict] + value_type = type(toml_value).__name__ + if ( + isinstance(toml_value, TOML_DICT_OBJECTS + (dict,)) + or value_type in TOML_DICT_NAMES + ): + return tomlkit_dict_to_python(toml_value) + elif isinstance(toml_value, AoT) or value_type == "AoT": + return [tomlkit_value_to_python(val) for val in toml_value._body] + elif isinstance(toml_value, Array) or value_type == "Array": + return [tomlkit_value_to_python(val) for val in list(toml_value)] + elif isinstance(toml_value, String) or value_type == "String": + return "{0!s}".format(toml_value) + elif isinstance(toml_value, Bool) or value_type == "Bool": + return toml_value.value + elif isinstance(toml_value, Item): + return toml_value.value + return toml_value + + +def tomlkit_dict_to_python(toml_dict): + # type: (TOML_DICT_TYPES) -> Dict + value_type = type(toml_dict).__name__ + if toml_dict is None: + raise TypeError("Invalid type NoneType when converting toml dict to python") + converted = None # type: Optional[Dict] + if isinstance(toml_dict, (InlineTable, Table)) or value_type in ( + "InlineTable", + "Table", + ): + converted = toml_dict.value + elif isinstance(toml_dict, (Package, PackageCollection)) or value_type in ( + "Package, PackageCollection" + ): + converted = toml_dict._data + if isinstance(converted, Container) or type(converted).__name__ == "Container": + converted = converted.value + elif isinstance(toml_dict, Container) or value_type == "Container": + converted = toml_dict.value + elif isinstance(toml_dict, dict): + converted = toml_dict.copy() + else: + raise TypeError( + "Invalid type for conversion: expected Container, Dict, or Table, " + "got {0!r}".format(toml_dict) + ) + if isinstance(converted, dict): + return {k: tomlkit_value_to_python(v) for k, v in converted.items()} + elif isinstance(converted, (TOML_DICT_OBJECTS)) or value_type in TOML_DICT_NAMES: + return tomlkit_dict_to_python(converted) + return converted + + +def get_url_name(url): + # type: (AnyStr) -> AnyStr + """ + Given a url, derive an appropriate name to use in a pipfile. + + :param str url: A url to derive a string from + :returns: The name of the corresponding pipfile entry + :rtype: Text + """ + if not isinstance(url, six.string_types): + raise TypeError("Expected a string, got {0!r}".format(url)) + return urllib3_util.parse_url(url).host + + def init_requirement(name): + # type: (AnyStr) -> TRequirement + + if not isinstance(name, six.string_types): + raise TypeError("must supply a name to generate a requirement") from pkg_resources import Requirement + req = Requirement.parse(name) req.vcs = None req.local_file = None @@ -51,73 +202,274 @@ def init_requirement(name): def extras_to_string(extras): - """Turn a list of extras into a string""" + # type: (Iterable[S]) -> S + """Turn a list of extras into a string + + :param List[str]] extras: a list of extras to format + :return: A string of extras + :rtype: str + """ if isinstance(extras, six.string_types): if extras.startswith("["): return extras - else: extras = [extras] - return "[{0}]".format(",".join(sorted(extras))) + if not extras: + return "" + return "[{0}]".format(",".join(sorted(set(extras)))) # type: ignore def parse_extras(extras_str): - """Turn a string of extras into a parsed extras list""" + # type: (AnyStr) -> List[AnyStr] + """Turn a string of extras into a parsed extras list + + :param str extras_str: An extras string + :return: A sorted list of extras + :rtype: List[str] + """ + from pkg_resources import Requirement + extras = Requirement.parse("fakepkg{0}".format(extras_to_string(extras_str))).extras return sorted(dedup([extra.lower() for extra in extras])) def specs_to_string(specs): - """Turn a list of specifier tuples into a string""" + # type: (List[Union[STRING_TYPE, Specifier]]) -> AnyStr + """Turn a list of specifier tuples into a string + + :param List[Union[Specifier, str]] specs: a list of specifiers to format + :return: A string of specifiers + :rtype: str + """ + if specs: if isinstance(specs, six.string_types): return specs try: extras = ",".join(["".join(spec) for spec in specs]) except TypeError: - extras = ",".join(["".join(spec._spec) for spec in specs]) + extras = ",".join(["".join(spec._spec) for spec in specs]) # type: ignore return extras return "" -def build_vcs_link(vcs, uri, name=None, ref=None, subdirectory=None, extras=None): +def build_vcs_uri( + vcs, # type: Optional[S] + uri, # type: S + name=None, # type: Optional[S] + ref=None, # type: Optional[S] + subdirectory=None, # type: Optional[S] + extras=None, # type: Optional[Iterable[S]] +): + # type: (...) -> STRING_TYPE if extras is None: extras = [] - vcs_start = "{0}+".format(vcs) - if not uri.startswith(vcs_start): - uri = "{0}{1}".format(vcs_start, uri) - uri = add_ssh_scheme_to_git_uri(uri) + vcs_start = "" + if vcs is not None: + vcs_start = "{0}+".format(vcs) + if not uri.startswith(vcs_start): + uri = "{0}{1}".format(vcs_start, uri) if ref: uri = "{0}@{1}".format(uri, ref) if name: uri = "{0}#egg={1}".format(uri, name) if extras: - extras = extras_to_string(extras) - uri = "{0}{1}".format(uri, extras) + extras_string = extras_to_string(extras) + uri = "{0}{1}".format(uri, extras_string) if subdirectory: uri = "{0}&subdirectory={1}".format(uri, subdirectory) - return create_link(uri) + return uri + + +def convert_direct_url_to_url(direct_url): + # type: (AnyStr) -> AnyStr + """Converts direct URLs to standard, link-style URLs + + Given a direct url as defined by *PEP 508*, convert to a :class:`~pip_shims.shims.Link` + compatible URL by moving the name and extras into an **egg_fragment**. + + :param str direct_url: A pep-508 compliant direct url. + :return: A reformatted URL for use with Link objects and :class:`~pip_shims.shims.InstallRequirement` objects. + :rtype: AnyStr + """ + direct_match = DIRECT_URL_RE.match(direct_url) # type: Optional[Match] + if direct_match is None: + url_match = URL_RE.match(direct_url) + if url_match or is_valid_url(direct_url): + return direct_url + match_dict = ( + {} + ) # type: Dict[STRING_TYPE, Union[Tuple[STRING_TYPE, ...], STRING_TYPE]] + if direct_match is not None: + match_dict = direct_match.groupdict() # type: ignore + if not match_dict: + raise ValueError( + "Failed converting value to normal URL, is it a direct URL? {0!r}".format( + direct_url + ) + ) + url_segments = [match_dict.get(s) for s in ("scheme", "host", "path", "pathsep")] + url = "" # type: STRING_TYPE + url = "".join([s for s in url_segments if s is not None]) # type: ignore + new_url = build_vcs_uri( + None, + url, + ref=match_dict.get("ref"), + name=match_dict.get("name"), + extras=match_dict.get("extras"), + subdirectory=match_dict.get("subdirectory"), + ) + return new_url + + +def convert_url_to_direct_url(url, name=None): + # type: (AnyStr, Optional[AnyStr]) -> AnyStr + """ + Converts normal link-style URLs to direct urls. + + Given a :class:`~pip_shims.shims.Link` compatible URL, convert to a direct url as + defined by *PEP 508* by extracting the name and extras from the **egg_fragment**. + + :param AnyStr url: A :class:`~pip_shims.shims.InstallRequirement` compliant URL. + :param Optiona[AnyStr] name: A name to use in case the supplied URL doesn't provide one. + :return: A pep-508 compliant direct url. + :rtype: AnyStr + + :raises ValueError: Raised when the URL can't be parsed or a name can't be found. + :raises TypeError: When a non-string input is provided. + """ + if not isinstance(url, six.string_types): + raise TypeError( + "Expected a string to convert to a direct url, got {0!r}".format(url) + ) + direct_match = DIRECT_URL_RE.match(url) + if direct_match: + return url + url_match = URL_RE.match(url) + if url_match is None or not url_match.groupdict(): + raise ValueError("Failed parse a valid URL from {0!r}".format(url)) + match_dict = url_match.groupdict() + url_segments = [match_dict.get(s) for s in ("scheme", "host", "path", "pathsep")] + name = match_dict.get("name", name) + extras = match_dict.get("extras") + new_url = "" + if extras and not name: + url_segments.append(extras) + elif extras and name: + new_url = "{0}{1}@ ".format(name, extras) + else: + if name is not None: + new_url = "{0}@ ".format(name) + else: + raise ValueError( + "Failed to construct direct url: " + "No name could be parsed from {0!r}".format(url) + ) + if match_dict.get("ref"): + url_segments.append("@{0}".format(match_dict.get("ref"))) + url = "".join([s for s in url if s is not None]) + url = "{0}{1}".format(new_url, url) + return url def get_version(pipfile_entry): + # type: (Union[STRING_TYPE, Dict[STRING_TYPE, Union[STRING_TYPE, bool, Iterable[STRING_TYPE]]]]) -> STRING_TYPE if str(pipfile_entry) == "{}" or is_star(pipfile_entry): return "" - elif hasattr(pipfile_entry, "keys") and "version" in pipfile_entry: + if hasattr(pipfile_entry, "keys") and "version" in pipfile_entry: if is_star(pipfile_entry.get("version")): return "" - return pipfile_entry.get("version", "") + return pipfile_entry.get("version", "").strip().lstrip("(").rstrip(")") if isinstance(pipfile_entry, six.string_types): - return pipfile_entry + return pipfile_entry.strip().lstrip("(").rstrip(")") return "" +def strip_extras_markers_from_requirement(req): + # type: (TRequirement) -> TRequirement + """ + Strips extras markers from requirement instances. + + Given a :class:`~packaging.requirements.Requirement` instance with markers defining + *extra == 'name'*, strip out the extras from the markers and return the cleaned + requirement + + :param PackagingRequirement req: A packaging requirement to clean + :return: A cleaned requirement + :rtype: PackagingRequirement + """ + if req is None: + raise TypeError("Must pass in a valid requirement, received {0!r}".format(req)) + if getattr(req, "marker", None) is not None: + marker = req.marker # type: TMarker + marker._markers = _strip_extras_markers(marker._markers) + if not marker._markers: + req.marker = None + else: + req.marker = marker + return req + + +def _strip_extras_markers(marker): + # type: (Union[MarkerTuple, List[Union[MarkerTuple, str]]]) -> List[Union[MarkerTuple, str]] + if marker is None or not isinstance(marker, (list, tuple)): + raise TypeError("Expecting a marker type, received {0!r}".format(marker)) + markers_to_remove = [] + # iterate forwards and generate a list of indexes to remove first, then reverse the + # list so we can remove the text that normally occurs after (but we will already + # be past it in the loop) + for i, marker_list in enumerate(marker): + if isinstance(marker_list, list): + cleaned = _strip_extras_markers(marker_list) + if not cleaned: + markers_to_remove.append(i) + elif isinstance(marker_list, tuple) and marker_list[0].value == "extra": + markers_to_remove.append(i) + for i in reversed(markers_to_remove): + del marker[i] + if i > 0 and marker[i - 1] == "and": + del marker[i - 1] + return marker + + +@lru_cache() +def get_setuptools_version(): + # type: () -> Optional[STRING_TYPE] + import pkg_resources + + setuptools_dist = pkg_resources.get_distribution( + pkg_resources.Requirement("setuptools") + ) + return getattr(setuptools_dist, "version", None) + + +def get_default_pyproject_backend(): + # type: () -> STRING_TYPE + st_version = get_setuptools_version() + if st_version is not None: + parsed_st_version = parse_version(st_version) + if parsed_st_version >= parse_version("40.8.0"): + return "setuptools.build_meta:__legacy__" + return "setuptools.build_meta" + + def get_pyproject(path): - from vistir.compat import Path + # type: (Union[STRING_TYPE, Path]) -> Optional[Tuple[List[STRING_TYPE], STRING_TYPE]] + """ + Given a base path, look for the corresponding ``pyproject.toml`` file and return its + build_requires and build_backend. + + :param AnyStr path: The root path of the project, should be a directory (will be truncated) + :return: A 2 tuple of build requirements and the build backend + :rtype: Optional[Tuple[List[AnyStr], AnyStr]] + """ if not path: return + from vistir.compat import Path + if not isinstance(path, Path): path = Path(path) if not path.is_dir(): @@ -125,8 +477,10 @@ def get_pyproject(path): pp_toml = path.joinpath("pyproject.toml") setup_py = path.joinpath("setup.py") if not pp_toml.exists(): - if setup_py.exists(): + if not setup_py.exists(): return None + requires = ["setuptools>=40.8", "wheel"] + backend = get_default_pyproject_backend() else: pyproject_data = {} with io.open(pp_toml.as_posix(), encoding="utf-8") as fh: @@ -134,24 +488,26 @@ def get_pyproject(path): build_system = pyproject_data.get("build-system", None) if build_system is None: if setup_py.exists(): - requires = ["setuptools", "wheel"] - backend = "setuptools.build_meta" + requires = ["setuptools>=40.8", "wheel"] + backend = get_default_pyproject_backend() else: - requires = ["setuptools>=38.2.5", "wheel"] - backend = "setuptools.build_meta" - build_system = { - "requires": requires, - "build-backend": backend - } + requires = ["setuptools>=40.8", "wheel"] + backend = get_default_pyproject_backend() + build_system = {"requires": requires, "build-backend": backend} pyproject_data["build_system"] = build_system else: - requires = build_system.get("requires") - backend = build_system.get("build-backend") - return (requires, backend) + requires = build_system.get("requires", ["setuptools>=40.8", "wheel"]) + backend = build_system.get("build-backend", get_default_pyproject_backend()) + return requires, backend def split_markers_from_line(line): + # type: (AnyStr) -> Tuple[AnyStr, Optional[AnyStr]] """Split markers from a dependency""" + quote_chars = ["'", '"'] + line_quote = next(iter(quote for quote in quote_chars if line.startswith(quote)), None) + if line_quote and line.endswith(line_quote): + line = line.strip(line_quote) if not any(line.startswith(uri_prefix) for uri_prefix in SCHEME_LIST): marker_sep = ";" else: @@ -164,14 +520,37 @@ def split_markers_from_line(line): def split_vcs_method_from_uri(uri): + # type: (AnyStr) -> Tuple[Optional[STRING_TYPE], STRING_TYPE] """Split a vcs+uri formatted uri into (vcs, uri)""" vcs_start = "{0}+" + vcs = None # type: Optional[STRING_TYPE] vcs = first([vcs for vcs in VCS_LIST if uri.startswith(vcs_start.format(vcs))]) if vcs: vcs, uri = uri.split("+", 1) return vcs, uri +def split_ref_from_uri(uri): + # type: (AnyStr) -> Tuple[AnyStr, Optional[AnyStr]] + """ + Given a path or URI, check for a ref and split it from the path if it is present, + returning a tuple of the original input and the ref or None. + + :param AnyStr uri: The path or URI to split + :returns: A 2-tuple of the path or URI and the ref + :rtype: Tuple[AnyStr, Optional[AnyStr]] + """ + if not isinstance(uri, six.string_types): + raise TypeError("Expected a string, received {0!r}".format(uri)) + parsed = urllib_parse.urlparse(uri) + path = parsed.path + ref = None + if parsed.scheme != "file" and "@" in path: + path, _, ref = path.rpartition("@") + parsed = parsed._replace(path=path) + return (urllib_parse.urlunparse(parsed), ref) + + def validate_vcs(instance, attr_, value): if value not in VCS_LIST: raise ValueError("Invalid vcs {0!r}".format(value)) @@ -208,28 +587,27 @@ def key_from_ireq(ireq): def key_from_req(req): """Get an all-lowercase version of the requirement's name.""" - if hasattr(req, 'key'): + if hasattr(req, "key"): # from pkg_resources, such as installed dists for pip-sync key = req.key else: # from packaging, such as install requirements from requirements.txt key = req.name - key = key.replace('_', '-').lower() + key = key.replace("_", "-").lower() return key def _requirement_to_str_lowercase_name(requirement): - """ - Formats a packaging.requirements.Requirement with a lowercase name. + """Formats a packaging.requirements.Requirement with a lowercase name. This is simply a copy of https://github.com/pypa/packaging/blob/16.8/packaging/requirements.py#L109-L124 modified to lowercase the dependency name. Previously, we were invoking the original Requirement.__str__ method and - lowercasing the entire result, which would lowercase the name, *and* other, - important stuff that should not be lowercased (such as the marker). See + lower-casing the entire result, which would lowercase the name, *and* other, + important stuff that should not be lower-cased (such as the marker). See this issue for more information: https://github.com/pypa/pipenv/issues/2113. """ parts = [requirement.name.lower()] @@ -250,35 +628,44 @@ def _requirement_to_str_lowercase_name(requirement): def format_requirement(ireq): - """ + """Formats an `InstallRequirement` instance as a string. + Generic formatter for pretty printing InstallRequirements to the terminal in a less verbose way than using its `__str__` method. + + :param :class:`InstallRequirement` ireq: A pip **InstallRequirement** instance. + :return: A formatted string for prettyprinting + :rtype: str """ if ireq.editable: - line = '-e {}'.format(ireq.link) + line = "-e {}".format(ireq.link) else: line = _requirement_to_str_lowercase_name(ireq.req) if str(ireq.req.marker) != str(ireq.markers): if not ireq.req.marker: - line = '{}; {}'.format(line, ireq.markers) + line = "{}; {}".format(line, ireq.markers) else: name, markers = line.split(";", 1) markers = markers.strip() - line = '{}; ({}) and ({})'.format(name, markers, ireq.markers) + line = "{}; ({}) and ({})".format(name, markers, ireq.markers) return line def format_specifier(ireq): - """ - Generic formatter for pretty printing the specifier part of - InstallRequirements to the terminal. + """Generic formatter for pretty printing specifiers. + + Pretty-prints specifiers from InstallRequirements for output to terminal. + + :param :class:`InstallRequirement` ireq: A pip **InstallRequirement** instance. + :return: A string of specifiers in the given install requirement or + :rtype: str """ # TODO: Ideally, this is carried over to the pip library itself specs = ireq.specifier._specs if ireq.req is not None else [] specs = sorted(specs, key=lambda x: x._spec[1]) - return ','.join(str(s) for s in specs) or '' + return ",".join(str(s) for s in specs) or "" def get_pinned_version(ireq): @@ -303,9 +690,7 @@ def get_pinned_version(ireq): try: specifier = ireq.specifier except AttributeError: - raise TypeError("Expected InstallRequirement, not {}".format( - type(ireq).__name__, - )) + raise TypeError("Expected InstallRequirement, not {}".format(type(ireq).__name__)) if ireq.editable: raise ValueError("InstallRequirement is editable") @@ -315,16 +700,15 @@ def get_pinned_version(ireq): raise ValueError("InstallRequirement has multiple specifications") op, version = next(iter(specifier._specs))._spec - if op not in ('==', '===') or version.endswith('.*'): - raise ValueError("InstallRequirement not pinned (is {0!r})".format( - op + version, - )) + if op not in ("==", "===") or version.endswith(".*"): + raise ValueError("InstallRequirement not pinned (is {0!r})".format(op + version)) return version def is_pinned_requirement(ireq): - """Returns whether an InstallRequirement is a "pinned" requirement. + """ + Returns whether an InstallRequirement is a "pinned" requirement. An InstallRequirement is considered pinned if: @@ -339,6 +723,7 @@ def is_pinned_requirement(ireq): django~=1.8 # NOT pinned django==1.* # NOT pinned """ + try: get_pinned_version(ireq) except (TypeError, ValueError): @@ -350,8 +735,9 @@ def as_tuple(ireq): """ Pulls out the (name: str, version:str, extras:(str)) tuple from the pinned InstallRequirement. """ + if not is_pinned_requirement(ireq): - raise TypeError('Expected a pinned InstallRequirement, got {}'.format(ireq)) + raise TypeError("Expected a pinned InstallRequirement, got {}".format(ireq)) name = key_from_req(ireq.req) version = first(ireq.specifier._specs)._spec[1] @@ -360,12 +746,18 @@ def as_tuple(ireq): def full_groupby(iterable, key=None): - """Like groupby(), but sorts the input on the group key first.""" + """ + Like groupby(), but sorts the input on the group key first. + """ + return groupby(sorted(iterable, key=key), key=key) def flat_map(fn, collection): - """Map a function over a collection and flatten the result by one-level""" + """ + Map a function over a collection and flatten the result by one-level + """ + return chain.from_iterable(map(fn, collection)) @@ -385,8 +777,7 @@ def lookup_table(values, key=None, keyval=None, unique=False, use_lists=False): For key functions that uniquely identify values, set unique=True: >>> assert lookup_table( - ... ['foo', 'bar', 'baz', 'qux', 'quux'], lambda s: s[0], - ... unique=True) == { + ... ['foo', 'bar', 'baz', 'qux', 'quux'], lambda s: s[0], unique=True) == { ... 'b': 'baz', ... 'f': 'foo', ... 'q': 'quux' @@ -404,13 +795,13 @@ def lookup_table(values, key=None, keyval=None, unique=False, use_lists=False): ... 'f': {'oo'}, ... 'q': {'uux', 'ux'} ... } - """ + if keyval is None: if key is None: - keyval = (lambda v: v) + keyval = lambda v: v else: - keyval = (lambda v: (key(v), v)) + keyval = lambda v: (key(v), v) if unique: return dict(keyval(v) for v in values) @@ -434,7 +825,7 @@ def lookup_table(values, key=None, keyval=None, unique=False, use_lists=False): def name_from_req(req): """Get the name of the requirement""" - if hasattr(req, 'project_name'): + if hasattr(req, "project_name"): # from pkg_resources, such as installed dists for pip-sync return req.project_name else: @@ -442,8 +833,11 @@ def name_from_req(req): return req.name -def make_install_requirement(name, version, extras, markers, constraint=False): - """make_install_requirement Generates an :class:`~pip._internal.req.req_install.InstallRequirement`. +def make_install_requirement( + name, version=None, extras=None, markers=None, constraint=False +): + """ + Generates an :class:`~pip._internal.req.req_install.InstallRequirement`. Create an InstallRequirement from the supplied metadata. @@ -463,19 +857,18 @@ def make_install_requirement(name, version, extras, markers, constraint=False): # If no extras are specified, the extras string is blank from pip_shims.shims import install_req_from_line + extras_string = "" + requirement_string = "{0}".format(name) if extras: # Sort extras for stability extras_string = "[{}]".format(",".join(sorted(extras))) - - if not markers: - return install_req_from_line( - str('{}{}=={}'.format(name, extras_string, version)), - constraint=constraint) - else: - return install_req_from_line( - str('{}{}=={}; {}'.format(name, extras_string, version, str(markers))), - constraint=constraint) + requirement_string = "{0}{1}".format(requirement_string, extras_string) + if version: + requirement_string = "{0}=={1}".format(requirement_string, str(version)) + if markers: + requirement_string = "{0}; {1}".format(requirement_string, str(markers)) + return install_req_from_line(requirement_string, constraint=constraint) def version_from_ireq(ireq): @@ -493,9 +886,10 @@ def version_from_ireq(ireq): def clean_requires_python(candidates): """Get a cleaned list of all the candidates with valid specifiers in the `requires_python` attributes.""" all_candidates = [] - sys_version = '.'.join(map(str, sys.version_info[:3])) + sys_version = ".".join(map(str, sys.version_info[:3])) from packaging.version import parse as parse_version - py_version = parse_version(os.environ.get('PIP_PYTHON_VERSION', sys_version)) + + py_version = parse_version(os.environ.get("PIP_PYTHON_VERSION", sys_version)) for c in candidates: from_location = attrgetter("location.requires_python") requires_python = getattr(c, "requires_python", from_location(c)) @@ -503,7 +897,9 @@ def clean_requires_python(candidates): # Old specifications had people setting this to single digits # which is effectively the same as '>=digit,=2.6"')` - marker_key = Variable('python_version') + marker_key = Variable("python_version") for spec in specifierset: operator, val = spec._spec cleaned_val = Value(val).serialize().replace('"', "") spec_dict[Op(operator).serialize()].add(cleaned_val) - marker_str = ' and '.join([ - "{0}{1}'{2}'".format(marker_key.serialize(), op, ','.join(vals)) - for op, vals in spec_dict.items() - ]) - marker_to_add = PackagingRequirement('fakepkg; {0}'.format(marker_str)).marker + marker_str = " and ".join( + [ + "{0}{1}'{2}'".format(marker_key.serialize(), op, ",".join(vals)) + for op, vals in spec_dict.items() + ] + ) + marker_to_add = PackagingRequirement("fakepkg; {0}".format(marker_str)).marker return marker_to_add def normalize_name(pkg): + # type: (AnyStr) -> AnyStr """Given a package name, return its normalized, non-canonicalized form. - :param str pkg: The name of a package + :param AnyStr pkg: The name of a package :return: A normalized package name - :rtype: str + :rtype: AnyStr """ assert isinstance(pkg, six.string_types) return pkg.replace("_", "-").lower() + + +def get_name_variants(pkg): + # type: (STRING_TYPE) -> Set[STRING_TYPE] + """ + Given a packager name, get the variants of its name for both the canonicalized + and "safe" forms. + + :param AnyStr pkg: The package to lookup + :returns: A list of names. + :rtype: Set + """ + + if not isinstance(pkg, six.string_types): + raise TypeError("must provide a string to derive package names") + from pkg_resources import safe_name + from packaging.utils import canonicalize_name + + pkg = pkg.lower() + names = {safe_name(pkg), canonicalize_name(pkg), pkg.replace("-", "_")} + return names + + +def read_source(path, encoding="utf-8"): + # type: (S, S) -> S + """ + Read a source file and get the contents with proper encoding for Python 2/3. + + :param AnyStr path: the file path + :param AnyStr encoding: the encoding that defaults to UTF-8 + :returns: The contents of the source file + :rtype: AnyStr + """ + if six.PY3: + with open(path, "r", encoding=encoding) as fp: + return fp.read() + else: + with open(path, "r") as fp: + return fp.read() + + +SETUPTOOLS_SHIM = ( + "import setuptools, tokenize;__file__=%r;" + "f=getattr(tokenize, 'open', open)(__file__);" + "code=f.read().replace('\\r\\n', '\\n');" + "f.close();" + "exec(compile(code, __file__, 'exec'))" +) diff --git a/pipenv/vendor/requirementslib/models/vcs.py b/pipenv/vendor/requirementslib/models/vcs.py index 6a15db3f..9296f605 100644 --- a/pipenv/vendor/requirementslib/models/vcs.py +++ b/pipenv/vendor/requirementslib/models/vcs.py @@ -1,11 +1,18 @@ # -*- coding=utf-8 -*- +from __future__ import absolute_import, print_function + import attr +import importlib import os import pip_shims +import six +import sys -@attr.s +@attr.s(hash=True) class VCSRepository(object): + DEFAULT_RUN_ARGS = None + url = attr.ib() name = attr.ib() checkout_directory = attr.ib() @@ -14,13 +21,21 @@ class VCSRepository(object): commit_sha = attr.ib(default=None) ref = attr.ib(default=None) repo_instance = attr.ib() + clone_log = attr.ib(default=None) @repo_instance.default def get_repo_instance(self): - from pip_shims import VcsSupport + if self.DEFAULT_RUN_ARGS is None: + default_run_args = self.monkeypatch_pip() + else: + default_run_args = self.DEFAULT_RUN_ARGS + from pip_shims.shims import VcsSupport VCS_SUPPORT = VcsSupport() backend = VCS_SUPPORT._registry.get(self.vcs_type) - return backend(url=self.url) + repo = backend(url=self.url) + if repo.run_command.__func__.__defaults__ != default_run_args: + repo.run_command.__func__.__defaults__ = default_run_args + return repo @property def is_local(self): @@ -58,3 +73,22 @@ class VCSRepository(object): def get_commit_hash(self, ref=None): return self.repo_instance.get_revision(self.checkout_directory) + + @classmethod + def monkeypatch_pip(cls): + target_module = pip_shims.shims.VcsSupport.__module__ + pip_vcs = importlib.import_module(target_module) + run_command_defaults = pip_vcs.VersionControl.run_command.__defaults__ + # set the default to not write stdout, the first option sets this value + new_defaults = [False,] + list(run_command_defaults)[1:] + new_defaults = tuple(new_defaults) + if six.PY3: + try: + pip_vcs.VersionControl.run_command.__defaults__ = new_defaults + except AttributeError: + pip_vcs.VersionControl.run_command.__func__.__defaults__ = new_defaults + else: + pip_vcs.VersionControl.run_command.__func__.__defaults__ = new_defaults + sys.modules[target_module] = pip_vcs + cls.DEFAULT_RUN_ARGS = new_defaults + return new_defaults diff --git a/pipenv/vendor/requirementslib/utils.py b/pipenv/vendor/requirementslib/utils.py index f3653e32..503a13d0 100644 --- a/pipenv/vendor/requirementslib/utils.py +++ b/pipenv/vendor/requirementslib/utils.py @@ -1,24 +1,45 @@ # -*- coding=utf-8 -*- -from __future__ import absolute_import +from __future__ import absolute_import, print_function -import contextlib import logging import os - -import six import sys -import tomlkit - -six.add_move(six.MovedAttribute("Mapping", "collections", "collections.abc")) -six.add_move(six.MovedAttribute("Sequence", "collections", "collections.abc")) -six.add_move(six.MovedAttribute("Set", "collections", "collections.abc")) -six.add_move(six.MovedAttribute("ItemsView", "collections", "collections.abc")) -from six.moves import Mapping, Sequence, Set, ItemsView -from six.moves.urllib.parse import urlparse, urlsplit import pip_shims.shims +import six +import six.moves +import tomlkit +import vistir +from six.moves.urllib.parse import urlparse, urlsplit, urlunparse from vistir.compat import Path -from vistir.path import is_valid_url, ensure_mkdir_p, create_tracked_tempdir +from vistir.path import ensure_mkdir_p, is_valid_url + +from .environment import MYPY_RUNNING + +# fmt: off +six.add_move( # type: ignore + six.MovedAttribute("Mapping", "collections", "collections.abc") # type: ignore +) # noqa # isort:skip +six.add_move( # type: ignore + six.MovedAttribute("Sequence", "collections", "collections.abc") # type: ignore +) # noqa # isort:skip +six.add_move( # type: ignore + six.MovedAttribute("Set", "collections", "collections.abc") # type: ignore +) # noqa # isort:skip +six.add_move( # type: ignore + six.MovedAttribute("ItemsView", "collections", "collections.abc") # type: ignore +) # noqa +from six.moves import ItemsView, Mapping, Sequence, Set # type: ignore # noqa # isort:skip +# fmt: on + + +if MYPY_RUNNING: + from typing import Dict, Any, Optional, Union, Tuple, List, Iterable, Text, TypeVar + + STRING_TYPE = Union[bytes, str, Text] + S = TypeVar("S", bytes, str, Text) + PipfileEntryType = Union[STRING_TYPE, bool, Tuple[STRING_TYPE], List[STRING_TYPE]] + PipfileType = Union[STRING_TYPE, Dict[STRING_TYPE, PipfileEntryType]] VCS_LIST = ("git", "svn", "hg", "bzr") @@ -68,11 +89,12 @@ VCS_SCHEMES = [ def is_installable_dir(path): + # type: (STRING_TYPE) -> bool if pip_shims.shims.is_installable_dir(path): return True - path = Path(path) - pyproject = path.joinpath("pyproject.toml") - if pyproject.exists(): + pyproject_path = os.path.join(path, "pyproject.toml") + if os.path.exists(pyproject_path): + pyproject = Path(pyproject_path) pyproject_toml = tomlkit.loads(pyproject.read_text()) build_system = pyproject_toml.get("build-system", {}).get("build-backend", "") if build_system: @@ -81,22 +103,39 @@ def is_installable_dir(path): def strip_ssh_from_git_uri(uri): + # type: (S) -> S """Return git+ssh:// formatted URI to git+git@ format""" if isinstance(uri, six.string_types): - uri = uri.replace("git+ssh://", "git+", 1) + if "git+ssh://" in uri: + parsed = urlparse(uri) + # split the path on the first separating / so we can put the first segment + # into the 'netloc' section with a : separator + path_part, _, path = parsed.path.lstrip("/").partition("/") + path = "/{0}".format(path) + parsed = parsed._replace( + netloc="{0}:{1}".format(parsed.netloc, path_part), path=path + ) + uri = urlunparse(parsed).replace("git+ssh://", "git+", 1) return uri def add_ssh_scheme_to_git_uri(uri): - """Cleans VCS uris from pipenv.patched.notpip format""" + # type: (S) -> S + """Cleans VCS uris from pip format""" if isinstance(uri, six.string_types): # Add scheme for parsing purposes, this is also what pip does if uri.startswith("git+") and "://" not in uri: uri = uri.replace("git+", "git+ssh://", 1) + parsed = urlparse(uri) + if ":" in parsed.netloc: + netloc, _, path_start = parsed.netloc.rpartition(":") + path = "/{0}{1}".format(path_start, parsed.path) + uri = urlunparse(parsed._replace(netloc=netloc, path=path)) return uri def is_vcs(pipfile_entry): + # type: (PipfileType) -> bool """Determine if dictionary entry from Pipfile is for a vcs dependency.""" if isinstance(pipfile_entry, Mapping): return any(key for key in pipfile_entry.keys() if key in VCS_LIST) @@ -111,34 +150,46 @@ def is_vcs(pipfile_entry): def is_editable(pipfile_entry): + # type: (PipfileType) -> bool if isinstance(pipfile_entry, Mapping): return pipfile_entry.get("editable", False) is True + if isinstance(pipfile_entry, six.string_types): + return pipfile_entry.startswith("-e ") return False -def multi_split(s, split): - """Splits on multiple given separators.""" - for r in split: - s = s.replace(r, "|") - return [i for i in s.split("|") if len(i) > 0] - - def is_star(val): + # type: (PipfileType) -> bool return (isinstance(val, six.string_types) and val == "*") or ( isinstance(val, Mapping) and val.get("version", "") == "*" ) +def convert_entry_to_path(path): + # type: (Dict[S, Union[S, bool, Tuple[S], List[S]]]) -> S + """Convert a pipfile entry to a string""" + + if not isinstance(path, Mapping): + raise TypeError("expecting a mapping, received {0!r}".format(path)) + + if not any(key in path for key in ["file", "path"]): + raise ValueError("missing path-like entry in supplied mapping {0!r}".format(path)) + + if "file" in path: + path = vistir.path.url_to_path(path["file"]) + + elif "path" in path: + path = path["path"] + return path + + def is_installable_file(path): + # type: (PipfileType) -> bool """Determine if a path can potentially be installed""" from packaging import specifiers - if hasattr(path, "keys") and any( - key for key in path.keys() if key in ["file", "path"] - ): - path = urlparse(path["file"]).path if "file" in path else path["path"] - if not isinstance(path, six.string_types) or path == "*": - return False + if isinstance(path, Mapping): + path = convert_entry_to_path(path) # If the string starts with a valid specifier operator, test if it is a valid # specifier set before making a path object (to avoid breaking windows) @@ -152,41 +203,94 @@ def is_installable_file(path): return False parsed = urlparse(path) - if parsed.scheme == "file": - path = parsed.path - - if not os.path.exists(os.path.abspath(path)): + is_local = ( + not parsed.scheme + or parsed.scheme == "file" + or (len(parsed.scheme) == 1 and os.name == "nt") + ) + if parsed.scheme and parsed.scheme == "file": + path = vistir.compat.fs_decode(vistir.path.url_to_path(path)) + normalized_path = vistir.path.normalize_path(path) + if is_local and not os.path.exists(normalized_path): return False - lookup_path = Path(path) - absolute_path = "{0}".format(lookup_path.absolute()) - if lookup_path.is_dir() and is_installable_dir(absolute_path): + is_archive = pip_shims.shims.is_archive_file(normalized_path) + is_local_project = os.path.isdir(normalized_path) and is_installable_dir( + normalized_path + ) + if is_local and is_local_project or is_archive: return True - elif lookup_path.is_file() and pip_shims.shims.is_archive_file(absolute_path): + if not is_local and pip_shims.shims.is_archive_file(parsed.path): return True return False +def get_dist_metadata(dist): + import pkg_resources + from email.parser import FeedParser + + if isinstance(dist, pkg_resources.DistInfoDistribution) and dist.has_metadata( + "METADATA" + ): + metadata = dist.get_metadata("METADATA") + elif dist.has_metadata("PKG-INFO"): + metadata = dist.get_metadata("PKG-INFO") + else: + metadata = "" + + feed_parser = FeedParser() + feed_parser.feed(metadata) + return feed_parser.close() + + +def get_setup_paths(base_path, subdirectory=None): + # type: (S, Optional[S]) -> Dict[S, Optional[S]] + if base_path is None: + raise TypeError("must provide a path to derive setup paths from") + setup_py = os.path.join(base_path, "setup.py") + setup_cfg = os.path.join(base_path, "setup.cfg") + pyproject_toml = os.path.join(base_path, "pyproject.toml") + if subdirectory is not None: + base_path = os.path.join(base_path, subdirectory) + subdir_setup_py = os.path.join(subdirectory, "setup.py") + subdir_setup_cfg = os.path.join(subdirectory, "setup.cfg") + subdir_pyproject_toml = os.path.join(subdirectory, "pyproject.toml") + if subdirectory and os.path.exists(subdir_setup_py): + setup_py = subdir_setup_py + if subdirectory and os.path.exists(subdir_setup_cfg): + setup_cfg = subdir_setup_cfg + if subdirectory and os.path.exists(subdir_pyproject_toml): + pyproject_toml = subdir_pyproject_toml + return { + "setup_py": setup_py if os.path.exists(setup_py) else None, + "setup_cfg": setup_cfg if os.path.exists(setup_cfg) else None, + "pyproject_toml": pyproject_toml if os.path.exists(pyproject_toml) else None, + } + + def prepare_pip_source_args(sources, pip_args=None): + # type: (List[Dict[S, Union[S, bool]]], Optional[List[S]]) -> List[S] if pip_args is None: pip_args = [] if sources: # Add the source to pip9. - pip_args.extend(["-i", sources[0]["url"]]) + pip_args.extend(["-i", sources[0]["url"]]) # type: ignore # Trust the host if it's not verified. if not sources[0].get("verify_ssl", True): - pip_args.extend(["--trusted-host", urlparse(sources[0]["url"]).hostname]) + pip_args.extend( + ["--trusted-host", urlparse(sources[0]["url"]).hostname] + ) # type: ignore # Add additional sources as extra indexes. if len(sources) > 1: for source in sources[1:]: - pip_args.extend(["--extra-index-url", source["url"]]) + pip_args.extend(["--extra-index-url", source["url"]]) # type: ignore # Trust the host if it's not verified. if not source.get("verify_ssl", True): pip_args.extend( ["--trusted-host", urlparse(source["url"]).hostname] - ) + ) # type: ignore return pip_args @@ -195,29 +299,6 @@ def _ensure_dir(path): return path -@contextlib.contextmanager -def ensure_setup_py(base_dir): - if not base_dir: - base_dir = create_tracked_tempdir(prefix="requirementslib-setup") - base_dir = Path(base_dir) - if base_dir.exists() and base_dir.name == "setup.py": - base_dir = base_dir.parent - elif not (base_dir.exists() and base_dir.is_dir()): - base_dir = base_dir.parent - if not (base_dir.exists() and base_dir.is_dir()): - base_dir = base_dir.parent - setup_py = base_dir.joinpath("setup.py") - - is_new = False if setup_py.exists() else True - if not setup_py.exists(): - setup_py.write_text(u"") - try: - yield - finally: - if is_new: - setup_py.unlink() - - _UNSET = object() _REMAP_EXIT = object() @@ -325,9 +406,7 @@ def get_path(root, path, default=_UNSET): cur = cur[seg] except (ValueError, KeyError, IndexError, TypeError): if not getattr(cur, "__iter__", None): - exc = TypeError( - "%r object is not indexable" % type(cur).__name__ - ) + exc = TypeError("%r object is not indexable" % type(cur).__name__) raise PathAccessError(exc, seg, path) except PathAccessError: if default is _UNSET: diff --git a/pipenv/vendor/scandir.py b/pipenv/vendor/scandir.py index 8bbae2c5..44f949fd 100644 --- a/pipenv/vendor/scandir.py +++ b/pipenv/vendor/scandir.py @@ -37,7 +37,7 @@ if _scandir is None and ctypes is None: warnings.warn("scandir can't find the compiled _scandir C module " "or ctypes, using slow generic fallback") -__version__ = '1.9.0' +__version__ = '1.10.0' __all__ = ['scandir', 'walk'] # Windows FILE_ATTRIBUTE constants for interpreting the @@ -583,7 +583,7 @@ elif sys.platform.startswith(('linux', 'darwin', 'sunos5')) or 'bsd' in sys.plat if _scandir is not None: scandir = scandir_c DirEntry = DirEntry_c - elif ctypes is not None: + elif ctypes is not None and have_dirent_d_type: scandir = scandir_python DirEntry = PosixDirEntry else: diff --git a/pipenv/vendor/shellingham/__init__.py b/pipenv/vendor/shellingham/__init__.py index 576c4224..b834b74b 100644 --- a/pipenv/vendor/shellingham/__init__.py +++ b/pipenv/vendor/shellingham/__init__.py @@ -4,7 +4,7 @@ import os from ._core import ShellDetectionFailure -__version__ = '1.2.7' +__version__ = '1.3.1' def detect_shell(pid=None, max_depth=6): diff --git a/pipenv/vendor/shellingham/_core.py b/pipenv/vendor/shellingham/_core.py index fb988eb3..da103a94 100644 --- a/pipenv/vendor/shellingham/_core.py +++ b/pipenv/vendor/shellingham/_core.py @@ -1,5 +1,5 @@ SHELL_NAMES = { - 'sh', 'bash', 'dash', # Bourne. + 'sh', 'bash', 'dash', 'ash', # Bourne. 'csh', 'tcsh', # C. 'ksh', 'zsh', 'fish', # Common alternatives. 'cmd', 'powershell', 'pwsh', # Microsoft. diff --git a/pipenv/vendor/shellingham/nt.py b/pipenv/vendor/shellingham/nt.py index 7b3cc6b4..d66bc33f 100644 --- a/pipenv/vendor/shellingham/nt.py +++ b/pipenv/vendor/shellingham/nt.py @@ -75,7 +75,18 @@ def _iter_process(): # looking for. We can fix this when it actually matters. (#8) continue raise WinError() - info = {'executable': str(pe.szExeFile.decode('utf-8'))} + + # The executable name would be encoded with the current code page if + # we're in ANSI mode (usually). Try to decode it into str/unicode, + # replacing invalid characters to be safe (not thoeratically necessary, + # I think). Note that we need to use 'mbcs' instead of encoding + # settings from sys because this is from the Windows API, not Python + # internals (which those settings reflect). (pypa/pipenv#3382) + executable = pe.szExeFile + if isinstance(executable, bytes): + executable = executable.decode('mbcs', 'replace') + + info = {'executable': executable} if pe.th32ParentProcessID: info['parent_pid'] = pe.th32ParentProcessID yield pe.th32ProcessID, info diff --git a/pipenv/vendor/shellingham/posix/ps.py b/pipenv/vendor/shellingham/posix/ps.py index 4a155ed5..cf7e56f2 100644 --- a/pipenv/vendor/shellingham/posix/ps.py +++ b/pipenv/vendor/shellingham/posix/ps.py @@ -1,5 +1,4 @@ import errno -import shlex import subprocess import sys @@ -34,9 +33,13 @@ def get_process_mapping(): for line in output.split('\n'): try: pid, ppid, args = line.strip().split(None, 2) - processes[pid] = Process( - args=tuple(shlex.split(args)), pid=pid, ppid=ppid, - ) + # XXX: This is not right, but we are really out of options. + # ps does not offer a sane way to decode the argument display, + # and this is "Good Enough" for obtaining shell names. Hopefully + # people don't name their shell with a space, or have something + # like "/usr/bin/xonsh is uber". (sarugaku/shellingham#14) + args = tuple(a.strip() for a in args.split(' ')) except ValueError: continue + processes[pid] = Process(args=args, pid=pid, ppid=ppid) return processes diff --git a/pipenv/vendor/shutilwhich/LICENSE b/pipenv/vendor/shutilwhich/LICENSE deleted file mode 100644 index 41ec01db..00000000 --- a/pipenv/vendor/shutilwhich/LICENSE +++ /dev/null @@ -1,292 +0,0 @@ -shutilwhich contains code from Python 3.3 and uses the same license: - -A. HISTORY OF THE SOFTWARE -========================== - -Python was created in the early 1990s by Guido van Rossum at Stichting -Mathematisch Centrum (CWI, see http://www.cwi.nl) in the Netherlands -as a successor of a language called ABC. Guido remains Python's -principal author, although it includes many contributions from others. - -In 1995, Guido continued his work on Python at the Corporation for -National Research Initiatives (CNRI, see http://www.cnri.reston.va.us) -in Reston, Virginia where he released several versions of the -software. - -In May 2000, Guido and the Python core development team moved to -BeOpen.com to form the BeOpen PythonLabs team. In October of the same -year, the PythonLabs team moved to Digital Creations (now Zope -Corporation, see http://www.zope.com). In 2001, the Python Software -Foundation (PSF, see http://www.python.org/psf/) was formed, a -non-profit organization created specifically to own Python-related -Intellectual Property. Zope Corporation is a sponsoring member of -the PSF. - -All Python releases are Open Source (see http://www.opensource.org for -the Open Source Definition). Historically, most, but not all, Python -releases have also been GPL-compatible; the table below summarizes -the various releases. - - Release Derived Year Owner GPL- - from compatible? (1) - - 0.9.0 thru 1.2 1991-1995 CWI yes - 1.3 thru 1.5.2 1.2 1995-1999 CNRI yes - 1.6 1.5.2 2000 CNRI no - 2.0 1.6 2000 BeOpen.com no - 1.6.1 1.6 2001 CNRI yes (2) - 2.1 2.0+1.6.1 2001 PSF no - 2.0.1 2.0+1.6.1 2001 PSF yes - 2.1.1 2.1+2.0.1 2001 PSF yes - 2.2 2.1.1 2001 PSF yes - 2.1.2 2.1.1 2002 PSF yes - 2.1.3 2.1.2 2002 PSF yes - 2.2.1 2.2 2002 PSF yes - 2.2.2 2.2.1 2002 PSF yes - 2.2.3 2.2.2 2003 PSF yes - 2.3 2.2.2 2002-2003 PSF yes - 2.3.1 2.3 2002-2003 PSF yes - 2.3.2 2.3.1 2002-2003 PSF yes - 2.3.3 2.3.2 2002-2003 PSF yes - 2.3.4 2.3.3 2004 PSF yes - 2.3.5 2.3.4 2005 PSF yes - 2.4 2.3 2004 PSF yes - 2.4.1 2.4 2005 PSF yes - 2.4.2 2.4.1 2005 PSF yes - 2.4.3 2.4.2 2006 PSF yes - 2.4.4 2.4.3 2006 PSF yes - 2.5 2.4 2006 PSF yes - 2.5.1 2.5 2007 PSF yes - 2.5.2 2.5.1 2008 PSF yes - 2.5.3 2.5.2 2008 PSF yes - 2.6 2.5 2008 PSF yes - 2.6.1 2.6 2008 PSF yes - 2.6.2 2.6.1 2009 PSF yes - 2.6.3 2.6.2 2009 PSF yes - 2.6.4 2.6.3 2009 PSF yes - 2.6.5 2.6.4 2010 PSF yes - 3.0 2.6 2008 PSF yes - 3.0.1 3.0 2009 PSF yes - 3.1 3.0.1 2009 PSF yes - 3.1.1 3.1 2009 PSF yes - 3.1.2 3.1.1 2010 PSF yes - 3.1.3 3.1.2 2010 PSF yes - 3.1.4 3.1.3 2011 PSF yes - 3.2 3.1 2011 PSF yes - 3.2.1 3.2 2011 PSF yes - 3.2.2 3.2.1 2011 PSF yes - 3.2.3 3.2.2 2012 PSF yes - 3.3.0 3.2 2012 PSF yes - -Footnotes: - -(1) GPL-compatible doesn't mean that we're distributing Python under - the GPL. All Python licenses, unlike the GPL, let you distribute - a modified version without making your changes open source. The - GPL-compatible licenses make it possible to combine Python with - other software that is released under the GPL; the others don't. - -(2) According to Richard Stallman, 1.6.1 is not GPL-compatible, - because its license has a choice of law clause. According to - CNRI, however, Stallman's lawyer has told CNRI's lawyer that 1.6.1 - is "not incompatible" with the GPL. - -Thanks to the many outside volunteers who have worked under Guido's -direction to make these releases possible. - - -B. TERMS AND CONDITIONS FOR ACCESSING OR OTHERWISE USING PYTHON -=============================================================== - -PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 --------------------------------------------- - -1. This LICENSE AGREEMENT is between the Python Software Foundation -("PSF"), and the Individual or Organization ("Licensee") accessing and -otherwise using this software ("Python") in source or binary form and -its associated documentation. - -2. Subject to the terms and conditions of this License Agreement, PSF hereby -grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, -analyze, test, perform and/or display publicly, prepare derivative works, -distribute, and otherwise use Python alone or in any derivative version, -provided, however, that PSF's License Agreement and PSF's notice of copyright, -i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, -2011, 2012 Python Software Foundation; All Rights Reserved" are retained in Python -alone or in any derivative version prepared by Licensee. - -3. In the event Licensee prepares a derivative work that is based on -or incorporates Python or any part thereof, and wants to make -the derivative work available to others as provided herein, then -Licensee hereby agrees to include in any such work a brief summary of -the changes made to Python. - -4. PSF is making Python available to Licensee on an "AS IS" -basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR -IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND -DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS -FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT -INFRINGE ANY THIRD PARTY RIGHTS. - -5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON -FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS -A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, -OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. - -6. This License Agreement will automatically terminate upon a material -breach of its terms and conditions. - -7. Nothing in this License Agreement shall be deemed to create any -relationship of agency, partnership, or joint venture between PSF and -Licensee. This License Agreement does not grant permission to use PSF -trademarks or trade name in a trademark sense to endorse or promote -products or services of Licensee, or any third party. - -8. By copying, installing or otherwise using Python, Licensee -agrees to be bound by the terms and conditions of this License -Agreement. - - -BEOPEN.COM LICENSE AGREEMENT FOR PYTHON 2.0 -------------------------------------------- - -BEOPEN PYTHON OPEN SOURCE LICENSE AGREEMENT VERSION 1 - -1. This LICENSE AGREEMENT is between BeOpen.com ("BeOpen"), having an -office at 160 Saratoga Avenue, Santa Clara, CA 95051, and the -Individual or Organization ("Licensee") accessing and otherwise using -this software in source or binary form and its associated -documentation ("the Software"). - -2. Subject to the terms and conditions of this BeOpen Python License -Agreement, BeOpen hereby grants Licensee a non-exclusive, -royalty-free, world-wide license to reproduce, analyze, test, perform -and/or display publicly, prepare derivative works, distribute, and -otherwise use the Software alone or in any derivative version, -provided, however, that the BeOpen Python License is retained in the -Software, alone or in any derivative version prepared by Licensee. - -3. BeOpen is making the Software available to Licensee on an "AS IS" -basis. BEOPEN MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR -IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, BEOPEN MAKES NO AND -DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS -FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE WILL NOT -INFRINGE ANY THIRD PARTY RIGHTS. - -4. BEOPEN SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE -SOFTWARE FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS -AS A RESULT OF USING, MODIFYING OR DISTRIBUTING THE SOFTWARE, OR ANY -DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. - -5. This License Agreement will automatically terminate upon a material -breach of its terms and conditions. - -6. This License Agreement shall be governed by and interpreted in all -respects by the law of the State of California, excluding conflict of -law provisions. Nothing in this License Agreement shall be deemed to -create any relationship of agency, partnership, or joint venture -between BeOpen and Licensee. This License Agreement does not grant -permission to use BeOpen trademarks or trade names in a trademark -sense to endorse or promote products or services of Licensee, or any -third party. As an exception, the "BeOpen Python" logos available at -http://www.pythonlabs.com/logos.html may be used according to the -permissions granted on that web page. - -7. By copying, installing or otherwise using the software, Licensee -agrees to be bound by the terms and conditions of this License -Agreement. - - -CNRI LICENSE AGREEMENT FOR PYTHON 1.6.1 ---------------------------------------- - -1. This LICENSE AGREEMENT is between the Corporation for National -Research Initiatives, having an office at 1895 Preston White Drive, -Reston, VA 20191 ("CNRI"), and the Individual or Organization -("Licensee") accessing and otherwise using Python 1.6.1 software in -source or binary form and its associated documentation. - -2. Subject to the terms and conditions of this License Agreement, CNRI -hereby grants Licensee a nonexclusive, royalty-free, world-wide -license to reproduce, analyze, test, perform and/or display publicly, -prepare derivative works, distribute, and otherwise use Python 1.6.1 -alone or in any derivative version, provided, however, that CNRI's -License Agreement and CNRI's notice of copyright, i.e., "Copyright (c) -1995-2001 Corporation for National Research Initiatives; All Rights -Reserved" are retained in Python 1.6.1 alone or in any derivative -version prepared by Licensee. Alternately, in lieu of CNRI's License -Agreement, Licensee may substitute the following text (omitting the -quotes): "Python 1.6.1 is made available subject to the terms and -conditions in CNRI's License Agreement. This Agreement together with -Python 1.6.1 may be located on the Internet using the following -unique, persistent identifier (known as a handle): 1895.22/1013. This -Agreement may also be obtained from a proxy server on the Internet -using the following URL: http://hdl.handle.net/1895.22/1013". - -3. In the event Licensee prepares a derivative work that is based on -or incorporates Python 1.6.1 or any part thereof, and wants to make -the derivative work available to others as provided herein, then -Licensee hereby agrees to include in any such work a brief summary of -the changes made to Python 1.6.1. - -4. CNRI is making Python 1.6.1 available to Licensee on an "AS IS" -basis. CNRI MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR -IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, CNRI MAKES NO AND -DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS -FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON 1.6.1 WILL NOT -INFRINGE ANY THIRD PARTY RIGHTS. - -5. CNRI SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON -1.6.1 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS -A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 1.6.1, -OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. - -6. This License Agreement will automatically terminate upon a material -breach of its terms and conditions. - -7. This License Agreement shall be governed by the federal -intellectual property law of the United States, including without -limitation the federal copyright law, and, to the extent such -U.S. federal law does not apply, by the law of the Commonwealth of -Virginia, excluding Virginia's conflict of law provisions. -Notwithstanding the foregoing, with regard to derivative works based -on Python 1.6.1 that incorporate non-separable material that was -previously distributed under the GNU General Public License (GPL), the -law of the Commonwealth of Virginia shall govern this License -Agreement only as to issues arising under or with respect to -Paragraphs 4, 5, and 7 of this License Agreement. Nothing in this -License Agreement shall be deemed to create any relationship of -agency, partnership, or joint venture between CNRI and Licensee. This -License Agreement does not grant permission to use CNRI trademarks or -trade name in a trademark sense to endorse or promote products or -services of Licensee, or any third party. - -8. By clicking on the "ACCEPT" button where indicated, or by copying, -installing or otherwise using Python 1.6.1, Licensee agrees to be -bound by the terms and conditions of this License Agreement. - - ACCEPT - - -CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2 --------------------------------------------------- - -Copyright (c) 1991 - 1995, Stichting Mathematisch Centrum Amsterdam, -The Netherlands. All rights reserved. - -Permission to use, copy, modify, and distribute this software and its -documentation for any purpose and without fee is hereby granted, -provided that the above copyright notice appear in all copies and that -both that copyright notice and this permission notice appear in -supporting documentation, and that the name of Stichting Mathematisch -Centrum or CWI not be used in advertising or publicity pertaining to -distribution of the software without specific, written prior -permission. - -STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO -THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND -FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE -FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/pipenv/vendor/shutilwhich/__init__.py b/pipenv/vendor/shutilwhich/__init__.py deleted file mode 100644 index 71528812..00000000 --- a/pipenv/vendor/shutilwhich/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -__version__ = '1.1.0' - -import shutil - -if not hasattr(shutil, 'which'): - from .lib import which - shutil.which = which -else: - from shutil import which diff --git a/pipenv/vendor/shutilwhich/lib.py b/pipenv/vendor/shutilwhich/lib.py deleted file mode 100644 index 6451083a..00000000 --- a/pipenv/vendor/shutilwhich/lib.py +++ /dev/null @@ -1,58 +0,0 @@ -import os -import sys - - -# Everything below this point has been copied verbatim from the Python-3.3 -# sources. -def which(cmd, mode=os.F_OK | os.X_OK, path=None): - """Given a command, mode, and a PATH string, return the path which - conforms to the given mode on the PATH, or None if there is no such - file. - - `mode` defaults to os.F_OK | os.X_OK. `path` defaults to the result - of os.environ.get("PATH"), or can be overridden with a custom search - path. - - """ - # Check that a given file can be accessed with the correct mode. - # Additionally check that `file` is not a directory, as on Windows - # directories pass the os.access check. - def _access_check(fn, mode): - return (os.path.exists(fn) and os.access(fn, mode) - and not os.path.isdir(fn)) - - # Short circuit. If we're given a full path which matches the mode - # and it exists, we're done here. - if _access_check(cmd, mode): - return cmd - - path = (path or os.environ.get("PATH", os.defpath)).split(os.pathsep) - - if sys.platform == "win32": - # The current directory takes precedence on Windows. - if not os.curdir in path: - path.insert(0, os.curdir) - - # PATHEXT is necessary to check on Windows. - pathext = os.environ.get("PATHEXT", "").split(os.pathsep) - # See if the given file matches any of the expected path extensions. - # This will allow us to short circuit when given "python.exe". - matches = [cmd for ext in pathext if cmd.lower().endswith(ext.lower())] - # If it does match, only test that one, otherwise we have to try - # others. - files = [cmd] if matches else [cmd + ext.lower() for ext in pathext] - else: - # On other platforms you don't have things like PATHEXT to tell you - # what file suffixes are executable, so just pass on cmd as-is. - files = [cmd] - - seen = set() - for dir in path: - dir = os.path.normcase(dir) - if not dir in seen: - seen.add(dir) - for thefile in files: - name = os.path.join(dir, thefile) - if _access_check(name, mode): - return name - return None diff --git a/pipenv/vendor/six.LICENSE b/pipenv/vendor/six.LICENSE index f3068bfd..365d1074 100644 --- a/pipenv/vendor/six.LICENSE +++ b/pipenv/vendor/six.LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2010-2017 Benjamin Peterson +Copyright (c) 2010-2018 Benjamin Peterson Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in diff --git a/pipenv/vendor/six.py b/pipenv/vendor/six.py index 6bf4fd38..89b2188f 100644 --- a/pipenv/vendor/six.py +++ b/pipenv/vendor/six.py @@ -1,4 +1,4 @@ -# Copyright (c) 2010-2017 Benjamin Peterson +# Copyright (c) 2010-2018 Benjamin Peterson # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -29,7 +29,7 @@ import sys import types __author__ = "Benjamin Peterson " -__version__ = "1.11.0" +__version__ = "1.12.0" # Useful for very coarse version differentiation. @@ -844,10 +844,71 @@ def add_metaclass(metaclass): orig_vars.pop(slots_var) orig_vars.pop('__dict__', None) orig_vars.pop('__weakref__', None) + if hasattr(cls, '__qualname__'): + orig_vars['__qualname__'] = cls.__qualname__ return metaclass(cls.__name__, cls.__bases__, orig_vars) return wrapper +def ensure_binary(s, encoding='utf-8', errors='strict'): + """Coerce **s** to six.binary_type. + + For Python 2: + - `unicode` -> encoded to `str` + - `str` -> `str` + + For Python 3: + - `str` -> encoded to `bytes` + - `bytes` -> `bytes` + """ + if isinstance(s, text_type): + return s.encode(encoding, errors) + elif isinstance(s, binary_type): + return s + else: + raise TypeError("not expecting type '%s'" % type(s)) + + +def ensure_str(s, encoding='utf-8', errors='strict'): + """Coerce *s* to `str`. + + For Python 2: + - `unicode` -> encoded to `str` + - `str` -> `str` + + For Python 3: + - `str` -> `str` + - `bytes` -> decoded to `str` + """ + if not isinstance(s, (text_type, binary_type)): + raise TypeError("not expecting type '%s'" % type(s)) + if PY2 and isinstance(s, text_type): + s = s.encode(encoding, errors) + elif PY3 and isinstance(s, binary_type): + s = s.decode(encoding, errors) + return s + + +def ensure_text(s, encoding='utf-8', errors='strict'): + """Coerce *s* to six.text_type. + + For Python 2: + - `unicode` -> `unicode` + - `str` -> `unicode` + + For Python 3: + - `str` -> `str` + - `bytes` -> decoded to `str` + """ + if isinstance(s, binary_type): + return s.decode(encoding, errors) + elif isinstance(s, text_type): + return s + else: + raise TypeError("not expecting type '%s'" % type(s)) + + + def python_2_unicode_compatible(klass): """ A decorator that defines __unicode__ and __str__ methods under Python 2. diff --git a/pipenv/vendor/tomlkit/__init__.py b/pipenv/vendor/tomlkit/__init__.py index 92bfa27c..9ab90e0a 100644 --- a/pipenv/vendor/tomlkit/__init__.py +++ b/pipenv/vendor/tomlkit/__init__.py @@ -22,4 +22,4 @@ from .api import value from .api import ws -__version__ = "0.5.2" +__version__ = "0.5.3" diff --git a/pipenv/vendor/tomlkit/container.py b/pipenv/vendor/tomlkit/container.py index 9b5db5cb..340491c1 100644 --- a/pipenv/vendor/tomlkit/container.py +++ b/pipenv/vendor/tomlkit/container.py @@ -1,5 +1,7 @@ from __future__ import unicode_literals +import copy + from ._compat import decode from .exceptions import KeyAlreadyPresent from .exceptions import NonExistentKey @@ -600,3 +602,16 @@ class Container(dict): self._map = state[0] self._body = state[1] self._parsed = state[2] + + def copy(self): # type: () -> Container + return copy.copy(self) + + def __copy__(self): # type: () -> Container + c = self.__class__(self._parsed) + for k, v in super(Container, self).copy().items(): + super(Container, c).__setitem__(k, v) + + c._body += self.body + c._map.update(self._map) + + return c diff --git a/pipenv/vendor/tomlkit/items.py b/pipenv/vendor/tomlkit/items.py index cccfd4a1..f199e8df 100644 --- a/pipenv/vendor/tomlkit/items.py +++ b/pipenv/vendor/tomlkit/items.py @@ -527,7 +527,10 @@ class DateTime(Item, datetime): def __sub__(self, other): result = super(DateTime, self).__sub__(other) - return self._new(result) + if isinstance(result, datetime): + result = self._new(result) + + return result def _new(self, result): raw = result.isoformat() diff --git a/pipenv/vendor/tomlkit/source.py b/pipenv/vendor/tomlkit/source.py index dcfdafd0..ddb580e4 100644 --- a/pipenv/vendor/tomlkit/source.py +++ b/pipenv/vendor/tomlkit/source.py @@ -45,10 +45,6 @@ class _State: if self._save_marker: self._source._marker = self._marker - # Restore exceptions are silently consumed, other exceptions need to - # propagate - return exception_type is None - class _StateHandler: """ diff --git a/pipenv/vendor/urllib3/LICENSE.txt b/pipenv/vendor/urllib3/LICENSE.txt index 1c3283ee..c89cf27b 100644 --- a/pipenv/vendor/urllib3/LICENSE.txt +++ b/pipenv/vendor/urllib3/LICENSE.txt @@ -1,19 +1,21 @@ -This is the MIT license: http://www.opensource.org/licenses/mit-license.php +MIT License -Copyright 2008-2016 Andrey Petrov and contributors (see CONTRIBUTORS.txt) +Copyright (c) 2008-2019 Andrey Petrov and contributors (see CONTRIBUTORS.txt) -Permission is hereby granted, free of charge, to any person obtaining a copy of this -software and associated documentation files (the "Software"), to deal in the Software -without restriction, including without limitation the rights to use, copy, modify, merge, -publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons -to whom the Software is furnished to do so, subject to the following conditions: +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: -The above copyright notice and this permission notice shall be included in all copies or -substantial portions of the Software. +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, -INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR -PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE -FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/pipenv/vendor/urllib3/__init__.py b/pipenv/vendor/urllib3/__init__.py index 75725167..eb915886 100644 --- a/pipenv/vendor/urllib3/__init__.py +++ b/pipenv/vendor/urllib3/__init__.py @@ -1,7 +1,6 @@ """ urllib3 - Thread-safe connection pooling and re-using. """ - from __future__ import absolute_import import warnings @@ -27,7 +26,7 @@ from logging import NullHandler __author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' __license__ = 'MIT' -__version__ = '1.24' +__version__ = '1.25.2' __all__ = ( 'HTTPConnectionPool', diff --git a/pipenv/vendor/urllib3/connection.py b/pipenv/vendor/urllib3/connection.py index 02b36654..f816ee80 100644 --- a/pipenv/vendor/urllib3/connection.py +++ b/pipenv/vendor/urllib3/connection.py @@ -19,10 +19,11 @@ except (ImportError, AttributeError): # Platform-specific: No SSL. pass -try: # Python 3: - # Not a no-op, we're adding this to the namespace so it can be imported. +try: + # Python 3: not a no-op, we're adding this to the namespace so it can be imported. ConnectionError = ConnectionError -except NameError: # Python 2: +except NameError: + # Python 2 class ConnectionError(Exception): pass @@ -101,7 +102,7 @@ class HTTPConnection(_HTTPConnection, object): is_verified = False def __init__(self, *args, **kw): - if six.PY3: # Python 3 + if six.PY3: kw.pop('strict', None) # Pre-set source_address. @@ -158,7 +159,7 @@ class HTTPConnection(_HTTPConnection, object): conn = connection.create_connection( (self._dns_host, self.port), self.timeout, **extra_kw) - except SocketTimeout as e: + except SocketTimeout: raise ConnectTimeoutError( self, "Connection to %s timed out. (connect timeout=%s)" % (self.host, self.timeout)) @@ -171,7 +172,8 @@ class HTTPConnection(_HTTPConnection, object): def _prepare_conn(self, conn): self.sock = conn - if self._tunnel_host: + # Google App Engine's httplib does not define _tunnel_host + if getattr(self, '_tunnel_host', None): # TODO: Fix tunnel so it doesn't depend on self.sock state. self._tunnel() # Mark this connection as not reusable @@ -226,7 +228,8 @@ class HTTPSConnection(HTTPConnection): ssl_version = None def __init__(self, host, port=None, key_file=None, cert_file=None, - strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, + key_password=None, strict=None, + timeout=socket._GLOBAL_DEFAULT_TIMEOUT, ssl_context=None, server_hostname=None, **kw): HTTPConnection.__init__(self, host, port, strict=strict, @@ -234,6 +237,7 @@ class HTTPSConnection(HTTPConnection): self.key_file = key_file self.cert_file = cert_file + self.key_password = key_password self.ssl_context = ssl_context self.server_hostname = server_hostname @@ -255,6 +259,7 @@ class HTTPSConnection(HTTPConnection): sock=conn, keyfile=self.key_file, certfile=self.cert_file, + key_password=self.key_password, ssl_context=self.ssl_context, server_hostname=self.server_hostname ) @@ -272,25 +277,24 @@ class VerifiedHTTPSConnection(HTTPSConnection): assert_fingerprint = None def set_cert(self, key_file=None, cert_file=None, - cert_reqs=None, ca_certs=None, + cert_reqs=None, key_password=None, ca_certs=None, assert_hostname=None, assert_fingerprint=None, ca_cert_dir=None): """ This method should only be called once, before the connection is used. """ - # If cert_reqs is not provided, we can try to guess. If the user gave - # us a cert database, we assume they want to use it: otherwise, if - # they gave us an SSL Context object we should use whatever is set for - # it. + # If cert_reqs is not provided we'll assume CERT_REQUIRED unless we also + # have an SSLContext object in which case we'll use its verify_mode. if cert_reqs is None: - if ca_certs or ca_cert_dir: - cert_reqs = 'CERT_REQUIRED' - elif self.ssl_context is not None: + if self.ssl_context is not None: cert_reqs = self.ssl_context.verify_mode + else: + cert_reqs = resolve_cert_reqs(None) self.key_file = key_file self.cert_file = cert_file self.cert_reqs = cert_reqs + self.key_password = key_password self.assert_hostname = assert_hostname self.assert_fingerprint = assert_fingerprint self.ca_certs = ca_certs and os.path.expanduser(ca_certs) @@ -301,7 +305,8 @@ class VerifiedHTTPSConnection(HTTPSConnection): conn = self._new_conn() hostname = self.host - if self._tunnel_host: + # Google App Engine's httplib does not define _tunnel_host + if getattr(self, '_tunnel_host', None): self.sock = conn # Calls self._set_hostport(), so self.host is # self._tunnel_host below. @@ -338,6 +343,7 @@ class VerifiedHTTPSConnection(HTTPSConnection): sock=conn, keyfile=self.key_file, certfile=self.cert_file, + key_password=self.key_password, ca_certs=self.ca_certs, ca_cert_dir=self.ca_cert_dir, server_hostname=server_hostname, diff --git a/pipenv/vendor/urllib3/connectionpool.py b/pipenv/vendor/urllib3/connectionpool.py index f7a8f193..157568a3 100644 --- a/pipenv/vendor/urllib3/connectionpool.py +++ b/pipenv/vendor/urllib3/connectionpool.py @@ -26,6 +26,7 @@ from .exceptions import ( from .packages.ssl_match_hostname import CertificateError from .packages import six from .packages.six.moves import queue +from .packages.rfc3986.normalizers import normalize_host from .connection import ( port_by_scheme, DummyConnection, @@ -65,7 +66,7 @@ class ConnectionPool(object): if not host: raise LocationValueError("No host specified.") - self.host = _ipv6_host(host, self.scheme) + self.host = _normalize_host(host, scheme=self.scheme) self._proxy_host = host.lower() self.port = port @@ -373,9 +374,11 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # Receive the response from the server try: - try: # Python 2.7, use buffering of HTTP responses + try: + # Python 2.7, use buffering of HTTP responses httplib_response = conn.getresponse(buffering=True) - except TypeError: # Python 3 + except TypeError: + # Python 3 try: httplib_response = conn.getresponse() except Exception as e: @@ -432,8 +435,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # TODO: Add optional support for socket.gethostbyname checking. scheme, host, port = get_host(url) - - host = _ipv6_host(host, self.scheme) + if host is not None: + host = _normalize_host(host, scheme=scheme) # Use explicit default port for comparison when none is given if self.port and not port: @@ -672,7 +675,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # released back to the pool once the entire response is read response.read() except (TimeoutError, HTTPException, SocketError, ProtocolError, - BaseSSLError, SSLError) as e: + BaseSSLError, SSLError): pass # Handle redirect? @@ -746,8 +749,8 @@ class HTTPSConnectionPool(HTTPConnectionPool): If ``assert_hostname`` is False, no verification is done. The ``key_file``, ``cert_file``, ``cert_reqs``, ``ca_certs``, - ``ca_cert_dir``, and ``ssl_version`` are only used if :mod:`ssl` is - available and are fed into :meth:`urllib3.util.ssl_wrap_socket` to upgrade + ``ca_cert_dir``, ``ssl_version``, ``key_password`` are only used if :mod:`ssl` + is available and are fed into :meth:`urllib3.util.ssl_wrap_socket` to upgrade the connection socket into an SSL socket. """ @@ -759,7 +762,7 @@ class HTTPSConnectionPool(HTTPConnectionPool): block=False, headers=None, retries=None, _proxy=None, _proxy_headers=None, key_file=None, cert_file=None, cert_reqs=None, - ca_certs=None, ssl_version=None, + key_password=None, ca_certs=None, ssl_version=None, assert_hostname=None, assert_fingerprint=None, ca_cert_dir=None, **conn_kw): @@ -767,12 +770,10 @@ class HTTPSConnectionPool(HTTPConnectionPool): block, headers, retries, _proxy, _proxy_headers, **conn_kw) - if ca_certs and cert_reqs is None: - cert_reqs = 'CERT_REQUIRED' - self.key_file = key_file self.cert_file = cert_file self.cert_reqs = cert_reqs + self.key_password = key_password self.ca_certs = ca_certs self.ca_cert_dir = ca_cert_dir self.ssl_version = ssl_version @@ -787,6 +788,7 @@ class HTTPSConnectionPool(HTTPConnectionPool): if isinstance(conn, VerifiedHTTPSConnection): conn.set_cert(key_file=self.key_file, + key_password=self.key_password, cert_file=self.cert_file, cert_reqs=self.cert_reqs, ca_certs=self.ca_certs, @@ -824,7 +826,9 @@ class HTTPSConnectionPool(HTTPConnectionPool): conn = self.ConnectionCls(host=actual_host, port=actual_port, timeout=self.timeout.connect_timeout, - strict=self.strict, **self.conn_kw) + strict=self.strict, cert_file=self.cert_file, + key_file=self.key_file, key_password=self.key_password, + **self.conn_kw) return self._prepare_conn(conn) @@ -875,9 +879,9 @@ def connection_from_url(url, **kw): return HTTPConnectionPool(host, port=port, **kw) -def _ipv6_host(host, scheme): +def _normalize_host(host, scheme): """ - Process IPv6 address literals + Normalize hosts for comparisons and use with sockets. """ # httplib doesn't like it when we include brackets in IPv6 addresses @@ -886,11 +890,8 @@ def _ipv6_host(host, scheme): # Instead, we need to make sure we never pass ``None`` as the port. # However, for backward compatibility reasons we can't actually # *assert* that. See http://bugs.python.org/issue28539 - # - # Also if an IPv6 address literal has a zone identifier, the - # percent sign might be URIencoded, convert it back into ASCII if host.startswith('[') and host.endswith(']'): - host = host.replace('%25', '%').strip('[]') + host = host.strip('[]') if scheme in NORMALIZABLE_SCHEMES: - host = host.lower() + host = normalize_host(host) return host diff --git a/pipenv/vendor/urllib3/contrib/_securetransport/bindings.py b/pipenv/vendor/urllib3/contrib/_securetransport/bindings.py index bcf41c02..be342153 100644 --- a/pipenv/vendor/urllib3/contrib/_securetransport/bindings.py +++ b/pipenv/vendor/urllib3/contrib/_securetransport/bindings.py @@ -516,6 +516,8 @@ class SecurityConst(object): kTLSProtocol1 = 4 kTLSProtocol11 = 7 kTLSProtocol12 = 8 + kTLSProtocol13 = 10 + kTLSProtocolMaxSupported = 999 kSSLClientSide = 1 kSSLStreamType = 0 @@ -558,30 +560,27 @@ class SecurityConst(object): errSecInvalidTrustSettings = -25262 # Cipher suites. We only pick the ones our default cipher string allows. + # Source: https://developer.apple.com/documentation/security/1550981-ssl_cipher_suite_values TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 = 0xC02C TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 = 0xC030 TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 = 0xC02B TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 = 0xC02F - TLS_DHE_DSS_WITH_AES_256_GCM_SHA384 = 0x00A3 + TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256 = 0xCCA9 + TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 = 0xCCA8 TLS_DHE_RSA_WITH_AES_256_GCM_SHA384 = 0x009F - TLS_DHE_DSS_WITH_AES_128_GCM_SHA256 = 0x00A2 TLS_DHE_RSA_WITH_AES_128_GCM_SHA256 = 0x009E TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384 = 0xC024 TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384 = 0xC028 TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA = 0xC00A TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA = 0xC014 TLS_DHE_RSA_WITH_AES_256_CBC_SHA256 = 0x006B - TLS_DHE_DSS_WITH_AES_256_CBC_SHA256 = 0x006A TLS_DHE_RSA_WITH_AES_256_CBC_SHA = 0x0039 - TLS_DHE_DSS_WITH_AES_256_CBC_SHA = 0x0038 TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256 = 0xC023 TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 = 0xC027 TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA = 0xC009 TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA = 0xC013 TLS_DHE_RSA_WITH_AES_128_CBC_SHA256 = 0x0067 - TLS_DHE_DSS_WITH_AES_128_CBC_SHA256 = 0x0040 TLS_DHE_RSA_WITH_AES_128_CBC_SHA = 0x0033 - TLS_DHE_DSS_WITH_AES_128_CBC_SHA = 0x0032 TLS_RSA_WITH_AES_256_GCM_SHA384 = 0x009D TLS_RSA_WITH_AES_128_GCM_SHA256 = 0x009C TLS_RSA_WITH_AES_256_CBC_SHA256 = 0x003D @@ -590,4 +589,5 @@ class SecurityConst(object): TLS_RSA_WITH_AES_128_CBC_SHA = 0x002F TLS_AES_128_GCM_SHA256 = 0x1301 TLS_AES_256_GCM_SHA384 = 0x1302 - TLS_CHACHA20_POLY1305_SHA256 = 0x1303 + TLS_AES_128_CCM_8_SHA256 = 0x1305 + TLS_AES_128_CCM_SHA256 = 0x1304 diff --git a/pipenv/vendor/urllib3/contrib/pyopenssl.py b/pipenv/vendor/urllib3/contrib/pyopenssl.py index 7c0e9465..821c174f 100644 --- a/pipenv/vendor/urllib3/contrib/pyopenssl.py +++ b/pipenv/vendor/urllib3/contrib/pyopenssl.py @@ -70,6 +70,7 @@ import sys from .. import util + __all__ = ['inject_into_urllib3', 'extract_from_urllib3'] # SNI always works. @@ -77,20 +78,19 @@ HAS_SNI = True # Map from urllib3 to PyOpenSSL compatible parameter-values. _openssl_versions = { - ssl.PROTOCOL_SSLv23: OpenSSL.SSL.SSLv23_METHOD, + util.PROTOCOL_TLS: OpenSSL.SSL.SSLv23_METHOD, ssl.PROTOCOL_TLSv1: OpenSSL.SSL.TLSv1_METHOD, } +if hasattr(ssl, 'PROTOCOL_SSLv3') and hasattr(OpenSSL.SSL, 'SSLv3_METHOD'): + _openssl_versions[ssl.PROTOCOL_SSLv3] = OpenSSL.SSL.SSLv3_METHOD + if hasattr(ssl, 'PROTOCOL_TLSv1_1') and hasattr(OpenSSL.SSL, 'TLSv1_1_METHOD'): _openssl_versions[ssl.PROTOCOL_TLSv1_1] = OpenSSL.SSL.TLSv1_1_METHOD if hasattr(ssl, 'PROTOCOL_TLSv1_2') and hasattr(OpenSSL.SSL, 'TLSv1_2_METHOD'): _openssl_versions[ssl.PROTOCOL_TLSv1_2] = OpenSSL.SSL.TLSv1_2_METHOD -try: - _openssl_versions.update({ssl.PROTOCOL_SSLv3: OpenSSL.SSL.SSLv3_METHOD}) -except AttributeError: - pass _stdlib_to_openssl_verify = { ssl.CERT_NONE: OpenSSL.SSL.VERIFY_NONE, @@ -117,6 +117,7 @@ def inject_into_urllib3(): _validate_dependencies_met() + util.SSLContext = PyOpenSSLContext util.ssl_.SSLContext = PyOpenSSLContext util.HAS_SNI = HAS_SNI util.ssl_.HAS_SNI = HAS_SNI @@ -127,6 +128,7 @@ def inject_into_urllib3(): def extract_from_urllib3(): 'Undo monkey-patching by :func:`inject_into_urllib3`.' + util.SSLContext = orig_util_SSLContext util.ssl_.SSLContext = orig_util_SSLContext util.HAS_SNI = orig_util_HAS_SNI util.ssl_.HAS_SNI = orig_util_HAS_SNI @@ -184,6 +186,10 @@ def _dnsname_to_stdlib(name): except idna.core.IDNAError: return None + # Don't send IPv6 addresses through the IDNA encoder. + if ':' in name: + return name + name = idna_encode(name) if name is None: return None @@ -276,7 +282,7 @@ class WrappedSocket(object): return b'' else: raise SocketError(str(e)) - except OpenSSL.SSL.ZeroReturnError as e: + except OpenSSL.SSL.ZeroReturnError: if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN: return b'' else: @@ -286,6 +292,10 @@ class WrappedSocket(object): raise timeout('The read operation timed out') else: return self.recv(*args, **kwargs) + + # TLS 1.3 post-handshake authentication + except OpenSSL.SSL.Error as e: + raise ssl.SSLError("read error: %r" % e) else: return data @@ -297,7 +307,7 @@ class WrappedSocket(object): return 0 else: raise SocketError(str(e)) - except OpenSSL.SSL.ZeroReturnError as e: + except OpenSSL.SSL.ZeroReturnError: if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN: return 0 else: @@ -308,6 +318,10 @@ class WrappedSocket(object): else: return self.recv_into(*args, **kwargs) + # TLS 1.3 post-handshake authentication + except OpenSSL.SSL.Error as e: + raise ssl.SSLError("read error: %r" % e) + def settimeout(self, timeout): return self.socket.settimeout(timeout) @@ -360,6 +374,9 @@ class WrappedSocket(object): 'subjectAltName': get_subj_alt_name(x509) } + def version(self): + return self.connection.get_protocol_version_name() + def _reuse(self): self._makefile_refs += 1 @@ -432,7 +449,9 @@ class PyOpenSSLContext(object): def load_cert_chain(self, certfile, keyfile=None, password=None): self._ctx.use_certificate_chain_file(certfile) if password is not None: - self._ctx.set_passwd_cb(lambda max_length, prompt_twice, userdata: password) + if not isinstance(password, six.binary_type): + password = password.encode('utf-8') + self._ctx.set_passwd_cb(lambda *_: password) self._ctx.use_privatekey_file(keyfile or certfile) def wrap_socket(self, sock, server_side=False, diff --git a/pipenv/vendor/urllib3/contrib/securetransport.py b/pipenv/vendor/urllib3/contrib/securetransport.py index 77cb59ed..4dc48484 100644 --- a/pipenv/vendor/urllib3/contrib/securetransport.py +++ b/pipenv/vendor/urllib3/contrib/securetransport.py @@ -23,6 +23,31 @@ To use this module, simply import and inject it:: urllib3.contrib.securetransport.inject_into_urllib3() Happy TLSing! + +This code is a bastardised version of the code found in Will Bond's oscrypto +library. An enormous debt is owed to him for blazing this trail for us. For +that reason, this code should be considered to be covered both by urllib3's +license and by oscrypto's: + + Copyright (c) 2015-2016 Will Bond + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. """ from __future__ import absolute_import @@ -86,35 +111,32 @@ SSL_WRITE_BLOCKSIZE = 16384 # individual cipher suites. We need to do this because this is how # SecureTransport wants them. CIPHER_SUITES = [ - SecurityConst.TLS_AES_256_GCM_SHA384, - SecurityConst.TLS_CHACHA20_POLY1305_SHA256, - SecurityConst.TLS_AES_128_GCM_SHA256, SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, - SecurityConst.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + SecurityConst.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, SecurityConst.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, - SecurityConst.TLS_DHE_DSS_WITH_AES_256_GCM_SHA384, + SecurityConst.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, + SecurityConst.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, SecurityConst.TLS_DHE_RSA_WITH_AES_256_GCM_SHA384, - SecurityConst.TLS_DHE_DSS_WITH_AES_128_GCM_SHA256, SecurityConst.TLS_DHE_RSA_WITH_AES_128_GCM_SHA256, SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384, - SecurityConst.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384, SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, - SecurityConst.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, - SecurityConst.TLS_DHE_RSA_WITH_AES_256_CBC_SHA256, - SecurityConst.TLS_DHE_DSS_WITH_AES_256_CBC_SHA256, - SecurityConst.TLS_DHE_RSA_WITH_AES_256_CBC_SHA, - SecurityConst.TLS_DHE_DSS_WITH_AES_256_CBC_SHA, SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256, - SecurityConst.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256, SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, + SecurityConst.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384, + SecurityConst.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, + SecurityConst.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256, SecurityConst.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, + SecurityConst.TLS_DHE_RSA_WITH_AES_256_CBC_SHA256, + SecurityConst.TLS_DHE_RSA_WITH_AES_256_CBC_SHA, SecurityConst.TLS_DHE_RSA_WITH_AES_128_CBC_SHA256, - SecurityConst.TLS_DHE_DSS_WITH_AES_128_CBC_SHA256, SecurityConst.TLS_DHE_RSA_WITH_AES_128_CBC_SHA, - SecurityConst.TLS_DHE_DSS_WITH_AES_128_CBC_SHA, + SecurityConst.TLS_AES_256_GCM_SHA384, + SecurityConst.TLS_AES_128_GCM_SHA256, SecurityConst.TLS_RSA_WITH_AES_256_GCM_SHA384, SecurityConst.TLS_RSA_WITH_AES_128_GCM_SHA256, + SecurityConst.TLS_AES_128_CCM_8_SHA256, + SecurityConst.TLS_AES_128_CCM_SHA256, SecurityConst.TLS_RSA_WITH_AES_256_CBC_SHA256, SecurityConst.TLS_RSA_WITH_AES_128_CBC_SHA256, SecurityConst.TLS_RSA_WITH_AES_256_CBC_SHA, @@ -122,9 +144,10 @@ CIPHER_SUITES = [ ] # Basically this is simple: for PROTOCOL_SSLv23 we turn it into a low of -# TLSv1 and a high of TLSv1.2. For everything else, we pin to that version. +# TLSv1 and a high of TLSv1.3. For everything else, we pin to that version. +# TLSv1 to 1.2 are supported on macOS 10.8+ and TLSv1.3 is macOS 10.13+ _protocol_to_min_max = { - ssl.PROTOCOL_SSLv23: (SecurityConst.kTLSProtocol1, SecurityConst.kTLSProtocol12), + util.PROTOCOL_TLS: (SecurityConst.kTLSProtocol1, SecurityConst.kTLSProtocolMaxSupported), } if hasattr(ssl, "PROTOCOL_SSLv2"): @@ -147,14 +170,13 @@ if hasattr(ssl, "PROTOCOL_TLSv1_2"): _protocol_to_min_max[ssl.PROTOCOL_TLSv1_2] = ( SecurityConst.kTLSProtocol12, SecurityConst.kTLSProtocol12 ) -if hasattr(ssl, "PROTOCOL_TLS"): - _protocol_to_min_max[ssl.PROTOCOL_TLS] = _protocol_to_min_max[ssl.PROTOCOL_SSLv23] def inject_into_urllib3(): """ Monkey-patch urllib3 with SecureTransport-backed SSL-support. """ + util.SSLContext = SecureTransportContext util.ssl_.SSLContext = SecureTransportContext util.HAS_SNI = HAS_SNI util.ssl_.HAS_SNI = HAS_SNI @@ -166,6 +188,7 @@ def extract_from_urllib3(): """ Undo monkey-patching by :func:`inject_into_urllib3`. """ + util.SSLContext = orig_util_SSLContext util.ssl_.SSLContext = orig_util_SSLContext util.HAS_SNI = orig_util_HAS_SNI util.ssl_.HAS_SNI = orig_util_HAS_SNI @@ -458,7 +481,14 @@ class WrappedSocket(object): # Set the minimum and maximum TLS versions. result = Security.SSLSetProtocolVersionMin(self.context, min_version) _assert_no_error(result) + + # TLS 1.3 isn't necessarily enabled by the OS + # so we have to detect when we error out and try + # setting TLS 1.3 if it's allowed. kTLSProtocolMaxSupported + # was added in macOS 10.13 along with kTLSProtocol13. result = Security.SSLSetProtocolVersionMax(self.context, max_version) + if result != 0 and max_version == SecurityConst.kTLSProtocolMaxSupported: + result = Security.SSLSetProtocolVersionMax(self.context, SecurityConst.kTLSProtocol12) _assert_no_error(result) # If there's a trust DB, we need to use it. We do that by telling @@ -667,6 +697,25 @@ class WrappedSocket(object): return der_bytes + def version(self): + protocol = Security.SSLProtocol() + result = Security.SSLGetNegotiatedProtocolVersion(self.context, ctypes.byref(protocol)) + _assert_no_error(result) + if protocol.value == SecurityConst.kTLSProtocol13: + return 'TLSv1.3' + elif protocol.value == SecurityConst.kTLSProtocol12: + return 'TLSv1.2' + elif protocol.value == SecurityConst.kTLSProtocol11: + return 'TLSv1.1' + elif protocol.value == SecurityConst.kTLSProtocol1: + return 'TLSv1' + elif protocol.value == SecurityConst.kSSLProtocol3: + return 'SSLv3' + elif protocol.value == SecurityConst.kSSLProtocol2: + return 'SSLv2' + else: + raise ssl.SSLError('Unknown TLS version: %r' % protocol) + def _reuse(self): self._makefile_refs += 1 diff --git a/pipenv/vendor/urllib3/contrib/socks.py b/pipenv/vendor/urllib3/contrib/socks.py index 811e312e..636d261f 100644 --- a/pipenv/vendor/urllib3/contrib/socks.py +++ b/pipenv/vendor/urllib3/contrib/socks.py @@ -1,25 +1,38 @@ # -*- coding: utf-8 -*- """ This module contains provisional support for SOCKS proxies from within -urllib3. This module supports SOCKS4 (specifically the SOCKS4A variant) and +urllib3. This module supports SOCKS4, SOCKS4A (an extension of SOCKS4), and SOCKS5. To enable its functionality, either install PySocks or install this module with the ``socks`` extra. The SOCKS implementation supports the full range of urllib3 features. It also supports the following SOCKS features: -- SOCKS4 -- SOCKS4a -- SOCKS5 +- SOCKS4A (``proxy_url='socks4a://...``) +- SOCKS4 (``proxy_url='socks4://...``) +- SOCKS5 with remote DNS (``proxy_url='socks5h://...``) +- SOCKS5 with local DNS (``proxy_url='socks5://...``) - Usernames and passwords for the SOCKS proxy -Known Limitations: + .. note:: + It is recommended to use ``socks5h://`` or ``socks4a://`` schemes in + your ``proxy_url`` to ensure that DNS resolution is done from the remote + server instead of client-side when connecting to a domain name. + +SOCKS4 supports IPv4 and domain names with the SOCKS4A extension. SOCKS5 +supports IPv4, IPv6, and domain names. + +When connecting to a SOCKS4 proxy the ``username`` portion of the ``proxy_url`` +will be sent as the ``userid`` section of the SOCKS request:: + + proxy_url="socks4a://@proxy-host" + +When connecting to a SOCKS5 proxy the ``username`` and ``password`` portion +of the ``proxy_url`` will be sent as the username/password to authenticate +with the proxy:: + + proxy_url="socks5h://:@proxy-host" -- Currently PySocks does not support contacting remote websites via literal - IPv6 addresses. Any such connection attempt will fail. You must use a domain - name. -- Currently PySocks does not support IPv6 connections to the SOCKS proxy. Any - such connection attempt will fail. """ from __future__ import absolute_import @@ -88,7 +101,7 @@ class SOCKSConnection(HTTPConnection): **extra_kw ) - except SocketTimeout as e: + except SocketTimeout: raise ConnectTimeoutError( self, "Connection to %s timed out. (connect timeout=%s)" % (self.host, self.timeout)) diff --git a/pipenv/vendor/urllib3/fields.py b/pipenv/vendor/urllib3/fields.py index 37fe64a3..6a9a5a7f 100644 --- a/pipenv/vendor/urllib3/fields.py +++ b/pipenv/vendor/urllib3/fields.py @@ -1,6 +1,7 @@ from __future__ import absolute_import import email.utils import mimetypes +import re from .packages import six @@ -19,57 +20,147 @@ def guess_content_type(filename, default='application/octet-stream'): return default -def format_header_param(name, value): +def format_header_param_rfc2231(name, value): """ - Helper function to format and quote a single header parameter. + Helper function to format and quote a single header parameter using the + strategy defined in RFC 2231. Particularly useful for header parameters which might contain - non-ASCII values, like file names. This follows RFC 2231, as - suggested by RFC 2388 Section 4.4. + non-ASCII values, like file names. This follows RFC 2388 Section 4.4. :param name: The name of the parameter, a string expected to be ASCII only. :param value: - The value of the parameter, provided as a unicode string. + The value of the parameter, provided as ``bytes`` or `str``. + :ret: + An RFC-2231-formatted unicode string. """ + if isinstance(value, six.binary_type): + value = value.decode("utf-8") + if not any(ch in value for ch in '"\\\r\n'): - result = '%s="%s"' % (name, value) + result = u'%s="%s"' % (name, value) try: result.encode('ascii') except (UnicodeEncodeError, UnicodeDecodeError): pass else: return result - if not six.PY3 and isinstance(value, six.text_type): # Python 2: + + if not six.PY3: # Python 2: value = value.encode('utf-8') + + # encode_rfc2231 accepts an encoded string and returns an ascii-encoded + # string in Python 2 but accepts and returns unicode strings in Python 3 value = email.utils.encode_rfc2231(value, 'utf-8') value = '%s*=%s' % (name, value) + + if not six.PY3: # Python 2: + value = value.decode('utf-8') + return value +_HTML5_REPLACEMENTS = { + u"\u0022": u"%22", + # Replace "\" with "\\". + u"\u005C": u"\u005C\u005C", + u"\u005C": u"\u005C\u005C", +} + +# All control characters from 0x00 to 0x1F *except* 0x1B. +_HTML5_REPLACEMENTS.update({ + six.unichr(cc): u"%{:02X}".format(cc) + for cc + in range(0x00, 0x1F+1) + if cc not in (0x1B,) +}) + + +def _replace_multiple(value, needles_and_replacements): + + def replacer(match): + return needles_and_replacements[match.group(0)] + + pattern = re.compile( + r"|".join([ + re.escape(needle) for needle in needles_and_replacements.keys() + ]) + ) + + result = pattern.sub(replacer, value) + + return result + + +def format_header_param_html5(name, value): + """ + Helper function to format and quote a single header parameter using the + HTML5 strategy. + + Particularly useful for header parameters which might contain + non-ASCII values, like file names. This follows the `HTML5 Working Draft + Section 4.10.22.7`_ and matches the behavior of curl and modern browsers. + + .. _HTML5 Working Draft Section 4.10.22.7: + https://w3c.github.io/html/sec-forms.html#multipart-form-data + + :param name: + The name of the parameter, a string expected to be ASCII only. + :param value: + The value of the parameter, provided as ``bytes`` or `str``. + :ret: + A unicode string, stripped of troublesome characters. + """ + if isinstance(value, six.binary_type): + value = value.decode("utf-8") + + value = _replace_multiple(value, _HTML5_REPLACEMENTS) + + return u'%s="%s"' % (name, value) + + +# For backwards-compatibility. +format_header_param = format_header_param_html5 + + class RequestField(object): """ A data container for request body parameters. :param name: - The name of this request field. + The name of this request field. Must be unicode. :param data: The data/value body. :param filename: - An optional filename of the request field. + An optional filename of the request field. Must be unicode. :param headers: An optional dict-like object of headers to initially use for the field. + :param header_formatter: + An optional callable that is used to encode and format the headers. By + default, this is :func:`format_header_param_html5`. """ - def __init__(self, name, data, filename=None, headers=None): + def __init__( + self, + name, + data, + filename=None, + headers=None, + header_formatter=format_header_param_html5): self._name = name self._filename = filename self.data = data self.headers = {} if headers: self.headers = dict(headers) + self.header_formatter = header_formatter @classmethod - def from_tuples(cls, fieldname, value): + def from_tuples( + cls, + fieldname, + value, + header_formatter=format_header_param_html5): """ A :class:`~urllib3.fields.RequestField` factory from old-style tuple parameters. @@ -97,21 +188,24 @@ class RequestField(object): content_type = None data = value - request_param = cls(fieldname, data, filename=filename) + request_param = cls( + fieldname, data, filename=filename, header_formatter=header_formatter) request_param.make_multipart(content_type=content_type) return request_param def _render_part(self, name, value): """ - Overridable helper function to format a single header parameter. + Overridable helper function to format a single header parameter. By + default, this calls ``self.header_formatter``. :param name: The name of the parameter, a string expected to be ASCII only. :param value: The value of the parameter, provided as a unicode string. """ - return format_header_param(name, value) + + return self.header_formatter(name, value) def _render_parts(self, header_parts): """ @@ -133,7 +227,7 @@ class RequestField(object): if value is not None: parts.append(self._render_part(name, value)) - return '; '.join(parts) + return u'; '.join(parts) def render_headers(self): """ @@ -144,15 +238,15 @@ class RequestField(object): sort_keys = ['Content-Disposition', 'Content-Type', 'Content-Location'] for sort_key in sort_keys: if self.headers.get(sort_key, False): - lines.append('%s: %s' % (sort_key, self.headers[sort_key])) + lines.append(u'%s: %s' % (sort_key, self.headers[sort_key])) for header_name, header_value in self.headers.items(): if header_name not in sort_keys: if header_value: - lines.append('%s: %s' % (header_name, header_value)) + lines.append(u'%s: %s' % (header_name, header_value)) - lines.append('\r\n') - return '\r\n'.join(lines) + lines.append(u'\r\n') + return u'\r\n'.join(lines) def make_multipart(self, content_disposition=None, content_type=None, content_location=None): @@ -168,10 +262,10 @@ class RequestField(object): The 'Content-Location' of the request body. """ - self.headers['Content-Disposition'] = content_disposition or 'form-data' - self.headers['Content-Disposition'] += '; '.join([ - '', self._render_parts( - (('name', self._name), ('filename', self._filename)) + self.headers['Content-Disposition'] = content_disposition or u'form-data' + self.headers['Content-Disposition'] += u'; '.join([ + u'', self._render_parts( + ((u'name', self._name), (u'filename', self._filename)) ) ]) self.headers['Content-Type'] = content_type diff --git a/pipenv/vendor/urllib3/packages/rfc3986/__init__.py b/pipenv/vendor/urllib3/packages/rfc3986/__init__.py new file mode 100644 index 00000000..9d3c3bc9 --- /dev/null +++ b/pipenv/vendor/urllib3/packages/rfc3986/__init__.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2014 Rackspace +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +An implementation of semantics and validations described in RFC 3986. + +See http://rfc3986.readthedocs.io/ for detailed documentation. + +:copyright: (c) 2014 Rackspace +:license: Apache v2.0, see LICENSE for details +""" + +from .api import iri_reference +from .api import IRIReference +from .api import is_valid_uri +from .api import normalize_uri +from .api import uri_reference +from .api import URIReference +from .api import urlparse +from .parseresult import ParseResult + +__title__ = 'rfc3986' +__author__ = 'Ian Stapleton Cordasco' +__author_email__ = 'graffatcolmingov@gmail.com' +__license__ = 'Apache v2.0' +__copyright__ = 'Copyright 2014 Rackspace' +__version__ = '1.3.1' + +__all__ = ( + 'ParseResult', + 'URIReference', + 'IRIReference', + 'is_valid_uri', + 'normalize_uri', + 'uri_reference', + 'iri_reference', + 'urlparse', + '__title__', + '__author__', + '__author_email__', + '__license__', + '__copyright__', + '__version__', +) diff --git a/pipenv/vendor/urllib3/packages/rfc3986/_mixin.py b/pipenv/vendor/urllib3/packages/rfc3986/_mixin.py new file mode 100644 index 00000000..543925cd --- /dev/null +++ b/pipenv/vendor/urllib3/packages/rfc3986/_mixin.py @@ -0,0 +1,353 @@ +"""Module containing the implementation of the URIMixin class.""" +import warnings + +from . import exceptions as exc +from . import misc +from . import normalizers +from . import validators + + +class URIMixin(object): + """Mixin with all shared methods for URIs and IRIs.""" + + __hash__ = tuple.__hash__ + + def authority_info(self): + """Return a dictionary with the ``userinfo``, ``host``, and ``port``. + + If the authority is not valid, it will raise a + :class:`~rfc3986.exceptions.InvalidAuthority` Exception. + + :returns: + ``{'userinfo': 'username:password', 'host': 'www.example.com', + 'port': '80'}`` + :rtype: dict + :raises rfc3986.exceptions.InvalidAuthority: + If the authority is not ``None`` and can not be parsed. + """ + if not self.authority: + return {'userinfo': None, 'host': None, 'port': None} + + match = self._match_subauthority() + + if match is None: + # In this case, we have an authority that was parsed from the URI + # Reference, but it cannot be further parsed by our + # misc.SUBAUTHORITY_MATCHER. In this case it must not be a valid + # authority. + raise exc.InvalidAuthority(self.authority.encode(self.encoding)) + + # We had a match, now let's ensure that it is actually a valid host + # address if it is IPv4 + matches = match.groupdict() + host = matches.get('host') + + if (host and misc.IPv4_MATCHER.match(host) and not + validators.valid_ipv4_host_address(host)): + # If we have a host, it appears to be IPv4 and it does not have + # valid bytes, it is an InvalidAuthority. + raise exc.InvalidAuthority(self.authority.encode(self.encoding)) + + return matches + + def _match_subauthority(self): + return misc.SUBAUTHORITY_MATCHER.match(self.authority) + + @property + def host(self): + """If present, a string representing the host.""" + try: + authority = self.authority_info() + except exc.InvalidAuthority: + return None + return authority['host'] + + @property + def port(self): + """If present, the port extracted from the authority.""" + try: + authority = self.authority_info() + except exc.InvalidAuthority: + return None + return authority['port'] + + @property + def userinfo(self): + """If present, the userinfo extracted from the authority.""" + try: + authority = self.authority_info() + except exc.InvalidAuthority: + return None + return authority['userinfo'] + + def is_absolute(self): + """Determine if this URI Reference is an absolute URI. + + See http://tools.ietf.org/html/rfc3986#section-4.3 for explanation. + + :returns: ``True`` if it is an absolute URI, ``False`` otherwise. + :rtype: bool + """ + return bool(misc.ABSOLUTE_URI_MATCHER.match(self.unsplit())) + + def is_valid(self, **kwargs): + """Determine if the URI is valid. + + .. deprecated:: 1.1.0 + + Use the :class:`~rfc3986.validators.Validator` object instead. + + :param bool require_scheme: Set to ``True`` if you wish to require the + presence of the scheme component. + :param bool require_authority: Set to ``True`` if you wish to require + the presence of the authority component. + :param bool require_path: Set to ``True`` if you wish to require the + presence of the path component. + :param bool require_query: Set to ``True`` if you wish to require the + presence of the query component. + :param bool require_fragment: Set to ``True`` if you wish to require + the presence of the fragment component. + :returns: ``True`` if the URI is valid. ``False`` otherwise. + :rtype: bool + """ + warnings.warn("Please use rfc3986.validators.Validator instead. " + "This method will be eventually removed.", + DeprecationWarning) + validators = [ + (self.scheme_is_valid, kwargs.get('require_scheme', False)), + (self.authority_is_valid, kwargs.get('require_authority', False)), + (self.path_is_valid, kwargs.get('require_path', False)), + (self.query_is_valid, kwargs.get('require_query', False)), + (self.fragment_is_valid, kwargs.get('require_fragment', False)), + ] + return all(v(r) for v, r in validators) + + def authority_is_valid(self, require=False): + """Determine if the authority component is valid. + + .. deprecated:: 1.1.0 + + Use the :class:`~rfc3986.validators.Validator` object instead. + + :param bool require: + Set to ``True`` to require the presence of this component. + :returns: + ``True`` if the authority is valid. ``False`` otherwise. + :rtype: + bool + """ + warnings.warn("Please use rfc3986.validators.Validator instead. " + "This method will be eventually removed.", + DeprecationWarning) + try: + self.authority_info() + except exc.InvalidAuthority: + return False + + return validators.authority_is_valid( + self.authority, + host=self.host, + require=require, + ) + + def scheme_is_valid(self, require=False): + """Determine if the scheme component is valid. + + .. deprecated:: 1.1.0 + + Use the :class:`~rfc3986.validators.Validator` object instead. + + :param str require: Set to ``True`` to require the presence of this + component. + :returns: ``True`` if the scheme is valid. ``False`` otherwise. + :rtype: bool + """ + warnings.warn("Please use rfc3986.validators.Validator instead. " + "This method will be eventually removed.", + DeprecationWarning) + return validators.scheme_is_valid(self.scheme, require) + + def path_is_valid(self, require=False): + """Determine if the path component is valid. + + .. deprecated:: 1.1.0 + + Use the :class:`~rfc3986.validators.Validator` object instead. + + :param str require: Set to ``True`` to require the presence of this + component. + :returns: ``True`` if the path is valid. ``False`` otherwise. + :rtype: bool + """ + warnings.warn("Please use rfc3986.validators.Validator instead. " + "This method will be eventually removed.", + DeprecationWarning) + return validators.path_is_valid(self.path, require) + + def query_is_valid(self, require=False): + """Determine if the query component is valid. + + .. deprecated:: 1.1.0 + + Use the :class:`~rfc3986.validators.Validator` object instead. + + :param str require: Set to ``True`` to require the presence of this + component. + :returns: ``True`` if the query is valid. ``False`` otherwise. + :rtype: bool + """ + warnings.warn("Please use rfc3986.validators.Validator instead. " + "This method will be eventually removed.", + DeprecationWarning) + return validators.query_is_valid(self.query, require) + + def fragment_is_valid(self, require=False): + """Determine if the fragment component is valid. + + .. deprecated:: 1.1.0 + + Use the Validator object instead. + + :param str require: Set to ``True`` to require the presence of this + component. + :returns: ``True`` if the fragment is valid. ``False`` otherwise. + :rtype: bool + """ + warnings.warn("Please use rfc3986.validators.Validator instead. " + "This method will be eventually removed.", + DeprecationWarning) + return validators.fragment_is_valid(self.fragment, require) + + def normalized_equality(self, other_ref): + """Compare this URIReference to another URIReference. + + :param URIReference other_ref: (required), The reference with which + we're comparing. + :returns: ``True`` if the references are equal, ``False`` otherwise. + :rtype: bool + """ + return tuple(self.normalize()) == tuple(other_ref.normalize()) + + def resolve_with(self, base_uri, strict=False): + """Use an absolute URI Reference to resolve this relative reference. + + Assuming this is a relative reference that you would like to resolve, + use the provided base URI to resolve it. + + See http://tools.ietf.org/html/rfc3986#section-5 for more information. + + :param base_uri: Either a string or URIReference. It must be an + absolute URI or it will raise an exception. + :returns: A new URIReference which is the result of resolving this + reference using ``base_uri``. + :rtype: :class:`URIReference` + :raises rfc3986.exceptions.ResolutionError: + If the ``base_uri`` is not an absolute URI. + """ + if not isinstance(base_uri, URIMixin): + base_uri = type(self).from_string(base_uri) + + if not base_uri.is_absolute(): + raise exc.ResolutionError(base_uri) + + # This is optional per + # http://tools.ietf.org/html/rfc3986#section-5.2.1 + base_uri = base_uri.normalize() + + # The reference we're resolving + resolving = self + + if not strict and resolving.scheme == base_uri.scheme: + resolving = resolving.copy_with(scheme=None) + + # http://tools.ietf.org/html/rfc3986#page-32 + if resolving.scheme is not None: + target = resolving.copy_with( + path=normalizers.normalize_path(resolving.path) + ) + else: + if resolving.authority is not None: + target = resolving.copy_with( + scheme=base_uri.scheme, + path=normalizers.normalize_path(resolving.path) + ) + else: + if resolving.path is None: + if resolving.query is not None: + query = resolving.query + else: + query = base_uri.query + target = resolving.copy_with( + scheme=base_uri.scheme, + authority=base_uri.authority, + path=base_uri.path, + query=query + ) + else: + if resolving.path.startswith('/'): + path = normalizers.normalize_path(resolving.path) + else: + path = normalizers.normalize_path( + misc.merge_paths(base_uri, resolving.path) + ) + target = resolving.copy_with( + scheme=base_uri.scheme, + authority=base_uri.authority, + path=path, + query=resolving.query + ) + return target + + def unsplit(self): + """Create a URI string from the components. + + :returns: The URI Reference reconstituted as a string. + :rtype: str + """ + # See http://tools.ietf.org/html/rfc3986#section-5.3 + result_list = [] + if self.scheme: + result_list.extend([self.scheme, ':']) + if self.authority: + result_list.extend(['//', self.authority]) + if self.path: + result_list.append(self.path) + if self.query is not None: + result_list.extend(['?', self.query]) + if self.fragment is not None: + result_list.extend(['#', self.fragment]) + return ''.join(result_list) + + def copy_with(self, scheme=misc.UseExisting, authority=misc.UseExisting, + path=misc.UseExisting, query=misc.UseExisting, + fragment=misc.UseExisting): + """Create a copy of this reference with the new components. + + :param str scheme: + (optional) The scheme to use for the new reference. + :param str authority: + (optional) The authority to use for the new reference. + :param str path: + (optional) The path to use for the new reference. + :param str query: + (optional) The query to use for the new reference. + :param str fragment: + (optional) The fragment to use for the new reference. + :returns: + New URIReference with provided components. + :rtype: + URIReference + """ + attributes = { + 'scheme': scheme, + 'authority': authority, + 'path': path, + 'query': query, + 'fragment': fragment, + } + for key, value in list(attributes.items()): + if value is misc.UseExisting: + del attributes[key] + uri = self._replace(**attributes) + uri.encoding = self.encoding + return uri diff --git a/pipenv/vendor/urllib3/packages/rfc3986/abnf_regexp.py b/pipenv/vendor/urllib3/packages/rfc3986/abnf_regexp.py new file mode 100644 index 00000000..24c9c3d0 --- /dev/null +++ b/pipenv/vendor/urllib3/packages/rfc3986/abnf_regexp.py @@ -0,0 +1,267 @@ +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Module for the regular expressions crafted from ABNF.""" + +import sys + +# https://tools.ietf.org/html/rfc3986#page-13 +GEN_DELIMS = GENERIC_DELIMITERS = ":/?#[]@" +GENERIC_DELIMITERS_SET = set(GENERIC_DELIMITERS) +# https://tools.ietf.org/html/rfc3986#page-13 +SUB_DELIMS = SUB_DELIMITERS = "!$&'()*+,;=" +SUB_DELIMITERS_SET = set(SUB_DELIMITERS) +# Escape the '*' for use in regular expressions +SUB_DELIMITERS_RE = r"!$&'()\*+,;=" +RESERVED_CHARS_SET = GENERIC_DELIMITERS_SET.union(SUB_DELIMITERS_SET) +ALPHA = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz' +DIGIT = '0123456789' +# https://tools.ietf.org/html/rfc3986#section-2.3 +UNRESERVED = UNRESERVED_CHARS = ALPHA + DIGIT + r'._!-' +UNRESERVED_CHARS_SET = set(UNRESERVED_CHARS) +NON_PCT_ENCODED_SET = RESERVED_CHARS_SET.union(UNRESERVED_CHARS_SET) +# We need to escape the '-' in this case: +UNRESERVED_RE = r'A-Za-z0-9._~\-' + +# Percent encoded character values +PERCENT_ENCODED = PCT_ENCODED = '%[A-Fa-f0-9]{2}' +PCHAR = '([' + UNRESERVED_RE + SUB_DELIMITERS_RE + ':@]|%s)' % PCT_ENCODED + +# NOTE(sigmavirus24): We're going to use more strict regular expressions +# than appear in Appendix B for scheme. This will prevent over-eager +# consuming of items that aren't schemes. +SCHEME_RE = '[a-zA-Z][a-zA-Z0-9+.-]*' +_AUTHORITY_RE = '[^/?#]*' +_PATH_RE = '[^?#]*' +_QUERY_RE = '[^#]*' +_FRAGMENT_RE = '.*' + +# Extracted from http://tools.ietf.org/html/rfc3986#appendix-B +COMPONENT_PATTERN_DICT = { + 'scheme': SCHEME_RE, + 'authority': _AUTHORITY_RE, + 'path': _PATH_RE, + 'query': _QUERY_RE, + 'fragment': _FRAGMENT_RE, +} + +# See http://tools.ietf.org/html/rfc3986#appendix-B +# In this case, we name each of the important matches so we can use +# SRE_Match#groupdict to parse the values out if we so choose. This is also +# modified to ignore other matches that are not important to the parsing of +# the reference so we can also simply use SRE_Match#groups. +URL_PARSING_RE = ( + r'(?:(?P{scheme}):)?(?://(?P{authority}))?' + r'(?P{path})(?:\?(?P{query}))?' + r'(?:#(?P{fragment}))?' +).format(**COMPONENT_PATTERN_DICT) + + +# ######################### +# Authority Matcher Section +# ######################### + +# Host patterns, see: http://tools.ietf.org/html/rfc3986#section-3.2.2 +# The pattern for a regular name, e.g., www.google.com, api.github.com +REGULAR_NAME_RE = REG_NAME = '((?:{0}|[{1}])*)'.format( + '%[0-9A-Fa-f]{2}', SUB_DELIMITERS_RE + UNRESERVED_RE +) +# The pattern for an IPv4 address, e.g., 192.168.255.255, 127.0.0.1, +IPv4_RE = r'([0-9]{1,3}\.){3}[0-9]{1,3}' +# Hexadecimal characters used in each piece of an IPv6 address +HEXDIG_RE = '[0-9A-Fa-f]{1,4}' +# Least-significant 32 bits of an IPv6 address +LS32_RE = '({hex}:{hex}|{ipv4})'.format(hex=HEXDIG_RE, ipv4=IPv4_RE) +# Substitutions into the following patterns for IPv6 patterns defined +# http://tools.ietf.org/html/rfc3986#page-20 +_subs = {'hex': HEXDIG_RE, 'ls32': LS32_RE} + +# Below: h16 = hexdig, see: https://tools.ietf.org/html/rfc5234 for details +# about ABNF (Augmented Backus-Naur Form) use in the comments +variations = [ + # 6( h16 ":" ) ls32 + '(%(hex)s:){6}%(ls32)s' % _subs, + # "::" 5( h16 ":" ) ls32 + '::(%(hex)s:){5}%(ls32)s' % _subs, + # [ h16 ] "::" 4( h16 ":" ) ls32 + '(%(hex)s)?::(%(hex)s:){4}%(ls32)s' % _subs, + # [ *1( h16 ":" ) h16 ] "::" 3( h16 ":" ) ls32 + '((%(hex)s:)?%(hex)s)?::(%(hex)s:){3}%(ls32)s' % _subs, + # [ *2( h16 ":" ) h16 ] "::" 2( h16 ":" ) ls32 + '((%(hex)s:){0,2}%(hex)s)?::(%(hex)s:){2}%(ls32)s' % _subs, + # [ *3( h16 ":" ) h16 ] "::" h16 ":" ls32 + '((%(hex)s:){0,3}%(hex)s)?::%(hex)s:%(ls32)s' % _subs, + # [ *4( h16 ":" ) h16 ] "::" ls32 + '((%(hex)s:){0,4}%(hex)s)?::%(ls32)s' % _subs, + # [ *5( h16 ":" ) h16 ] "::" h16 + '((%(hex)s:){0,5}%(hex)s)?::%(hex)s' % _subs, + # [ *6( h16 ":" ) h16 ] "::" + '((%(hex)s:){0,6}%(hex)s)?::' % _subs, +] + +IPv6_RE = '(({0})|({1})|({2})|({3})|({4})|({5})|({6})|({7})|({8}))'.format( + *variations +) + +IPv_FUTURE_RE = r'v[0-9A-Fa-f]+\.[%s]+' % ( + UNRESERVED_RE + SUB_DELIMITERS_RE + ':' +) + +# RFC 6874 Zone ID ABNF +ZONE_ID = '(?:[' + UNRESERVED_RE + ']|' + PCT_ENCODED + ')+' + +IPv6_ADDRZ_RFC4007_RE = IPv6_RE + '(?:(?:%25|%)' + ZONE_ID + ')?' +IPv6_ADDRZ_RE = IPv6_RE + '(?:%25' + ZONE_ID + ')?' + +IP_LITERAL_RE = r'\[({0}|{1})\]'.format( + IPv6_ADDRZ_RFC4007_RE, + IPv_FUTURE_RE, +) + +# Pattern for matching the host piece of the authority +HOST_RE = HOST_PATTERN = '({0}|{1}|{2})'.format( + REG_NAME, + IPv4_RE, + IP_LITERAL_RE, +) +USERINFO_RE = '^([' + UNRESERVED_RE + SUB_DELIMITERS_RE + ':]|%s)+' % ( + PCT_ENCODED +) +PORT_RE = '[0-9]{1,5}' + +# #################### +# Path Matcher Section +# #################### + +# See http://tools.ietf.org/html/rfc3986#section-3.3 for more information +# about the path patterns defined below. +segments = { + 'segment': PCHAR + '*', + # Non-zero length segment + 'segment-nz': PCHAR + '+', + # Non-zero length segment without ":" + 'segment-nz-nc': PCHAR.replace(':', '') + '+' +} + +# Path types taken from Section 3.3 (linked above) +PATH_EMPTY = '^$' +PATH_ROOTLESS = '%(segment-nz)s(/%(segment)s)*' % segments +PATH_NOSCHEME = '%(segment-nz-nc)s(/%(segment)s)*' % segments +PATH_ABSOLUTE = '/(%s)?' % PATH_ROOTLESS +PATH_ABEMPTY = '(/%(segment)s)*' % segments +PATH_RE = '^(%s|%s|%s|%s|%s)$' % ( + PATH_ABEMPTY, PATH_ABSOLUTE, PATH_NOSCHEME, PATH_ROOTLESS, PATH_EMPTY +) + +FRAGMENT_RE = QUERY_RE = ( + '^([/?:@' + UNRESERVED_RE + SUB_DELIMITERS_RE + ']|%s)*$' % PCT_ENCODED +) + +# ########################## +# Relative reference matcher +# ########################## + +# See http://tools.ietf.org/html/rfc3986#section-4.2 for details +RELATIVE_PART_RE = '(//%s%s|%s|%s|%s)' % ( + COMPONENT_PATTERN_DICT['authority'], + PATH_ABEMPTY, + PATH_ABSOLUTE, + PATH_NOSCHEME, + PATH_EMPTY, +) + +# See http://tools.ietf.org/html/rfc3986#section-3 for definition +HIER_PART_RE = '(//%s%s|%s|%s|%s)' % ( + COMPONENT_PATTERN_DICT['authority'], + PATH_ABEMPTY, + PATH_ABSOLUTE, + PATH_ROOTLESS, + PATH_EMPTY, +) + +# ############### +# IRIs / RFC 3987 +# ############### + +# Only wide-unicode gets the high-ranges of UCSCHAR +if sys.maxunicode > 0xFFFF: # pragma: no cover + IPRIVATE = u'\uE000-\uF8FF\U000F0000-\U000FFFFD\U00100000-\U0010FFFD' + UCSCHAR_RE = ( + u'\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF' + u'\U00010000-\U0001FFFD\U00020000-\U0002FFFD' + u'\U00030000-\U0003FFFD\U00040000-\U0004FFFD' + u'\U00050000-\U0005FFFD\U00060000-\U0006FFFD' + u'\U00070000-\U0007FFFD\U00080000-\U0008FFFD' + u'\U00090000-\U0009FFFD\U000A0000-\U000AFFFD' + u'\U000B0000-\U000BFFFD\U000C0000-\U000CFFFD' + u'\U000D0000-\U000DFFFD\U000E1000-\U000EFFFD' + ) +else: # pragma: no cover + IPRIVATE = u'\uE000-\uF8FF' + UCSCHAR_RE = ( + u'\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF' + ) + +IUNRESERVED_RE = u'A-Za-z0-9\\._~\\-' + UCSCHAR_RE +IPCHAR = u'([' + IUNRESERVED_RE + SUB_DELIMITERS_RE + u':@]|%s)' % PCT_ENCODED + +isegments = { + 'isegment': IPCHAR + u'*', + # Non-zero length segment + 'isegment-nz': IPCHAR + u'+', + # Non-zero length segment without ":" + 'isegment-nz-nc': IPCHAR.replace(':', '') + u'+' +} + +IPATH_ROOTLESS = u'%(isegment-nz)s(/%(isegment)s)*' % isegments +IPATH_NOSCHEME = u'%(isegment-nz-nc)s(/%(isegment)s)*' % isegments +IPATH_ABSOLUTE = u'/(?:%s)?' % IPATH_ROOTLESS +IPATH_ABEMPTY = u'(?:/%(isegment)s)*' % isegments +IPATH_RE = u'^(?:%s|%s|%s|%s|%s)$' % ( + IPATH_ABEMPTY, IPATH_ABSOLUTE, IPATH_NOSCHEME, IPATH_ROOTLESS, PATH_EMPTY +) + +IREGULAR_NAME_RE = IREG_NAME = u'(?:{0}|[{1}])*'.format( + u'%[0-9A-Fa-f]{2}', SUB_DELIMITERS_RE + IUNRESERVED_RE +) + +IHOST_RE = IHOST_PATTERN = u'({0}|{1}|{2})'.format( + IREG_NAME, + IPv4_RE, + IP_LITERAL_RE, +) + +IUSERINFO_RE = u'^(?:[' + IUNRESERVED_RE + SUB_DELIMITERS_RE + u':]|%s)+' % ( + PCT_ENCODED +) + +IFRAGMENT_RE = (u'^(?:[/?:@' + IUNRESERVED_RE + SUB_DELIMITERS_RE + + u']|%s)*$' % PCT_ENCODED) +IQUERY_RE = (u'^(?:[/?:@' + IUNRESERVED_RE + SUB_DELIMITERS_RE + + IPRIVATE + u']|%s)*$' % PCT_ENCODED) + +IRELATIVE_PART_RE = u'(//%s%s|%s|%s|%s)' % ( + COMPONENT_PATTERN_DICT['authority'], + IPATH_ABEMPTY, + IPATH_ABSOLUTE, + IPATH_NOSCHEME, + PATH_EMPTY, +) + +IHIER_PART_RE = u'(//%s%s|%s|%s|%s)' % ( + COMPONENT_PATTERN_DICT['authority'], + IPATH_ABEMPTY, + IPATH_ABSOLUTE, + IPATH_ROOTLESS, + PATH_EMPTY, +) diff --git a/pipenv/vendor/urllib3/packages/rfc3986/api.py b/pipenv/vendor/urllib3/packages/rfc3986/api.py new file mode 100644 index 00000000..ddc4a1cd --- /dev/null +++ b/pipenv/vendor/urllib3/packages/rfc3986/api.py @@ -0,0 +1,106 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2014 Rackspace +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Module containing the simple and functional API for rfc3986. + +This module defines functions and provides access to the public attributes +and classes of rfc3986. +""" + +from .iri import IRIReference +from .parseresult import ParseResult +from .uri import URIReference + + +def uri_reference(uri, encoding='utf-8'): + """Parse a URI string into a URIReference. + + This is a convenience function. You could achieve the same end by using + ``URIReference.from_string(uri)``. + + :param str uri: The URI which needs to be parsed into a reference. + :param str encoding: The encoding of the string provided + :returns: A parsed URI + :rtype: :class:`URIReference` + """ + return URIReference.from_string(uri, encoding) + + +def iri_reference(iri, encoding='utf-8'): + """Parse a IRI string into an IRIReference. + + This is a convenience function. You could achieve the same end by using + ``IRIReference.from_string(iri)``. + + :param str iri: The IRI which needs to be parsed into a reference. + :param str encoding: The encoding of the string provided + :returns: A parsed IRI + :rtype: :class:`IRIReference` + """ + return IRIReference.from_string(iri, encoding) + + +def is_valid_uri(uri, encoding='utf-8', **kwargs): + """Determine if the URI given is valid. + + This is a convenience function. You could use either + ``uri_reference(uri).is_valid()`` or + ``URIReference.from_string(uri).is_valid()`` to achieve the same result. + + :param str uri: The URI to be validated. + :param str encoding: The encoding of the string provided + :param bool require_scheme: Set to ``True`` if you wish to require the + presence of the scheme component. + :param bool require_authority: Set to ``True`` if you wish to require the + presence of the authority component. + :param bool require_path: Set to ``True`` if you wish to require the + presence of the path component. + :param bool require_query: Set to ``True`` if you wish to require the + presence of the query component. + :param bool require_fragment: Set to ``True`` if you wish to require the + presence of the fragment component. + :returns: ``True`` if the URI is valid, ``False`` otherwise. + :rtype: bool + """ + return URIReference.from_string(uri, encoding).is_valid(**kwargs) + + +def normalize_uri(uri, encoding='utf-8'): + """Normalize the given URI. + + This is a convenience function. You could use either + ``uri_reference(uri).normalize().unsplit()`` or + ``URIReference.from_string(uri).normalize().unsplit()`` instead. + + :param str uri: The URI to be normalized. + :param str encoding: The encoding of the string provided + :returns: The normalized URI. + :rtype: str + """ + normalized_reference = URIReference.from_string(uri, encoding).normalize() + return normalized_reference.unsplit() + + +def urlparse(uri, encoding='utf-8'): + """Parse a given URI and return a ParseResult. + + This is a partial replacement of the standard library's urlparse function. + + :param str uri: The URI to be parsed. + :param str encoding: The encoding of the string provided. + :returns: A parsed URI + :rtype: :class:`~rfc3986.parseresult.ParseResult` + """ + return ParseResult.from_string(uri, encoding, strict=False) diff --git a/pipenv/vendor/urllib3/packages/rfc3986/builder.py b/pipenv/vendor/urllib3/packages/rfc3986/builder.py new file mode 100644 index 00000000..79342799 --- /dev/null +++ b/pipenv/vendor/urllib3/packages/rfc3986/builder.py @@ -0,0 +1,298 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2017 Ian Stapleton Cordasco +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Module containing the logic for the URIBuilder object.""" +from . import compat +from . import normalizers +from . import uri + + +class URIBuilder(object): + """Object to aid in building up a URI Reference from parts. + + .. note:: + + This object should be instantiated by the user, but it's recommended + that it is not provided with arguments. Instead, use the available + method to populate the fields. + + """ + + def __init__(self, scheme=None, userinfo=None, host=None, port=None, + path=None, query=None, fragment=None): + """Initialize our URI builder. + + :param str scheme: + (optional) + :param str userinfo: + (optional) + :param str host: + (optional) + :param int port: + (optional) + :param str path: + (optional) + :param str query: + (optional) + :param str fragment: + (optional) + """ + self.scheme = scheme + self.userinfo = userinfo + self.host = host + self.port = port + self.path = path + self.query = query + self.fragment = fragment + + def __repr__(self): + """Provide a convenient view of our builder object.""" + formatstr = ('URIBuilder(scheme={b.scheme}, userinfo={b.userinfo}, ' + 'host={b.host}, port={b.port}, path={b.path}, ' + 'query={b.query}, fragment={b.fragment})') + return formatstr.format(b=self) + + def add_scheme(self, scheme): + """Add a scheme to our builder object. + + After normalizing, this will generate a new URIBuilder instance with + the specified scheme and all other attributes the same. + + .. code-block:: python + + >>> URIBuilder().add_scheme('HTTPS') + URIBuilder(scheme='https', userinfo=None, host=None, port=None, + path=None, query=None, fragment=None) + + """ + scheme = normalizers.normalize_scheme(scheme) + return URIBuilder( + scheme=scheme, + userinfo=self.userinfo, + host=self.host, + port=self.port, + path=self.path, + query=self.query, + fragment=self.fragment, + ) + + def add_credentials(self, username, password): + """Add credentials as the userinfo portion of the URI. + + .. code-block:: python + + >>> URIBuilder().add_credentials('root', 's3crete') + URIBuilder(scheme=None, userinfo='root:s3crete', host=None, + port=None, path=None, query=None, fragment=None) + + >>> URIBuilder().add_credentials('root', None) + URIBuilder(scheme=None, userinfo='root', host=None, + port=None, path=None, query=None, fragment=None) + """ + if username is None: + raise ValueError('Username cannot be None') + userinfo = normalizers.normalize_username(username) + + if password is not None: + userinfo = '{}:{}'.format( + userinfo, + normalizers.normalize_password(password), + ) + + return URIBuilder( + scheme=self.scheme, + userinfo=userinfo, + host=self.host, + port=self.port, + path=self.path, + query=self.query, + fragment=self.fragment, + ) + + def add_host(self, host): + """Add hostname to the URI. + + .. code-block:: python + + >>> URIBuilder().add_host('google.com') + URIBuilder(scheme=None, userinfo=None, host='google.com', + port=None, path=None, query=None, fragment=None) + + """ + return URIBuilder( + scheme=self.scheme, + userinfo=self.userinfo, + host=normalizers.normalize_host(host), + port=self.port, + path=self.path, + query=self.query, + fragment=self.fragment, + ) + + def add_port(self, port): + """Add port to the URI. + + .. code-block:: python + + >>> URIBuilder().add_port(80) + URIBuilder(scheme=None, userinfo=None, host=None, port='80', + path=None, query=None, fragment=None) + + >>> URIBuilder().add_port(443) + URIBuilder(scheme=None, userinfo=None, host=None, port='443', + path=None, query=None, fragment=None) + + """ + port_int = int(port) + if port_int < 0: + raise ValueError( + 'ports are not allowed to be negative. You provided {}'.format( + port_int, + ) + ) + if port_int > 65535: + raise ValueError( + 'ports are not allowed to be larger than 65535. ' + 'You provided {}'.format( + port_int, + ) + ) + + return URIBuilder( + scheme=self.scheme, + userinfo=self.userinfo, + host=self.host, + port='{}'.format(port_int), + path=self.path, + query=self.query, + fragment=self.fragment, + ) + + def add_path(self, path): + """Add a path to the URI. + + .. code-block:: python + + >>> URIBuilder().add_path('sigmavirus24/rfc3985') + URIBuilder(scheme=None, userinfo=None, host=None, port=None, + path='/sigmavirus24/rfc3986', query=None, fragment=None) + + >>> URIBuilder().add_path('/checkout.php') + URIBuilder(scheme=None, userinfo=None, host=None, port=None, + path='/checkout.php', query=None, fragment=None) + + """ + if not path.startswith('/'): + path = '/{}'.format(path) + + return URIBuilder( + scheme=self.scheme, + userinfo=self.userinfo, + host=self.host, + port=self.port, + path=normalizers.normalize_path(path), + query=self.query, + fragment=self.fragment, + ) + + def add_query_from(self, query_items): + """Generate and add a query a dictionary or list of tuples. + + .. code-block:: python + + >>> URIBuilder().add_query_from({'a': 'b c'}) + URIBuilder(scheme=None, userinfo=None, host=None, port=None, + path=None, query='a=b+c', fragment=None) + + >>> URIBuilder().add_query_from([('a', 'b c')]) + URIBuilder(scheme=None, userinfo=None, host=None, port=None, + path=None, query='a=b+c', fragment=None) + + """ + query = normalizers.normalize_query(compat.urlencode(query_items)) + + return URIBuilder( + scheme=self.scheme, + userinfo=self.userinfo, + host=self.host, + port=self.port, + path=self.path, + query=query, + fragment=self.fragment, + ) + + def add_query(self, query): + """Add a pre-formated query string to the URI. + + .. code-block:: python + + >>> URIBuilder().add_query('a=b&c=d') + URIBuilder(scheme=None, userinfo=None, host=None, port=None, + path=None, query='a=b&c=d', fragment=None) + + """ + return URIBuilder( + scheme=self.scheme, + userinfo=self.userinfo, + host=self.host, + port=self.port, + path=self.path, + query=normalizers.normalize_query(query), + fragment=self.fragment, + ) + + def add_fragment(self, fragment): + """Add a fragment to the URI. + + .. code-block:: python + + >>> URIBuilder().add_fragment('section-2.6.1') + URIBuilder(scheme=None, userinfo=None, host=None, port=None, + path=None, query=None, fragment='section-2.6.1') + + """ + return URIBuilder( + scheme=self.scheme, + userinfo=self.userinfo, + host=self.host, + port=self.port, + path=self.path, + query=self.query, + fragment=normalizers.normalize_fragment(fragment), + ) + + def finalize(self): + """Create a URIReference from our builder. + + .. code-block:: python + + >>> URIBuilder().add_scheme('https').add_host('github.com' + ... ).add_path('sigmavirus24/rfc3986').finalize().unsplit() + 'https://github.com/sigmavirus24/rfc3986' + + >>> URIBuilder().add_scheme('https').add_host('github.com' + ... ).add_path('sigmavirus24/rfc3986').add_credentials( + ... 'sigmavirus24', 'not-re@l').finalize().unsplit() + 'https://sigmavirus24:not-re%40l@github.com/sigmavirus24/rfc3986' + + """ + return uri.URIReference( + self.scheme, + normalizers.normalize_authority( + (self.userinfo, self.host, self.port) + ), + self.path, + self.query, + self.fragment, + ) diff --git a/pipenv/vendor/urllib3/packages/rfc3986/compat.py b/pipenv/vendor/urllib3/packages/rfc3986/compat.py new file mode 100644 index 00000000..8968c384 --- /dev/null +++ b/pipenv/vendor/urllib3/packages/rfc3986/compat.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2014 Rackspace +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Compatibility module for Python 2 and 3 support.""" +import sys + +try: + from urllib.parse import quote as urlquote +except ImportError: # Python 2.x + from urllib import quote as urlquote + +try: + from urllib.parse import urlencode +except ImportError: # Python 2.x + from urllib import urlencode + +__all__ = ( + 'to_bytes', + 'to_str', + 'urlquote', + 'urlencode', +) + +PY3 = (3, 0) <= sys.version_info < (4, 0) +PY2 = (2, 6) <= sys.version_info < (2, 8) + + +if PY3: + unicode = str # Python 3.x + + +def to_str(b, encoding='utf-8'): + """Ensure that b is text in the specified encoding.""" + if hasattr(b, 'decode') and not isinstance(b, unicode): + b = b.decode(encoding) + return b + + +def to_bytes(s, encoding='utf-8'): + """Ensure that s is converted to bytes from the encoding.""" + if hasattr(s, 'encode') and not isinstance(s, bytes): + s = s.encode(encoding) + return s diff --git a/pipenv/vendor/urllib3/packages/rfc3986/exceptions.py b/pipenv/vendor/urllib3/packages/rfc3986/exceptions.py new file mode 100644 index 00000000..da8ca7cb --- /dev/null +++ b/pipenv/vendor/urllib3/packages/rfc3986/exceptions.py @@ -0,0 +1,118 @@ +# -*- coding: utf-8 -*- +"""Exceptions module for rfc3986.""" + +from . import compat + + +class RFC3986Exception(Exception): + """Base class for all rfc3986 exception classes.""" + + pass + + +class InvalidAuthority(RFC3986Exception): + """Exception when the authority string is invalid.""" + + def __init__(self, authority): + """Initialize the exception with the invalid authority.""" + super(InvalidAuthority, self).__init__( + u"The authority ({0}) is not valid.".format( + compat.to_str(authority))) + + +class InvalidPort(RFC3986Exception): + """Exception when the port is invalid.""" + + def __init__(self, port): + """Initialize the exception with the invalid port.""" + super(InvalidPort, self).__init__( + 'The port ("{0}") is not valid.'.format(port)) + + +class ResolutionError(RFC3986Exception): + """Exception to indicate a failure to resolve a URI.""" + + def __init__(self, uri): + """Initialize the error with the failed URI.""" + super(ResolutionError, self).__init__( + "{0} is not an absolute URI.".format(uri.unsplit())) + + +class ValidationError(RFC3986Exception): + """Exception raised during Validation of a URI.""" + + pass + + +class MissingComponentError(ValidationError): + """Exception raised when a required component is missing.""" + + def __init__(self, uri, *component_names): + """Initialize the error with the missing component name.""" + verb = 'was' + if len(component_names) > 1: + verb = 'were' + + self.uri = uri + self.components = sorted(component_names) + components = ', '.join(self.components) + super(MissingComponentError, self).__init__( + "{} {} required but missing".format(components, verb), + uri, + self.components, + ) + + +class UnpermittedComponentError(ValidationError): + """Exception raised when a component has an unpermitted value.""" + + def __init__(self, component_name, component_value, allowed_values): + """Initialize the error with the unpermitted component.""" + super(UnpermittedComponentError, self).__init__( + "{} was required to be one of {!r} but was {!r}".format( + component_name, list(sorted(allowed_values)), component_value, + ), + component_name, + component_value, + allowed_values, + ) + self.component_name = component_name + self.component_value = component_value + self.allowed_values = allowed_values + + +class PasswordForbidden(ValidationError): + """Exception raised when a URL has a password in the userinfo section.""" + + def __init__(self, uri): + """Initialize the error with the URI that failed validation.""" + unsplit = getattr(uri, 'unsplit', lambda: uri) + super(PasswordForbidden, self).__init__( + '"{}" contained a password when validation forbade it'.format( + unsplit() + ) + ) + self.uri = uri + + +class InvalidComponentsError(ValidationError): + """Exception raised when one or more components are invalid.""" + + def __init__(self, uri, *component_names): + """Initialize the error with the invalid component name(s).""" + verb = 'was' + if len(component_names) > 1: + verb = 'were' + + self.uri = uri + self.components = sorted(component_names) + components = ', '.join(self.components) + super(InvalidComponentsError, self).__init__( + "{} {} found to be invalid".format(components, verb), + uri, + self.components, + ) + + +class MissingDependencyError(RFC3986Exception): + """Exception raised when an IRI is encoded without the 'idna' module.""" diff --git a/pipenv/vendor/urllib3/packages/rfc3986/iri.py b/pipenv/vendor/urllib3/packages/rfc3986/iri.py new file mode 100644 index 00000000..9c01fe1c --- /dev/null +++ b/pipenv/vendor/urllib3/packages/rfc3986/iri.py @@ -0,0 +1,147 @@ +"""Module containing the implementation of the IRIReference class.""" +# -*- coding: utf-8 -*- +# Copyright (c) 2014 Rackspace +# Copyright (c) 2015 Ian Stapleton Cordasco +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from collections import namedtuple + +from . import compat +from . import exceptions +from . import misc +from . import normalizers +from . import uri + + +try: + import idna +except ImportError: # pragma: no cover + idna = None + + +class IRIReference(namedtuple('IRIReference', misc.URI_COMPONENTS), + uri.URIMixin): + """Immutable object representing a parsed IRI Reference. + + Can be encoded into an URIReference object via the procedure + specified in RFC 3987 Section 3.1 + + .. note:: + The IRI submodule is a new interface and may possibly change in + the future. Check for changes to the interface when upgrading. + """ + + slots = () + + def __new__(cls, scheme, authority, path, query, fragment, + encoding='utf-8'): + """Create a new IRIReference.""" + ref = super(IRIReference, cls).__new__( + cls, + scheme or None, + authority or None, + path or None, + query, + fragment) + ref.encoding = encoding + return ref + + def __eq__(self, other): + """Compare this reference to another.""" + other_ref = other + if isinstance(other, tuple): + other_ref = self.__class__(*other) + elif not isinstance(other, IRIReference): + try: + other_ref = self.__class__.from_string(other) + except TypeError: + raise TypeError( + 'Unable to compare {0}() to {1}()'.format( + type(self).__name__, type(other).__name__)) + + # See http://tools.ietf.org/html/rfc3986#section-6.2 + return tuple(self) == tuple(other_ref) + + def _match_subauthority(self): + return misc.ISUBAUTHORITY_MATCHER.match(self.authority) + + @classmethod + def from_string(cls, iri_string, encoding='utf-8'): + """Parse a IRI reference from the given unicode IRI string. + + :param str iri_string: Unicode IRI to be parsed into a reference. + :param str encoding: The encoding of the string provided + :returns: :class:`IRIReference` or subclass thereof + """ + iri_string = compat.to_str(iri_string, encoding) + + split_iri = misc.IRI_MATCHER.match(iri_string).groupdict() + return cls( + split_iri['scheme'], split_iri['authority'], + normalizers.encode_component(split_iri['path'], encoding), + normalizers.encode_component(split_iri['query'], encoding), + normalizers.encode_component(split_iri['fragment'], encoding), + encoding, + ) + + def encode(self, idna_encoder=None): # noqa: C901 + """Encode an IRIReference into a URIReference instance. + + If the ``idna`` module is installed or the ``rfc3986[idna]`` + extra is used then unicode characters in the IRI host + component will be encoded with IDNA2008. + + :param idna_encoder: + Function that encodes each part of the host component + If not given will raise an exception if the IRI + contains a host component. + :rtype: uri.URIReference + :returns: A URI reference + """ + authority = self.authority + if authority: + if idna_encoder is None: + if idna is None: # pragma: no cover + raise exceptions.MissingDependencyError( + "Could not import the 'idna' module " + "and the IRI hostname requires encoding" + ) + + def idna_encoder(name): + if any(ord(c) > 128 for c in name): + try: + return idna.encode(name.lower(), + strict=True, + std3_rules=True) + except idna.IDNAError: + raise exceptions.InvalidAuthority(self.authority) + return name + + authority = "" + if self.host: + authority = ".".join([compat.to_str(idna_encoder(part)) + for part in self.host.split(".")]) + + if self.userinfo is not None: + authority = (normalizers.encode_component( + self.userinfo, self.encoding) + '@' + authority) + + if self.port is not None: + authority += ":" + str(self.port) + + return uri.URIReference(self.scheme, + authority, + path=self.path, + query=self.query, + fragment=self.fragment, + encoding=self.encoding) diff --git a/pipenv/vendor/urllib3/packages/rfc3986/misc.py b/pipenv/vendor/urllib3/packages/rfc3986/misc.py new file mode 100644 index 00000000..00f9f3b9 --- /dev/null +++ b/pipenv/vendor/urllib3/packages/rfc3986/misc.py @@ -0,0 +1,146 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2014 Rackspace +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Module containing compiled regular expressions and constants. + +This module contains important constants, patterns, and compiled regular +expressions for parsing and validating URIs and their components. +""" + +import re + +from . import abnf_regexp + +# These are enumerated for the named tuple used as a superclass of +# URIReference +URI_COMPONENTS = ['scheme', 'authority', 'path', 'query', 'fragment'] + +important_characters = { + 'generic_delimiters': abnf_regexp.GENERIC_DELIMITERS, + 'sub_delimiters': abnf_regexp.SUB_DELIMITERS, + # We need to escape the '*' in this case + 're_sub_delimiters': abnf_regexp.SUB_DELIMITERS_RE, + 'unreserved_chars': abnf_regexp.UNRESERVED_CHARS, + # We need to escape the '-' in this case: + 're_unreserved': abnf_regexp.UNRESERVED_RE, +} + +# For details about delimiters and reserved characters, see: +# http://tools.ietf.org/html/rfc3986#section-2.2 +GENERIC_DELIMITERS = abnf_regexp.GENERIC_DELIMITERS_SET +SUB_DELIMITERS = abnf_regexp.SUB_DELIMITERS_SET +RESERVED_CHARS = abnf_regexp.RESERVED_CHARS_SET +# For details about unreserved characters, see: +# http://tools.ietf.org/html/rfc3986#section-2.3 +UNRESERVED_CHARS = abnf_regexp.UNRESERVED_CHARS_SET +NON_PCT_ENCODED = abnf_regexp.NON_PCT_ENCODED_SET + +URI_MATCHER = re.compile(abnf_regexp.URL_PARSING_RE) + +SUBAUTHORITY_MATCHER = re.compile(( + '^(?:(?P{0})@)?' # userinfo + '(?P{1})' # host + ':?(?P{2})?$' # port + ).format(abnf_regexp.USERINFO_RE, + abnf_regexp.HOST_PATTERN, + abnf_regexp.PORT_RE)) + + +HOST_MATCHER = re.compile('^' + abnf_regexp.HOST_RE + '$') +IPv4_MATCHER = re.compile('^' + abnf_regexp.IPv4_RE + '$') +IPv6_MATCHER = re.compile(r'^\[' + abnf_regexp.IPv6_ADDRZ_RFC4007_RE + r'\]$') + +# Used by host validator +IPv6_NO_RFC4007_MATCHER = re.compile(r'^\[%s\]$' % ( + abnf_regexp.IPv6_ADDRZ_RE +)) + +# Matcher used to validate path components +PATH_MATCHER = re.compile(abnf_regexp.PATH_RE) + + +# ################################## +# Query and Fragment Matcher Section +# ################################## + +QUERY_MATCHER = re.compile(abnf_regexp.QUERY_RE) + +FRAGMENT_MATCHER = QUERY_MATCHER + +# Scheme validation, see: http://tools.ietf.org/html/rfc3986#section-3.1 +SCHEME_MATCHER = re.compile('^{0}$'.format(abnf_regexp.SCHEME_RE)) + +RELATIVE_REF_MATCHER = re.compile(r'^%s(\?%s)?(#%s)?$' % ( + abnf_regexp.RELATIVE_PART_RE, + abnf_regexp.QUERY_RE, + abnf_regexp.FRAGMENT_RE, +)) + +# See http://tools.ietf.org/html/rfc3986#section-4.3 +ABSOLUTE_URI_MATCHER = re.compile(r'^%s:%s(\?%s)?$' % ( + abnf_regexp.COMPONENT_PATTERN_DICT['scheme'], + abnf_regexp.HIER_PART_RE, + abnf_regexp.QUERY_RE[1:-1], +)) + +# ############### +# IRIs / RFC 3987 +# ############### + +IRI_MATCHER = re.compile(abnf_regexp.URL_PARSING_RE, re.UNICODE) + +ISUBAUTHORITY_MATCHER = re.compile(( + u'^(?:(?P{0})@)?' # iuserinfo + u'(?P{1})' # ihost + u':?(?P{2})?$' # port + ).format(abnf_regexp.IUSERINFO_RE, + abnf_regexp.IHOST_RE, + abnf_regexp.PORT_RE), re.UNICODE) + + +IHOST_MATCHER = re.compile('^' + abnf_regexp.IHOST_RE + '$', re.UNICODE) + +IPATH_MATCHER = re.compile(abnf_regexp.IPATH_RE, re.UNICODE) + +IQUERY_MATCHER = re.compile(abnf_regexp.IQUERY_RE, re.UNICODE) + +IFRAGMENT_MATCHER = re.compile(abnf_regexp.IFRAGMENT_RE, re.UNICODE) + + +RELATIVE_IRI_MATCHER = re.compile(u'^%s(?:\\?%s)?(?:%s)?$' % ( + abnf_regexp.IRELATIVE_PART_RE, + abnf_regexp.IQUERY_RE, + abnf_regexp.IFRAGMENT_RE +), re.UNICODE) + +ABSOLUTE_IRI_MATCHER = re.compile(u'^%s:%s(?:\\?%s)?$' % ( + abnf_regexp.COMPONENT_PATTERN_DICT['scheme'], + abnf_regexp.IHIER_PART_RE, + abnf_regexp.IQUERY_RE[1:-1] +), re.UNICODE) + + +# Path merger as defined in http://tools.ietf.org/html/rfc3986#section-5.2.3 +def merge_paths(base_uri, relative_path): + """Merge a base URI's path with a relative URI's path.""" + if base_uri.path is None and base_uri.authority is not None: + return '/' + relative_path + else: + path = base_uri.path or '' + index = path.rfind('/') + return path[:index] + '/' + relative_path + + +UseExisting = object() diff --git a/pipenv/vendor/urllib3/packages/rfc3986/normalizers.py b/pipenv/vendor/urllib3/packages/rfc3986/normalizers.py new file mode 100644 index 00000000..2eb1bb36 --- /dev/null +++ b/pipenv/vendor/urllib3/packages/rfc3986/normalizers.py @@ -0,0 +1,167 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2014 Rackspace +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Module with functions to normalize components.""" +import re + +from . import compat +from . import misc + + +def normalize_scheme(scheme): + """Normalize the scheme component.""" + return scheme.lower() + + +def normalize_authority(authority): + """Normalize an authority tuple to a string.""" + userinfo, host, port = authority + result = '' + if userinfo: + result += normalize_percent_characters(userinfo) + '@' + if host: + result += normalize_host(host) + if port: + result += ':' + port + return result + + +def normalize_username(username): + """Normalize a username to make it safe to include in userinfo.""" + return compat.urlquote(username) + + +def normalize_password(password): + """Normalize a password to make safe for userinfo.""" + return compat.urlquote(password) + + +def normalize_host(host): + """Normalize a host string.""" + if misc.IPv6_MATCHER.match(host): + percent = host.find('%') + if percent != -1: + percent_25 = host.find('%25') + + # Replace RFC 4007 IPv6 Zone ID delimiter '%' with '%25' + # from RFC 6874. If the host is '[%25]' then we + # assume RFC 4007 and normalize to '[%2525]' + if percent_25 == -1 or percent < percent_25 or \ + (percent == percent_25 and percent_25 == len(host) - 4): + host = host.replace('%', '%25', 1) + + # Don't normalize the casing of the Zone ID + return host[:percent].lower() + host[percent:] + + return host.lower() + + +def normalize_path(path): + """Normalize the path string.""" + if not path: + return path + + path = normalize_percent_characters(path) + return remove_dot_segments(path) + + +def normalize_query(query): + """Normalize the query string.""" + if not query: + return query + return normalize_percent_characters(query) + + +def normalize_fragment(fragment): + """Normalize the fragment string.""" + if not fragment: + return fragment + return normalize_percent_characters(fragment) + + +PERCENT_MATCHER = re.compile('%[A-Fa-f0-9]{2}') + + +def normalize_percent_characters(s): + """All percent characters should be upper-cased. + + For example, ``"%3afoo%DF%ab"`` should be turned into ``"%3Afoo%DF%AB"``. + """ + matches = set(PERCENT_MATCHER.findall(s)) + for m in matches: + if not m.isupper(): + s = s.replace(m, m.upper()) + return s + + +def remove_dot_segments(s): + """Remove dot segments from the string. + + See also Section 5.2.4 of :rfc:`3986`. + """ + # See http://tools.ietf.org/html/rfc3986#section-5.2.4 for pseudo-code + segments = s.split('/') # Turn the path into a list of segments + output = [] # Initialize the variable to use to store output + + for segment in segments: + # '.' is the current directory, so ignore it, it is superfluous + if segment == '.': + continue + # Anything other than '..', should be appended to the output + elif segment != '..': + output.append(segment) + # In this case segment == '..', if we can, we should pop the last + # element + elif output: + output.pop() + + # If the path starts with '/' and the output is empty or the first string + # is non-empty + if s.startswith('/') and (not output or output[0]): + output.insert(0, '') + + # If the path starts with '/.' or '/..' ensure we add one more empty + # string to add a trailing '/' + if s.endswith(('/.', '/..')): + output.append('') + + return '/'.join(output) + + +def encode_component(uri_component, encoding): + """Encode the specific component in the provided encoding.""" + if uri_component is None: + return uri_component + + # Try to see if the component we're encoding is already percent-encoded + # so we can skip all '%' characters but still encode all others. + percent_encodings = len(PERCENT_MATCHER.findall( + compat.to_str(uri_component, encoding))) + + uri_bytes = compat.to_bytes(uri_component, encoding) + is_percent_encoded = percent_encodings == uri_bytes.count(b'%') + + encoded_uri = bytearray() + + for i in range(0, len(uri_bytes)): + # Will return a single character bytestring on both Python 2 & 3 + byte = uri_bytes[i:i+1] + byte_ord = ord(byte) + if ((is_percent_encoded and byte == b'%') + or (byte_ord < 128 and byte.decode() in misc.NON_PCT_ENCODED)): + encoded_uri.extend(byte) + continue + encoded_uri.extend('%{0:02x}'.format(byte_ord).encode().upper()) + + return encoded_uri.decode(encoding) diff --git a/pipenv/vendor/urllib3/packages/rfc3986/parseresult.py b/pipenv/vendor/urllib3/packages/rfc3986/parseresult.py new file mode 100644 index 00000000..0a734566 --- /dev/null +++ b/pipenv/vendor/urllib3/packages/rfc3986/parseresult.py @@ -0,0 +1,385 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2015 Ian Stapleton Cordasco +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Module containing the urlparse compatibility logic.""" +from collections import namedtuple + +from . import compat +from . import exceptions +from . import misc +from . import normalizers +from . import uri + +__all__ = ('ParseResult', 'ParseResultBytes') + +PARSED_COMPONENTS = ('scheme', 'userinfo', 'host', 'port', 'path', 'query', + 'fragment') + + +class ParseResultMixin(object): + def _generate_authority(self, attributes): + # I swear I did not align the comparisons below. That's just how they + # happened to align based on pep8 and attribute lengths. + userinfo, host, port = (attributes[p] + for p in ('userinfo', 'host', 'port')) + if (self.userinfo != userinfo or + self.host != host or + self.port != port): + if port: + port = '{0}'.format(port) + return normalizers.normalize_authority( + (compat.to_str(userinfo, self.encoding), + compat.to_str(host, self.encoding), + port) + ) + return self.authority + + def geturl(self): + """Shim to match the standard library method.""" + return self.unsplit() + + @property + def hostname(self): + """Shim to match the standard library.""" + return self.host + + @property + def netloc(self): + """Shim to match the standard library.""" + return self.authority + + @property + def params(self): + """Shim to match the standard library.""" + return self.query + + +class ParseResult(namedtuple('ParseResult', PARSED_COMPONENTS), + ParseResultMixin): + """Implementation of urlparse compatibility class. + + This uses the URIReference logic to handle compatibility with the + urlparse.ParseResult class. + """ + + slots = () + + def __new__(cls, scheme, userinfo, host, port, path, query, fragment, + uri_ref, encoding='utf-8'): + """Create a new ParseResult.""" + parse_result = super(ParseResult, cls).__new__( + cls, + scheme or None, + userinfo or None, + host, + port or None, + path or None, + query, + fragment) + parse_result.encoding = encoding + parse_result.reference = uri_ref + return parse_result + + @classmethod + def from_parts(cls, scheme=None, userinfo=None, host=None, port=None, + path=None, query=None, fragment=None, encoding='utf-8'): + """Create a ParseResult instance from its parts.""" + authority = '' + if userinfo is not None: + authority += userinfo + '@' + if host is not None: + authority += host + if port is not None: + authority += ':{0}'.format(port) + uri_ref = uri.URIReference(scheme=scheme, + authority=authority, + path=path, + query=query, + fragment=fragment, + encoding=encoding).normalize() + userinfo, host, port = authority_from(uri_ref, strict=True) + return cls(scheme=uri_ref.scheme, + userinfo=userinfo, + host=host, + port=port, + path=uri_ref.path, + query=uri_ref.query, + fragment=uri_ref.fragment, + uri_ref=uri_ref, + encoding=encoding) + + @classmethod + def from_string(cls, uri_string, encoding='utf-8', strict=True, + lazy_normalize=True): + """Parse a URI from the given unicode URI string. + + :param str uri_string: Unicode URI to be parsed into a reference. + :param str encoding: The encoding of the string provided + :param bool strict: Parse strictly according to :rfc:`3986` if True. + If False, parse similarly to the standard library's urlparse + function. + :returns: :class:`ParseResult` or subclass thereof + """ + reference = uri.URIReference.from_string(uri_string, encoding) + if not lazy_normalize: + reference = reference.normalize() + userinfo, host, port = authority_from(reference, strict) + + return cls(scheme=reference.scheme, + userinfo=userinfo, + host=host, + port=port, + path=reference.path, + query=reference.query, + fragment=reference.fragment, + uri_ref=reference, + encoding=encoding) + + @property + def authority(self): + """Return the normalized authority.""" + return self.reference.authority + + def copy_with(self, scheme=misc.UseExisting, userinfo=misc.UseExisting, + host=misc.UseExisting, port=misc.UseExisting, + path=misc.UseExisting, query=misc.UseExisting, + fragment=misc.UseExisting): + """Create a copy of this instance replacing with specified parts.""" + attributes = zip(PARSED_COMPONENTS, + (scheme, userinfo, host, port, path, query, fragment)) + attrs_dict = {} + for name, value in attributes: + if value is misc.UseExisting: + value = getattr(self, name) + attrs_dict[name] = value + authority = self._generate_authority(attrs_dict) + ref = self.reference.copy_with(scheme=attrs_dict['scheme'], + authority=authority, + path=attrs_dict['path'], + query=attrs_dict['query'], + fragment=attrs_dict['fragment']) + return ParseResult(uri_ref=ref, encoding=self.encoding, **attrs_dict) + + def encode(self, encoding=None): + """Convert to an instance of ParseResultBytes.""" + encoding = encoding or self.encoding + attrs = dict( + zip(PARSED_COMPONENTS, + (attr.encode(encoding) if hasattr(attr, 'encode') else attr + for attr in self))) + return ParseResultBytes( + uri_ref=self.reference, + encoding=encoding, + **attrs + ) + + def unsplit(self, use_idna=False): + """Create a URI string from the components. + + :returns: The parsed URI reconstituted as a string. + :rtype: str + """ + parse_result = self + if use_idna and self.host: + hostbytes = self.host.encode('idna') + host = hostbytes.decode(self.encoding) + parse_result = self.copy_with(host=host) + return parse_result.reference.unsplit() + + +class ParseResultBytes(namedtuple('ParseResultBytes', PARSED_COMPONENTS), + ParseResultMixin): + """Compatibility shim for the urlparse.ParseResultBytes object.""" + + def __new__(cls, scheme, userinfo, host, port, path, query, fragment, + uri_ref, encoding='utf-8', lazy_normalize=True): + """Create a new ParseResultBytes instance.""" + parse_result = super(ParseResultBytes, cls).__new__( + cls, + scheme or None, + userinfo or None, + host, + port or None, + path or None, + query or None, + fragment or None) + parse_result.encoding = encoding + parse_result.reference = uri_ref + parse_result.lazy_normalize = lazy_normalize + return parse_result + + @classmethod + def from_parts(cls, scheme=None, userinfo=None, host=None, port=None, + path=None, query=None, fragment=None, encoding='utf-8', + lazy_normalize=True): + """Create a ParseResult instance from its parts.""" + authority = '' + if userinfo is not None: + authority += userinfo + '@' + if host is not None: + authority += host + if port is not None: + authority += ':{0}'.format(int(port)) + uri_ref = uri.URIReference(scheme=scheme, + authority=authority, + path=path, + query=query, + fragment=fragment, + encoding=encoding) + if not lazy_normalize: + uri_ref = uri_ref.normalize() + to_bytes = compat.to_bytes + userinfo, host, port = authority_from(uri_ref, strict=True) + return cls(scheme=to_bytes(scheme, encoding), + userinfo=to_bytes(userinfo, encoding), + host=to_bytes(host, encoding), + port=port, + path=to_bytes(path, encoding), + query=to_bytes(query, encoding), + fragment=to_bytes(fragment, encoding), + uri_ref=uri_ref, + encoding=encoding, + lazy_normalize=lazy_normalize) + + @classmethod + def from_string(cls, uri_string, encoding='utf-8', strict=True, + lazy_normalize=True): + """Parse a URI from the given unicode URI string. + + :param str uri_string: Unicode URI to be parsed into a reference. + :param str encoding: The encoding of the string provided + :param bool strict: Parse strictly according to :rfc:`3986` if True. + If False, parse similarly to the standard library's urlparse + function. + :returns: :class:`ParseResultBytes` or subclass thereof + """ + reference = uri.URIReference.from_string(uri_string, encoding) + if not lazy_normalize: + reference = reference.normalize() + userinfo, host, port = authority_from(reference, strict) + + to_bytes = compat.to_bytes + return cls(scheme=to_bytes(reference.scheme, encoding), + userinfo=to_bytes(userinfo, encoding), + host=to_bytes(host, encoding), + port=port, + path=to_bytes(reference.path, encoding), + query=to_bytes(reference.query, encoding), + fragment=to_bytes(reference.fragment, encoding), + uri_ref=reference, + encoding=encoding, + lazy_normalize=lazy_normalize) + + @property + def authority(self): + """Return the normalized authority.""" + return self.reference.authority.encode(self.encoding) + + def copy_with(self, scheme=misc.UseExisting, userinfo=misc.UseExisting, + host=misc.UseExisting, port=misc.UseExisting, + path=misc.UseExisting, query=misc.UseExisting, + fragment=misc.UseExisting, lazy_normalize=True): + """Create a copy of this instance replacing with specified parts.""" + attributes = zip(PARSED_COMPONENTS, + (scheme, userinfo, host, port, path, query, fragment)) + attrs_dict = {} + for name, value in attributes: + if value is misc.UseExisting: + value = getattr(self, name) + if not isinstance(value, bytes) and hasattr(value, 'encode'): + value = value.encode(self.encoding) + attrs_dict[name] = value + authority = self._generate_authority(attrs_dict) + to_str = compat.to_str + ref = self.reference.copy_with( + scheme=to_str(attrs_dict['scheme'], self.encoding), + authority=to_str(authority, self.encoding), + path=to_str(attrs_dict['path'], self.encoding), + query=to_str(attrs_dict['query'], self.encoding), + fragment=to_str(attrs_dict['fragment'], self.encoding) + ) + if not lazy_normalize: + ref = ref.normalize() + return ParseResultBytes( + uri_ref=ref, + encoding=self.encoding, + lazy_normalize=lazy_normalize, + **attrs_dict + ) + + def unsplit(self, use_idna=False): + """Create a URI bytes object from the components. + + :returns: The parsed URI reconstituted as a string. + :rtype: bytes + """ + parse_result = self + if use_idna and self.host: + # self.host is bytes, to encode to idna, we need to decode it + # first + host = self.host.decode(self.encoding) + hostbytes = host.encode('idna') + parse_result = self.copy_with(host=hostbytes) + if self.lazy_normalize: + parse_result = parse_result.copy_with(lazy_normalize=False) + uri = parse_result.reference.unsplit() + return uri.encode(self.encoding) + + +def split_authority(authority): + # Initialize our expected return values + userinfo = host = port = None + # Initialize an extra var we may need to use + extra_host = None + # Set-up rest in case there is no userinfo portion + rest = authority + + if '@' in authority: + userinfo, rest = authority.rsplit('@', 1) + + # Handle IPv6 host addresses + if rest.startswith('['): + host, rest = rest.split(']', 1) + host += ']' + + if ':' in rest: + extra_host, port = rest.split(':', 1) + elif not host and rest: + host = rest + + if extra_host and not host: + host = extra_host + + return userinfo, host, port + + +def authority_from(reference, strict): + try: + subauthority = reference.authority_info() + except exceptions.InvalidAuthority: + if strict: + raise + userinfo, host, port = split_authority(reference.authority) + else: + # Thanks to Richard Barrell for this idea: + # https://twitter.com/0x2ba22e11/status/617338811975139328 + userinfo, host, port = (subauthority.get(p) + for p in ('userinfo', 'host', 'port')) + + if port: + try: + port = int(port) + except ValueError: + raise exceptions.InvalidPort(port) + return userinfo, host, port diff --git a/pipenv/vendor/urllib3/packages/rfc3986/uri.py b/pipenv/vendor/urllib3/packages/rfc3986/uri.py new file mode 100644 index 00000000..d1d71505 --- /dev/null +++ b/pipenv/vendor/urllib3/packages/rfc3986/uri.py @@ -0,0 +1,153 @@ +"""Module containing the implementation of the URIReference class.""" +# -*- coding: utf-8 -*- +# Copyright (c) 2014 Rackspace +# Copyright (c) 2015 Ian Stapleton Cordasco +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from collections import namedtuple + +from . import compat +from . import misc +from . import normalizers +from ._mixin import URIMixin + + +class URIReference(namedtuple('URIReference', misc.URI_COMPONENTS), URIMixin): + """Immutable object representing a parsed URI Reference. + + .. note:: + + This class is not intended to be directly instantiated by the user. + + This object exposes attributes for the following components of a + URI: + + - scheme + - authority + - path + - query + - fragment + + .. attribute:: scheme + + The scheme that was parsed for the URI Reference. For example, + ``http``, ``https``, ``smtp``, ``imap``, etc. + + .. attribute:: authority + + Component of the URI that contains the user information, host, + and port sub-components. For example, + ``google.com``, ``127.0.0.1:5000``, ``username@[::1]``, + ``username:password@example.com:443``, etc. + + .. attribute:: path + + The path that was parsed for the given URI Reference. For example, + ``/``, ``/index.php``, etc. + + .. attribute:: query + + The query component for a given URI Reference. For example, ``a=b``, + ``a=b%20c``, ``a=b+c``, ``a=b,c=d,e=%20f``, etc. + + .. attribute:: fragment + + The fragment component of a URI. For example, ``section-3.1``. + + This class also provides extra attributes for easier access to information + like the subcomponents of the authority component. + + .. attribute:: userinfo + + The user information parsed from the authority. + + .. attribute:: host + + The hostname, IPv4, or IPv6 adddres parsed from the authority. + + .. attribute:: port + + The port parsed from the authority. + """ + + slots = () + + def __new__(cls, scheme, authority, path, query, fragment, + encoding='utf-8'): + """Create a new URIReference.""" + ref = super(URIReference, cls).__new__( + cls, + scheme or None, + authority or None, + path or None, + query, + fragment) + ref.encoding = encoding + return ref + + __hash__ = tuple.__hash__ + + def __eq__(self, other): + """Compare this reference to another.""" + other_ref = other + if isinstance(other, tuple): + other_ref = URIReference(*other) + elif not isinstance(other, URIReference): + try: + other_ref = URIReference.from_string(other) + except TypeError: + raise TypeError( + 'Unable to compare URIReference() to {0}()'.format( + type(other).__name__)) + + # See http://tools.ietf.org/html/rfc3986#section-6.2 + naive_equality = tuple(self) == tuple(other_ref) + return naive_equality or self.normalized_equality(other_ref) + + def normalize(self): + """Normalize this reference as described in Section 6.2.2. + + This is not an in-place normalization. Instead this creates a new + URIReference. + + :returns: A new reference object with normalized components. + :rtype: URIReference + """ + # See http://tools.ietf.org/html/rfc3986#section-6.2.2 for logic in + # this method. + return URIReference(normalizers.normalize_scheme(self.scheme or ''), + normalizers.normalize_authority( + (self.userinfo, self.host, self.port)), + normalizers.normalize_path(self.path or ''), + normalizers.normalize_query(self.query), + normalizers.normalize_fragment(self.fragment), + self.encoding) + + @classmethod + def from_string(cls, uri_string, encoding='utf-8'): + """Parse a URI reference from the given unicode URI string. + + :param str uri_string: Unicode URI to be parsed into a reference. + :param str encoding: The encoding of the string provided + :returns: :class:`URIReference` or subclass thereof + """ + uri_string = compat.to_str(uri_string, encoding) + + split_uri = misc.URI_MATCHER.match(uri_string).groupdict() + return cls( + split_uri['scheme'], split_uri['authority'], + normalizers.encode_component(split_uri['path'], encoding), + normalizers.encode_component(split_uri['query'], encoding), + normalizers.encode_component(split_uri['fragment'], encoding), + encoding, + ) diff --git a/pipenv/vendor/urllib3/packages/rfc3986/validators.py b/pipenv/vendor/urllib3/packages/rfc3986/validators.py new file mode 100644 index 00000000..7fc97215 --- /dev/null +++ b/pipenv/vendor/urllib3/packages/rfc3986/validators.py @@ -0,0 +1,450 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2017 Ian Stapleton Cordasco +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Module containing the validation logic for rfc3986.""" +from . import exceptions +from . import misc +from . import normalizers + + +class Validator(object): + """Object used to configure validation of all objects in rfc3986. + + .. versionadded:: 1.0 + + Example usage:: + + >>> from rfc3986 import api, validators + >>> uri = api.uri_reference('https://github.com/') + >>> validator = validators.Validator().require_presence_of( + ... 'scheme', 'host', 'path', + ... ).allow_schemes( + ... 'http', 'https', + ... ).allow_hosts( + ... '127.0.0.1', 'github.com', + ... ) + >>> validator.validate(uri) + >>> invalid_uri = rfc3986.uri_reference('imap://mail.google.com') + >>> validator.validate(invalid_uri) + Traceback (most recent call last): + ... + rfc3986.exceptions.MissingComponentError: ('path was required but + missing', URIReference(scheme=u'imap', authority=u'mail.google.com', + path=None, query=None, fragment=None), ['path']) + + """ + + COMPONENT_NAMES = frozenset([ + 'scheme', + 'userinfo', + 'host', + 'port', + 'path', + 'query', + 'fragment', + ]) + + def __init__(self): + """Initialize our default validations.""" + self.allowed_schemes = set() + self.allowed_hosts = set() + self.allowed_ports = set() + self.allow_password = True + self.required_components = { + 'scheme': False, + 'userinfo': False, + 'host': False, + 'port': False, + 'path': False, + 'query': False, + 'fragment': False, + } + self.validated_components = self.required_components.copy() + + def allow_schemes(self, *schemes): + """Require the scheme to be one of the provided schemes. + + .. versionadded:: 1.0 + + :param schemes: + Schemes, without ``://`` that are allowed. + :returns: + The validator instance. + :rtype: + Validator + """ + for scheme in schemes: + self.allowed_schemes.add(normalizers.normalize_scheme(scheme)) + return self + + def allow_hosts(self, *hosts): + """Require the host to be one of the provided hosts. + + .. versionadded:: 1.0 + + :param hosts: + Hosts that are allowed. + :returns: + The validator instance. + :rtype: + Validator + """ + for host in hosts: + self.allowed_hosts.add(normalizers.normalize_host(host)) + return self + + def allow_ports(self, *ports): + """Require the port to be one of the provided ports. + + .. versionadded:: 1.0 + + :param ports: + Ports that are allowed. + :returns: + The validator instance. + :rtype: + Validator + """ + for port in ports: + port_int = int(port, base=10) + if 0 <= port_int <= 65535: + self.allowed_ports.add(port) + return self + + def allow_use_of_password(self): + """Allow passwords to be present in the URI. + + .. versionadded:: 1.0 + + :returns: + The validator instance. + :rtype: + Validator + """ + self.allow_password = True + return self + + def forbid_use_of_password(self): + """Prevent passwords from being included in the URI. + + .. versionadded:: 1.0 + + :returns: + The validator instance. + :rtype: + Validator + """ + self.allow_password = False + return self + + def check_validity_of(self, *components): + """Check the validity of the components provided. + + This can be specified repeatedly. + + .. versionadded:: 1.1 + + :param components: + Names of components from :attr:`Validator.COMPONENT_NAMES`. + :returns: + The validator instance. + :rtype: + Validator + """ + components = [c.lower() for c in components] + for component in components: + if component not in self.COMPONENT_NAMES: + raise ValueError( + '"{}" is not a valid component'.format(component) + ) + self.validated_components.update({ + component: True for component in components + }) + return self + + def require_presence_of(self, *components): + """Require the components provided. + + This can be specified repeatedly. + + .. versionadded:: 1.0 + + :param components: + Names of components from :attr:`Validator.COMPONENT_NAMES`. + :returns: + The validator instance. + :rtype: + Validator + """ + components = [c.lower() for c in components] + for component in components: + if component not in self.COMPONENT_NAMES: + raise ValueError( + '"{}" is not a valid component'.format(component) + ) + self.required_components.update({ + component: True for component in components + }) + return self + + def validate(self, uri): + """Check a URI for conditions specified on this validator. + + .. versionadded:: 1.0 + + :param uri: + Parsed URI to validate. + :type uri: + rfc3986.uri.URIReference + :raises MissingComponentError: + When a required component is missing. + :raises UnpermittedComponentError: + When a component is not one of those allowed. + :raises PasswordForbidden: + When a password is present in the userinfo component but is + not permitted by configuration. + :raises InvalidComponentsError: + When a component was found to be invalid. + """ + if not self.allow_password: + check_password(uri) + + required_components = [ + component + for component, required in self.required_components.items() + if required + ] + validated_components = [ + component + for component, required in self.validated_components.items() + if required + ] + if required_components: + ensure_required_components_exist(uri, required_components) + if validated_components: + ensure_components_are_valid(uri, validated_components) + + ensure_one_of(self.allowed_schemes, uri, 'scheme') + ensure_one_of(self.allowed_hosts, uri, 'host') + ensure_one_of(self.allowed_ports, uri, 'port') + + +def check_password(uri): + """Assert that there is no password present in the uri.""" + userinfo = uri.userinfo + if not userinfo: + return + credentials = userinfo.split(':', 1) + if len(credentials) <= 1: + return + raise exceptions.PasswordForbidden(uri) + + +def ensure_one_of(allowed_values, uri, attribute): + """Assert that the uri's attribute is one of the allowed values.""" + value = getattr(uri, attribute) + if value is not None and allowed_values and value not in allowed_values: + raise exceptions.UnpermittedComponentError( + attribute, value, allowed_values, + ) + + +def ensure_required_components_exist(uri, required_components): + """Assert that all required components are present in the URI.""" + missing_components = sorted([ + component + for component in required_components + if getattr(uri, component) is None + ]) + if missing_components: + raise exceptions.MissingComponentError(uri, *missing_components) + + +def is_valid(value, matcher, require): + """Determine if a value is valid based on the provided matcher. + + :param str value: + Value to validate. + :param matcher: + Compiled regular expression to use to validate the value. + :param require: + Whether or not the value is required. + """ + if require: + return (value is not None + and matcher.match(value)) + + # require is False and value is not None + return value is None or matcher.match(value) + + +def authority_is_valid(authority, host=None, require=False): + """Determine if the authority string is valid. + + :param str authority: + The authority to validate. + :param str host: + (optional) The host portion of the authority to validate. + :param bool require: + (optional) Specify if authority must not be None. + :returns: + ``True`` if valid, ``False`` otherwise + :rtype: + bool + """ + validated = is_valid(authority, misc.SUBAUTHORITY_MATCHER, require) + if validated and host is not None: + return host_is_valid(host, require) + return validated + + +def host_is_valid(host, require=False): + """Determine if the host string is valid. + + :param str host: + The host to validate. + :param bool require: + (optional) Specify if host must not be None. + :returns: + ``True`` if valid, ``False`` otherwise + :rtype: + bool + """ + validated = is_valid(host, misc.HOST_MATCHER, require) + if validated and host is not None and misc.IPv4_MATCHER.match(host): + return valid_ipv4_host_address(host) + elif validated and host is not None and misc.IPv6_MATCHER.match(host): + return misc.IPv6_NO_RFC4007_MATCHER.match(host) is not None + return validated + + +def scheme_is_valid(scheme, require=False): + """Determine if the scheme is valid. + + :param str scheme: + The scheme string to validate. + :param bool require: + (optional) Set to ``True`` to require the presence of a scheme. + :returns: + ``True`` if the scheme is valid. ``False`` otherwise. + :rtype: + bool + """ + return is_valid(scheme, misc.SCHEME_MATCHER, require) + + +def path_is_valid(path, require=False): + """Determine if the path component is valid. + + :param str path: + The path string to validate. + :param bool require: + (optional) Set to ``True`` to require the presence of a path. + :returns: + ``True`` if the path is valid. ``False`` otherwise. + :rtype: + bool + """ + return is_valid(path, misc.PATH_MATCHER, require) + + +def query_is_valid(query, require=False): + """Determine if the query component is valid. + + :param str query: + The query string to validate. + :param bool require: + (optional) Set to ``True`` to require the presence of a query. + :returns: + ``True`` if the query is valid. ``False`` otherwise. + :rtype: + bool + """ + return is_valid(query, misc.QUERY_MATCHER, require) + + +def fragment_is_valid(fragment, require=False): + """Determine if the fragment component is valid. + + :param str fragment: + The fragment string to validate. + :param bool require: + (optional) Set to ``True`` to require the presence of a fragment. + :returns: + ``True`` if the fragment is valid. ``False`` otherwise. + :rtype: + bool + """ + return is_valid(fragment, misc.FRAGMENT_MATCHER, require) + + +def valid_ipv4_host_address(host): + """Determine if the given host is a valid IPv4 address.""" + # If the host exists, and it might be IPv4, check each byte in the + # address. + return all([0 <= int(byte, base=10) <= 255 for byte in host.split('.')]) + + +_COMPONENT_VALIDATORS = { + 'scheme': scheme_is_valid, + 'path': path_is_valid, + 'query': query_is_valid, + 'fragment': fragment_is_valid, +} + +_SUBAUTHORITY_VALIDATORS = set(['userinfo', 'host', 'port']) + + +def subauthority_component_is_valid(uri, component): + """Determine if the userinfo, host, and port are valid.""" + try: + subauthority_dict = uri.authority_info() + except exceptions.InvalidAuthority: + return False + + # If we can parse the authority into sub-components and we're not + # validating the port, we can assume it's valid. + if component == 'host': + return host_is_valid(subauthority_dict['host']) + elif component != 'port': + return True + + try: + port = int(subauthority_dict['port']) + except TypeError: + # If the port wasn't provided it'll be None and int(None) raises a + # TypeError + return True + + return (0 <= port <= 65535) + + +def ensure_components_are_valid(uri, validated_components): + """Assert that all components are valid in the URI.""" + invalid_components = set([]) + for component in validated_components: + if component in _SUBAUTHORITY_VALIDATORS: + if not subauthority_component_is_valid(uri, component): + invalid_components.add(component) + # Python's peephole optimizer means that while this continue *is* + # actually executed, coverage.py cannot detect that. See also, + # https://bitbucket.org/ned/coveragepy/issues/198/continue-marked-as-not-covered + continue # nocov: Python 2.7, 3.3, 3.4 + + validator = _COMPONENT_VALIDATORS[component] + if not validator(getattr(uri, component)): + invalid_components.add(component) + + if invalid_components: + raise exceptions.InvalidComponentsError(uri, *invalid_components) diff --git a/pipenv/vendor/urllib3/poolmanager.py b/pipenv/vendor/urllib3/poolmanager.py index fe5491cf..a6ade6e9 100644 --- a/pipenv/vendor/urllib3/poolmanager.py +++ b/pipenv/vendor/urllib3/poolmanager.py @@ -7,6 +7,7 @@ from ._collections import RecentlyUsedContainer from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool from .connectionpool import port_by_scheme from .exceptions import LocationValueError, MaxRetryError, ProxySchemeUnknown +from .packages import six from .packages.six.moves.urllib.parse import urljoin from .request import RequestMethods from .util.url import parse_url @@ -19,7 +20,8 @@ __all__ = ['PoolManager', 'ProxyManager', 'proxy_from_url'] log = logging.getLogger(__name__) SSL_KEYWORDS = ('key_file', 'cert_file', 'cert_reqs', 'ca_certs', - 'ssl_version', 'ca_cert_dir', 'ssl_context') + 'ssl_version', 'ca_cert_dir', 'ssl_context', + 'key_password') # All known keyword arguments that could be provided to the pool manager, its # pools, or the underlying connections. This is used to construct a pool key. @@ -33,6 +35,7 @@ _key_fields = ( 'key_block', # bool 'key_source_address', # str 'key_key_file', # str + 'key_key_password', # str 'key_cert_file', # str 'key_cert_reqs', # str 'key_ca_certs', # str @@ -47,7 +50,7 @@ _key_fields = ( 'key__socks_options', # dict 'key_assert_hostname', # bool or string 'key_assert_fingerprint', # str - 'key_server_hostname', #str + 'key_server_hostname', # str ) #: The namedtuple class used to construct keys for the connection pool. @@ -342,8 +345,10 @@ class PoolManager(RequestMethods): # conn.is_same_host() which may use socket.gethostbyname() in the future. if (retries.remove_headers_on_redirect and not conn.is_same_host(redirect_location)): - for header in retries.remove_headers_on_redirect: - kw['headers'].pop(header, None) + headers = list(six.iterkeys(kw['headers'])) + for header in headers: + if header.lower() in retries.remove_headers_on_redirect: + kw['headers'].pop(header, None) try: retries = retries.increment(method, url, response=response, _pool=conn) diff --git a/pipenv/vendor/urllib3/response.py b/pipenv/vendor/urllib3/response.py index f0cfbb54..4f857932 100644 --- a/pipenv/vendor/urllib3/response.py +++ b/pipenv/vendor/urllib3/response.py @@ -6,6 +6,11 @@ import logging from socket import timeout as SocketTimeout from socket import error as SocketError +try: + import brotli +except ImportError: + brotli = None + from ._collections import HTTPHeaderDict from .exceptions import ( BodyNotHttplibCompatible, ProtocolError, DecodeError, ReadTimeoutError, @@ -69,9 +74,9 @@ class GzipDecoder(object): return getattr(self._obj, name) def decompress(self, data): - ret = b'' + ret = bytearray() if self._state == GzipDecoderState.SWALLOW_DATA or not data: - return ret + return bytes(ret) while True: try: ret += self._obj.decompress(data) @@ -81,15 +86,34 @@ class GzipDecoder(object): self._state = GzipDecoderState.SWALLOW_DATA if previous_state == GzipDecoderState.OTHER_MEMBERS: # Allow trailing garbage acceptable in other gzip clients - return ret + return bytes(ret) raise data = self._obj.unused_data if not data: - return ret + return bytes(ret) self._state = GzipDecoderState.OTHER_MEMBERS self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS) +if brotli is not None: + class BrotliDecoder(object): + # Supports both 'brotlipy' and 'Brotli' packages + # since they share an import name. The top branches + # are for 'brotlipy' and bottom branches for 'Brotli' + def __init__(self): + self._obj = brotli.Decompressor() + + def decompress(self, data): + if hasattr(self._obj, 'decompress'): + return self._obj.decompress(data) + return self._obj.process(data) + + def flush(self): + if hasattr(self._obj, 'flush'): + return self._obj.flush() + return b'' + + class MultiDecoder(object): """ From RFC7231: @@ -118,6 +142,9 @@ def _get_decoder(mode): if mode == 'gzip': return GzipDecoder() + if brotli is not None and mode == 'br': + return BrotliDecoder() + return DeflateDecoder() @@ -155,6 +182,8 @@ class HTTPResponse(io.IOBase): """ CONTENT_DECODERS = ['gzip', 'deflate'] + if brotli is not None: + CONTENT_DECODERS += ['br'] REDIRECT_STATUSES = [301, 302, 303, 307, 308] def __init__(self, body='', headers=None, status=0, version=0, reason=None, @@ -311,24 +340,32 @@ class HTTPResponse(io.IOBase): if content_encoding in self.CONTENT_DECODERS: self._decoder = _get_decoder(content_encoding) elif ',' in content_encoding: - encodings = [e.strip() for e in content_encoding.split(',') if e.strip() in self.CONTENT_DECODERS] + encodings = [ + e.strip() for e in content_encoding.split(',') + if e.strip() in self.CONTENT_DECODERS] if len(encodings): self._decoder = _get_decoder(content_encoding) + DECODER_ERROR_CLASSES = (IOError, zlib.error) + if brotli is not None: + DECODER_ERROR_CLASSES += (brotli.error,) + def _decode(self, data, decode_content, flush_decoder): """ Decode the data passed in and potentially flush the decoder. """ + if not decode_content: + return data + try: - if decode_content and self._decoder: + if self._decoder: data = self._decoder.decompress(data) - except (IOError, zlib.error) as e: + except self.DECODER_ERROR_CLASSES as e: content_encoding = self.headers.get('content-encoding', '').lower() raise DecodeError( "Received response with content-encoding: %s, but " "failed to decode it." % content_encoding, e) - - if flush_decoder and decode_content: + if flush_decoder: data += self._flush_decoder() return data @@ -508,9 +545,10 @@ class HTTPResponse(io.IOBase): headers = r.msg if not isinstance(headers, HTTPHeaderDict): - if PY3: # Python 3 + if PY3: headers = HTTPHeaderDict(headers.items()) - else: # Python 2 + else: + # Python 2.7 headers = HTTPHeaderDict.from_httplib(headers) # HTTPResponse objects in Python 3 don't have a .strict attribute @@ -703,3 +741,20 @@ class HTTPResponse(io.IOBase): return self.retries.history[-1].redirect_location else: return self._request_url + + def __iter__(self): + buffer = [b""] + for chunk in self.stream(decode_content=True): + if b"\n" in chunk: + chunk = chunk.split(b"\n") + yield b"".join(buffer) + chunk[0] + b"\n" + for x in chunk[1:-1]: + yield x + b"\n" + if chunk[-1]: + buffer = [chunk[-1]] + else: + buffer = [] + else: + buffer.append(chunk) + if buffer: + yield b"".join(buffer) diff --git a/pipenv/vendor/urllib3/util/__init__.py b/pipenv/vendor/urllib3/util/__init__.py index 2f2770b6..2914bb46 100644 --- a/pipenv/vendor/urllib3/util/__init__.py +++ b/pipenv/vendor/urllib3/util/__init__.py @@ -12,6 +12,7 @@ from .ssl_ import ( resolve_cert_reqs, resolve_ssl_version, ssl_wrap_socket, + PROTOCOL_TLS, ) from .timeout import ( current_time, @@ -35,6 +36,7 @@ __all__ = ( 'IS_PYOPENSSL', 'IS_SECURETRANSPORT', 'SSLContext', + 'PROTOCOL_TLS', 'Retry', 'Timeout', 'Url', diff --git a/pipenv/vendor/urllib3/util/request.py b/pipenv/vendor/urllib3/util/request.py index 3ddfcd55..280b8530 100644 --- a/pipenv/vendor/urllib3/util/request.py +++ b/pipenv/vendor/urllib3/util/request.py @@ -5,6 +5,13 @@ from ..packages.six import b, integer_types from ..exceptions import UnrewindableBodyError ACCEPT_ENCODING = 'gzip,deflate' +try: + import brotli as _unused_module_brotli # noqa: F401 +except ImportError: + pass +else: + ACCEPT_ENCODING += ',br' + _FAILEDTELL = object() diff --git a/pipenv/vendor/urllib3/util/retry.py b/pipenv/vendor/urllib3/util/retry.py index e7d0abd6..02429ee8 100644 --- a/pipenv/vendor/urllib3/util/retry.py +++ b/pipenv/vendor/urllib3/util/retry.py @@ -179,7 +179,8 @@ class Retry(object): self.raise_on_status = raise_on_status self.history = history or tuple() self.respect_retry_after_header = respect_retry_after_header - self.remove_headers_on_redirect = remove_headers_on_redirect + self.remove_headers_on_redirect = frozenset([ + h.lower() for h in remove_headers_on_redirect]) def new(self, **kw): params = dict( diff --git a/pipenv/vendor/urllib3/util/ssl_.py b/pipenv/vendor/urllib3/util/ssl_.py index 24ee26d6..f271ce93 100644 --- a/pipenv/vendor/urllib3/util/ssl_.py +++ b/pipenv/vendor/urllib3/util/ssl_.py @@ -2,13 +2,14 @@ from __future__ import absolute_import import errno import warnings import hmac -import socket +import re from binascii import hexlify, unhexlify from hashlib import md5, sha1, sha256 from ..exceptions import SSLError, InsecurePlatformWarning, SNIMissingWarning from ..packages import six +from ..packages.rfc3986 import abnf_regexp SSLContext = None @@ -40,14 +41,33 @@ def _const_compare_digest_backport(a, b): _const_compare_digest = getattr(hmac, 'compare_digest', _const_compare_digest_backport) +# Borrow rfc3986's regular expressions for IPv4 +# and IPv6 addresses for use in is_ipaddress() +_IP_ADDRESS_REGEX = re.compile( + r'^(?:%s|%s|%s)$' % ( + abnf_regexp.IPv4_RE, + abnf_regexp.IPv6_RE, + abnf_regexp.IPv6_ADDRZ_RFC4007_RE + ) +) try: # Test for SSL features import ssl - from ssl import wrap_socket, CERT_NONE, PROTOCOL_SSLv23 + from ssl import wrap_socket, CERT_REQUIRED from ssl import HAS_SNI # Has SNI? except ImportError: pass +try: # Platform-specific: Python 3.6 + from ssl import PROTOCOL_TLS + PROTOCOL_SSLv23 = PROTOCOL_TLS +except ImportError: + try: + from ssl import PROTOCOL_SSLv23 as PROTOCOL_TLS + PROTOCOL_SSLv23 = PROTOCOL_TLS + except ImportError: + PROTOCOL_SSLv23 = PROTOCOL_TLS = 2 + try: from ssl import OP_NO_SSLv2, OP_NO_SSLv3, OP_NO_COMPRESSION @@ -56,25 +76,6 @@ except ImportError: OP_NO_COMPRESSION = 0x20000 -# Python 2.7 doesn't have inet_pton on non-Linux so we fallback on inet_aton in -# those cases. This means that we can only detect IPv4 addresses in this case. -if hasattr(socket, 'inet_pton'): - inet_pton = socket.inet_pton -else: - # Maybe we can use ipaddress if the user has urllib3[secure]? - try: - import ipaddress - - def inet_pton(_, host): - if isinstance(host, bytes): - host = host.decode('ascii') - return ipaddress.ip_address(host) - - except ImportError: # Platform-specific: Non-Linux - def inet_pton(_, host): - return socket.inet_aton(host) - - # A secure default. # Sources for more information on TLS ciphers: # @@ -83,37 +84,35 @@ else: # - https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/ # # The general intent is: -# - Prefer TLS 1.3 cipher suites # - prefer cipher suites that offer perfect forward secrecy (DHE/ECDHE), # - prefer ECDHE over DHE for better performance, # - prefer any AES-GCM and ChaCha20 over any AES-CBC for better performance and # security, # - prefer AES-GCM over ChaCha20 because hardware-accelerated AES is common, -# - disable NULL authentication, MD5 MACs and DSS for security reasons. +# - disable NULL authentication, MD5 MACs, DSS, and other +# insecure ciphers for security reasons. +# - NOTE: TLS 1.3 cipher suites are managed through a different interface +# not exposed by CPython (yet!) and are enabled by default if they're available. DEFAULT_CIPHERS = ':'.join([ - 'TLS13-AES-256-GCM-SHA384', - 'TLS13-CHACHA20-POLY1305-SHA256', - 'TLS13-AES-128-GCM-SHA256', + 'ECDHE+AESGCM', + 'ECDHE+CHACHA20', + 'DHE+AESGCM', + 'DHE+CHACHA20', 'ECDH+AESGCM', - 'ECDH+CHACHA20', 'DH+AESGCM', - 'DH+CHACHA20', - 'ECDH+AES256', - 'DH+AES256', - 'ECDH+AES128', + 'ECDH+AES', 'DH+AES', 'RSA+AESGCM', 'RSA+AES', '!aNULL', '!eNULL', '!MD5', + '!DSS', ]) try: from ssl import SSLContext # Modern SSL? except ImportError: - import sys - class SSLContext(object): # Platform-specific: Python 2 def __init__(self, protocol_version): self.protocol = protocol_version @@ -199,7 +198,7 @@ def resolve_cert_reqs(candidate): constant which can directly be passed to wrap_socket. """ if candidate is None: - return CERT_NONE + return CERT_REQUIRED if isinstance(candidate, str): res = getattr(ssl, candidate, None) @@ -215,7 +214,7 @@ def resolve_ssl_version(candidate): like resolve_cert_reqs """ if candidate is None: - return PROTOCOL_SSLv23 + return PROTOCOL_TLS if isinstance(candidate, str): res = getattr(ssl, candidate, None) @@ -261,7 +260,9 @@ def create_urllib3_context(ssl_version=None, cert_reqs=None, Constructed SSLContext object with specified options :rtype: SSLContext """ - context = SSLContext(ssl_version or ssl.PROTOCOL_SSLv23) + context = SSLContext(ssl_version or PROTOCOL_TLS) + + context.set_ciphers(ciphers or DEFAULT_CIPHERS) # Setting the default here, as we may have no ssl module on import cert_reqs = ssl.CERT_REQUIRED if cert_reqs is None else cert_reqs @@ -289,7 +290,7 @@ def create_urllib3_context(ssl_version=None, cert_reqs=None, def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, ca_certs=None, server_hostname=None, ssl_version=None, ciphers=None, ssl_context=None, - ca_cert_dir=None): + ca_cert_dir=None, key_password=None): """ All arguments except for server_hostname, ssl_context, and ca_cert_dir have the same meaning as they do when using :func:`ssl.wrap_socket`. @@ -305,6 +306,8 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, A directory containing CA certificates in multiple separate files, as supported by OpenSSL's -CApath flag or the capath argument to SSLContext.load_verify_locations(). + :param key_password: + Optional password if the keyfile is encrypted. """ context = ssl_context if context is None: @@ -325,12 +328,22 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, if e.errno == errno.ENOENT: raise SSLError(e) raise - elif getattr(context, 'load_default_certs', None) is not None: + + elif ssl_context is None and hasattr(context, 'load_default_certs'): # try to load OS default certs; works well on Windows (require Python3.4+) context.load_default_certs() + # Attempt to detect if we get the goofy behavior of the + # keyfile being encrypted and OpenSSL asking for the + # passphrase via the terminal and instead error out. + if keyfile and key_password is None and _is_key_file_encrypted(keyfile): + raise SSLError("Client private key is encrypted, password is required") + if certfile: - context.load_cert_chain(certfile, keyfile) + if key_password is None: + context.load_cert_chain(certfile, keyfile) + else: + context.load_cert_chain(certfile, keyfile, key_password) # If we detect server_hostname is an IP address then the SNI # extension should not be used according to RFC3546 Section 3.1 @@ -356,7 +369,8 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, def is_ipaddress(hostname): - """Detects whether the hostname given is an IP address. + """Detects whether the hostname given is an IPv4 or IPv6 address. + Also detects IPv6 addresses with Zone IDs. :param str hostname: Hostname to examine. :return: True if the hostname is an IP address, False otherwise. @@ -364,16 +378,15 @@ def is_ipaddress(hostname): if six.PY3 and isinstance(hostname, bytes): # IDN A-label bytes are ASCII compatible. hostname = hostname.decode('ascii') + return _IP_ADDRESS_REGEX.match(hostname) is not None - families = [socket.AF_INET] - if hasattr(socket, 'AF_INET6'): - families.append(socket.AF_INET6) - for af in families: - try: - inet_pton(af, hostname) - except (socket.error, ValueError, OSError): - pass - else: - return True +def _is_key_file_encrypted(key_file): + """Detects if a key file is encrypted or not.""" + with open(key_file, 'r') as f: + for line in f: + # Look for Proc-Type: 4,ENCRYPTED + if 'ENCRYPTED' in line: + return True + return False diff --git a/pipenv/vendor/urllib3/util/timeout.py b/pipenv/vendor/urllib3/util/timeout.py index cec817e6..a4d004a8 100644 --- a/pipenv/vendor/urllib3/util/timeout.py +++ b/pipenv/vendor/urllib3/util/timeout.py @@ -131,7 +131,8 @@ class Timeout(object): raise ValueError("Attempted to set %s timeout to %s, but the " "timeout cannot be set to a value less " "than or equal to 0." % (name, value)) - except TypeError: # Python 3 + except TypeError: + # Python 3 raise ValueError("Timeout value %s was %s, but it must be an " "int, float or None." % (name, value)) diff --git a/pipenv/vendor/urllib3/util/url.py b/pipenv/vendor/urllib3/util/url.py index 6b6f9968..0bc6ced7 100644 --- a/pipenv/vendor/urllib3/util/url.py +++ b/pipenv/vendor/urllib3/util/url.py @@ -1,7 +1,12 @@ from __future__ import absolute_import +import re from collections import namedtuple from ..exceptions import LocationParseError +from ..packages import six, rfc3986 +from ..packages.rfc3986.exceptions import RFC3986Exception, ValidationError +from ..packages.rfc3986.validators import Validator +from ..packages.rfc3986 import abnf_regexp, normalizers, compat, misc url_attrs = ['scheme', 'auth', 'host', 'port', 'path', 'query', 'fragment'] @@ -10,10 +15,16 @@ url_attrs = ['scheme', 'auth', 'host', 'port', 'path', 'query', 'fragment'] # urllib3 infers URLs without a scheme (None) to be http. NORMALIZABLE_SCHEMES = ('http', 'https', None) +# Regex for detecting URLs with schemes. RFC 3986 Section 3.1 +SCHEME_REGEX = re.compile(r"^(?:[a-zA-Z][a-zA-Z0-9+\-]*:|/)") + +PATH_CHARS = abnf_regexp.UNRESERVED_CHARS_SET | abnf_regexp.SUB_DELIMITERS_SET | {':', '@', '/'} +QUERY_CHARS = FRAGMENT_CHARS = PATH_CHARS | {'?'} + class Url(namedtuple('Url', url_attrs)): """ - Datastructure for representing an HTTP URL. Used as a return value for + Data structure for representing an HTTP URL. Used as a return value for :func:`parse_url`. Both the scheme and host are normalized as they are both case-insensitive according to RFC 3986. """ @@ -23,10 +34,8 @@ class Url(namedtuple('Url', url_attrs)): query=None, fragment=None): if path and not path.startswith('/'): path = '/' + path - if scheme: + if scheme is not None: scheme = scheme.lower() - if host and scheme in NORMALIZABLE_SCHEMES: - host = host.lower() return super(Url, cls).__new__(cls, scheme, auth, host, port, path, query, fragment) @@ -72,23 +81,23 @@ class Url(namedtuple('Url', url_attrs)): 'http://username:password@host.com:80/path?query#fragment' """ scheme, auth, host, port, path, query, fragment = self - url = '' + url = u'' # We use "is not None" we want things to happen with empty strings (or 0 port) if scheme is not None: - url += scheme + '://' + url += scheme + u'://' if auth is not None: - url += auth + '@' + url += auth + u'@' if host is not None: url += host if port is not None: - url += ':' + str(port) + url += u':' + str(port) if path is not None: url += path if query is not None: - url += '?' + query + url += u'?' + query if fragment is not None: - url += '#' + fragment + url += u'#' + fragment return url @@ -98,6 +107,8 @@ class Url(namedtuple('Url', url_attrs)): def split_first(s, delims): """ + .. deprecated:: 1.25 + Given a string and an iterable of delimiters, split on the first found delimiter. Return two split parts and the matched delimiter. @@ -129,10 +140,44 @@ def split_first(s, delims): return s[:min_idx], s[min_idx + 1:], min_delim +def _encode_invalid_chars(component, allowed_chars, encoding='utf-8'): + """Percent-encodes a URI component without reapplying + onto an already percent-encoded component. Based on + rfc3986.normalizers.encode_component() + """ + if component is None: + return component + + # Try to see if the component we're encoding is already percent-encoded + # so we can skip all '%' characters but still encode all others. + percent_encodings = len(normalizers.PERCENT_MATCHER.findall( + compat.to_str(component, encoding))) + + uri_bytes = component.encode('utf-8', 'surrogatepass') + is_percent_encoded = percent_encodings == uri_bytes.count(b'%') + + encoded_component = bytearray() + + for i in range(0, len(uri_bytes)): + # Will return a single character bytestring on both Python 2 & 3 + byte = uri_bytes[i:i+1] + byte_ord = ord(byte) + if ((is_percent_encoded and byte == b'%') + or (byte_ord < 128 and byte.decode() in allowed_chars)): + encoded_component.extend(byte) + continue + encoded_component.extend('%{0:02x}'.format(byte_ord).encode().upper()) + + return encoded_component.decode(encoding) + + def parse_url(url): """ Given a url, return a parsed :class:`.Url` namedtuple. Best-effort is performed to parse incomplete urls. Fields not provided will be None. + This parser is RFC 3986 compliant. + + :param str url: URL to parse into a :class:`.Url` namedtuple. Partly backwards-compatible with :mod:`urlparse`. @@ -145,81 +190,95 @@ def parse_url(url): >>> parse_url('/foo?bar') Url(scheme=None, host=None, port=None, path='/foo', query='bar', ...) """ - - # While this code has overlap with stdlib's urlparse, it is much - # simplified for our needs and less annoying. - # Additionally, this implementations does silly things to be optimal - # on CPython. - if not url: # Empty return Url() - scheme = None - auth = None - host = None - port = None - path = None - fragment = None - query = None + is_string = not isinstance(url, six.binary_type) - # Scheme - if '://' in url: - scheme, url = url.split('://', 1) + # RFC 3986 doesn't like URLs that have a host but don't start + # with a scheme and we support URLs like that so we need to + # detect that problem and add an empty scheme indication. + # We don't get hurt on path-only URLs here as it's stripped + # off and given an empty scheme anyways. + if not SCHEME_REGEX.search(url): + url = "//" + url - # Find the earliest Authority Terminator - # (http://tools.ietf.org/html/rfc3986#section-3.2) - url, path_, delim = split_first(url, ['/', '?', '#']) - - if delim: - # Reassemble the path - path = delim + path_ - - # Auth - if '@' in url: - # Last '@' denotes end of auth part - auth, url = url.rsplit('@', 1) - - # IPv6 - if url and url[0] == '[': - host, url = url.split(']', 1) - host += ']' - - # Port - if ':' in url: - _host, port = url.split(':', 1) - - if not host: - host = _host - - if port: - # If given, ports must be integers. No whitespace, no plus or - # minus prefixes, no non-integer digits such as ^2 (superscript). - if not port.isdigit(): - raise LocationParseError(url) + def idna_encode(name): + if name and any([ord(x) > 128 for x in name]): try: - port = int(port) - except ValueError: - raise LocationParseError(url) - else: - # Blank ports are cool, too. (rfc3986#section-3.2.3) - port = None + import idna + except ImportError: + raise LocationParseError("Unable to parse URL without the 'idna' module") + try: + return idna.encode(name.lower(), strict=True, std3_rules=True) + except idna.IDNAError: + raise LocationParseError(u"Name '%s' is not a valid IDNA label" % name) + return name - elif not host and url: - host = url + try: + split_iri = misc.IRI_MATCHER.match(compat.to_str(url)).groupdict() + iri_ref = rfc3986.IRIReference( + split_iri['scheme'], split_iri['authority'], + _encode_invalid_chars(split_iri['path'], PATH_CHARS), + _encode_invalid_chars(split_iri['query'], QUERY_CHARS), + _encode_invalid_chars(split_iri['fragment'], FRAGMENT_CHARS) + ) + has_authority = iri_ref.authority is not None + uri_ref = iri_ref.encode(idna_encoder=idna_encode) + except (ValueError, RFC3986Exception): + return six.raise_from(LocationParseError(url), None) + # rfc3986 strips the authority if it's invalid + if has_authority and uri_ref.authority is None: + raise LocationParseError(url) + + # Only normalize schemes we understand to not break http+unix + # or other schemes that don't follow RFC 3986. + if uri_ref.scheme is None or uri_ref.scheme.lower() in NORMALIZABLE_SCHEMES: + uri_ref = uri_ref.normalize() + + # Validate all URIReference components and ensure that all + # components that were set before are still set after + # normalization has completed. + validator = Validator() + try: + validator.check_validity_of( + *validator.COMPONENT_NAMES + ).validate(uri_ref) + except ValidationError: + return six.raise_from(LocationParseError(url), None) + + # For the sake of backwards compatibility we put empty + # string values for path if there are any defined values + # beyond the path in the URL. + # TODO: Remove this when we break backwards compatibility. + path = uri_ref.path if not path: - return Url(scheme, auth, host, port, path, query, fragment) + if (uri_ref.query is not None + or uri_ref.fragment is not None): + path = "" + else: + path = None - # Fragment - if '#' in path: - path, fragment = path.split('#', 1) + # Ensure that each part of the URL is a `str` for + # backwards compatibility. + def to_input_type(x): + if x is None: + return None + elif not is_string and not isinstance(x, six.binary_type): + return x.encode('utf-8') + return x - # Query - if '?' in path: - path, query = path.split('?', 1) - - return Url(scheme, auth, host, port, path, query, fragment) + return Url( + scheme=to_input_type(uri_ref.scheme), + auth=to_input_type(uri_ref.userinfo), + host=to_input_type(uri_ref.host), + port=int(uri_ref.port) if uri_ref.port is not None else None, + path=to_input_type(path), + query=to_input_type(uri_ref.query), + fragment=to_input_type(uri_ref.fragment) + ) def get_host(url): diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index ff7226b2..ac5c524d 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -1,52 +1,51 @@ appdirs==1.4.3 backports.shutil_get_terminal_size==1.0.0 backports.weakref==1.0.post1 -blindspin==2.0.1 click==7.0 -click-completion==0.5.0 +click-completion==0.5.1 click-didyoumean==0.0.3 -colorama==0.3.9 +colorama==0.4.1 delegator.py==0.1.1 - pexpect==4.6.0 + pexpect==4.7.0 ptyprocess==0.6.0 -python-dotenv==0.9.1 +python-dotenv==0.10.2 first==2.0.1 iso8601==0.1.12 -jinja2==2.10 -markupsafe==1.0 -parse==1.9.0 -pathlib2==2.3.2 - scandir==1.9 -pipdeptree==0.13.0 +jinja2==2.10. +markupsafe==1.1.1 +parse==1.12.0 +pathlib2==2.3.3 + scandir==1.10 +pipdeptree==0.13.2 pipreqs==0.4.9 docopt==0.6.2 yarg==0.1.9 -pythonfinder==1.1.9.post1 -requests==2.20.1 +pythonfinder==1.2.1 +requests==2.22.0 chardet==3.0.4 - idna==2.7 - urllib3==1.24 - certifi==2018.10.15 -requirementslib==1.3.1.post1 - attrs==18.2.0 - distlib==0.2.8 - packaging==18.0 - pyparsing==2.2.2 - plette==0.2.2 - tomlkit==0.5.2 -shellingham==1.2.7 -six==1.11.0 + idna==2.8 + urllib3==1.25.2 + certifi==2019.3.9 +requirementslib==1.5.1 + attrs==19.1.0 + distlib==0.2.9 + packaging==19.0 + pyparsing==2.3.1 + git+https://github.com/sarugaku/plette.git@master#egg=plette + tomlkit==0.5.3 +shellingham==1.3.1 +six==1.12.0 semver==2.8.1 -shutilwhich==1.1.0 toml==0.10.0 -cached-property==1.4.3 -vistir==0.2.4 +cached-property==1.5.1 +vistir==0.4.2 pip-shims==0.3.2 -ptyprocess==0.6.0 enum34==1.1.6 -yaspin==0.14.0 -cerberus==1.2 -git+https://github.com/sarugaku/passa.git@master#egg=passa -cursor==1.2.0 +yaspin==0.14.3 +cerberus==1.3.1 resolvelib==0.2.2 backports.functools_lru_cache==1.5 +pep517==0.5.0 + pytoml==0.1.20 +git+https://github.com/sarugaku/passa.git@master#egg=passa +orderedmultidict==1.0 diff --git a/pipenv/vendor/vendor_pip.txt b/pipenv/vendor/vendor_pip.txt index 9389dd94..7b548255 100644 --- a/pipenv/vendor/vendor_pip.txt +++ b/pipenv/vendor/vendor_pip.txt @@ -1,23 +1,23 @@ appdirs==1.4.3 -distlib==0.2.7 -distro==1.3.0 -html5lib==1.0.1 -six==1.11.0 -colorama==0.3.9 CacheControl==0.12.5 -msgpack-python==0.5.6 -lockfile==0.12.2 -progress==1.4 +colorama==0.4.1 +distlib==0.2.8 +distro==1.4.0 +html5lib==1.0.1 ipaddress==1.0.22 # Only needed on 2.6 and 2.7 -packaging==18.0 -pep517==0.2 -pyparsing==2.2.1 -pytoml==0.1.19 -retrying==1.3.3 -requests==2.19.1 +lockfile==0.12.2 +msgpack==0.5.6 +packaging==19.0 +pep517==0.5.0 +progress==1.5 +pyparsing==2.4.0 +pytoml==0.1.20 +requests==2.21.0 + certifi==2019.3.9 chardet==3.0.4 - idna==2.7 - urllib3==1.23 - certifi==2018.8.24 -setuptools==40.4.3 + idna==2.8 + urllib3==1.25.2 +retrying==1.3.3 +setuptools==41.0.1 +six==1.12.0 webencodings==0.5.1 diff --git a/pipenv/vendor/vistir/__init__.py b/pipenv/vendor/vistir/__init__.py index 809c973c..821ea29b 100644 --- a/pipenv/vendor/vistir/__init__.py +++ b/pipenv/vendor/vistir/__init__.py @@ -3,6 +3,7 @@ from __future__ import absolute_import, unicode_literals from .compat import ( NamedTemporaryFile, + StringIO, TemporaryDirectory, partialmethod, to_native_string, @@ -11,27 +12,31 @@ from .contextmanagers import ( atomic_open_for_write, cd, open_file, + replaced_stream, + replaced_streams, + spinner, temp_environ, temp_path, - spinner, ) +from .cursor import hide_cursor, show_cursor from .misc import ( + StreamWrapper, + chunked, + decode_for_output, + divide, + get_wrapped_stream, load_path, partialclass, run, shell_escape, - decode_for_output, - to_text, - to_bytes, take, - chunked, - divide, + to_bytes, + to_text, ) -from .path import mkdir_p, rmtree, create_tracked_tempdir, create_tracked_tempfile -from .spin import VistirSpinner, create_spinner +from .path import create_tracked_tempdir, create_tracked_tempfile, mkdir_p, rmtree +from .spin import create_spinner - -__version__ = '0.2.4' +__version__ = "0.4.2" __all__ = [ @@ -50,7 +55,6 @@ __all__ = [ "NamedTemporaryFile", "partialmethod", "spinner", - "VistirSpinner", "create_spinner", "create_tracked_tempdir", "create_tracked_tempfile", @@ -61,4 +65,11 @@ __all__ = [ "take", "chunked", "divide", + "StringIO", + "get_wrapped_stream", + "StreamWrapper", + "replaced_stream", + "replaced_streams", + "show_cursor", + "hide_cursor", ] diff --git a/pipenv/vendor/vistir/_winconsole.py b/pipenv/vendor/vistir/_winconsole.py new file mode 100644 index 00000000..a29c22d8 --- /dev/null +++ b/pipenv/vendor/vistir/_winconsole.py @@ -0,0 +1,523 @@ +# -*- coding: utf-8 -*- + +# This Module is taken in full from the click project +# see https://github.com/pallets/click/blob/6cafd32/click/_winconsole.py +# Copyright © 2014 by the Pallets team. + +# Some rights reserved. + +# Redistribution and use in source and binary forms of the software as well as +# documentation, with or without modification, are permitted provided that the +# following conditions are met: +# Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this +# software without specific prior written permission. + +# THIS SOFTWARE AND DOCUMENTATION IS PROVIDED BY THE COPYRIGHT HOLDERS AND +# CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT +# NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; +# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE AND +# DOCUMENTATION, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# This module is based on the excellent work by Adam Bartoš who +# provided a lot of what went into the implementation here in +# the discussion to issue1602 in the Python bug tracker. +# +# There are some general differences in regards to how this works +# compared to the original patches as we do not need to patch +# the entire interpreter but just work in our little world of +# echo and prmopt. + +import ctypes +import io +import os +import sys +import time +import zlib +from ctypes import ( + POINTER, + WINFUNCTYPE, + Structure, + byref, + c_char, + c_char_p, + c_int, + c_ssize_t, + c_ulong, + c_void_p, + create_unicode_buffer, + py_object, + windll, +) +from ctypes.wintypes import LPCWSTR, LPWSTR +from itertools import count + +import msvcrt +from six import PY2, text_type + +from .compat import IS_TYPE_CHECKING +from .misc import StreamWrapper, run, to_text + +try: + from ctypes import pythonapi + + PyObject_GetBuffer = pythonapi.PyObject_GetBuffer + PyBuffer_Release = pythonapi.PyBuffer_Release +except ImportError: + pythonapi = None + + +if IS_TYPE_CHECKING: + from typing import Text + + +c_ssize_p = POINTER(c_ssize_t) + +kernel32 = windll.kernel32 +GetStdHandle = kernel32.GetStdHandle +ReadConsoleW = kernel32.ReadConsoleW +WriteConsoleW = kernel32.WriteConsoleW +GetLastError = kernel32.GetLastError +GetConsoleCursorInfo = kernel32.GetConsoleCursorInfo +SetConsoleCursorInfo = kernel32.SetConsoleCursorInfo +GetCommandLineW = WINFUNCTYPE(LPWSTR)(("GetCommandLineW", windll.kernel32)) +CommandLineToArgvW = WINFUNCTYPE(POINTER(LPWSTR), LPCWSTR, POINTER(c_int))( + ("CommandLineToArgvW", windll.shell32) +) + + +# XXX: Added for cursor hiding on windows +STDOUT_HANDLE_ID = ctypes.c_ulong(-11) +STDERR_HANDLE_ID = ctypes.c_ulong(-12) +STDIN_HANDLE = GetStdHandle(-10) +STDOUT_HANDLE = GetStdHandle(-11) +STDERR_HANDLE = GetStdHandle(-12) + +STREAM_MAP = {0: STDIN_HANDLE, 1: STDOUT_HANDLE, 2: STDERR_HANDLE} + + +PyBUF_SIMPLE = 0 +PyBUF_WRITABLE = 1 + +ERROR_SUCCESS = 0 +ERROR_NOT_ENOUGH_MEMORY = 8 +ERROR_OPERATION_ABORTED = 995 + +STDIN_FILENO = 0 +STDOUT_FILENO = 1 +STDERR_FILENO = 2 + +EOF = b"\x1a" +MAX_BYTES_WRITTEN = 32767 + + +class Py_buffer(Structure): + _fields_ = [ + ("buf", c_void_p), + ("obj", py_object), + ("len", c_ssize_t), + ("itemsize", c_ssize_t), + ("readonly", c_int), + ("ndim", c_int), + ("format", c_char_p), + ("shape", c_ssize_p), + ("strides", c_ssize_p), + ("suboffsets", c_ssize_p), + ("internal", c_void_p), + ] + + if PY2: + _fields_.insert(-1, ("smalltable", c_ssize_t * 2)) + + +# XXX: This was added for the use of cursors +class CONSOLE_CURSOR_INFO(Structure): + _fields_ = [("dwSize", ctypes.c_int), ("bVisible", ctypes.c_int)] + + +# On PyPy we cannot get buffers so our ability to operate here is +# serverly limited. +if pythonapi is None: + get_buffer = None +else: + + def get_buffer(obj, writable=False): + buf = Py_buffer() + flags = PyBUF_WRITABLE if writable else PyBUF_SIMPLE + PyObject_GetBuffer(py_object(obj), byref(buf), flags) + try: + buffer_type = c_char * buf.len + return buffer_type.from_address(buf.buf) + finally: + PyBuffer_Release(byref(buf)) + + +def get_long_path(short_path): + # type: (Text, str) -> Text + BUFFER_SIZE = 500 + buffer = create_unicode_buffer(BUFFER_SIZE) + get_long_path_name = windll.kernel32.GetLongPathNameW + get_long_path_name(to_text(short_path), buffer, BUFFER_SIZE) + return buffer.value + + +class _WindowsConsoleRawIOBase(io.RawIOBase): + def __init__(self, handle): + self.handle = handle + + def isatty(self): + io.RawIOBase.isatty(self) + return True + + +class _WindowsConsoleReader(_WindowsConsoleRawIOBase): + def readable(self): + return True + + def readinto(self, b): + bytes_to_be_read = len(b) + if not bytes_to_be_read: + return 0 + elif bytes_to_be_read % 2: + raise ValueError( + "cannot read odd number of bytes from " "UTF-16-LE encoded console" + ) + + buffer = get_buffer(b, writable=True) + code_units_to_be_read = bytes_to_be_read // 2 + code_units_read = c_ulong() + + rv = ReadConsoleW( + self.handle, buffer, code_units_to_be_read, byref(code_units_read), None + ) + if GetLastError() == ERROR_OPERATION_ABORTED: + # wait for KeyboardInterrupt + time.sleep(0.1) + if not rv: + raise OSError("Windows error: %s" % GetLastError()) + + if buffer[0] == EOF: + return 0 + return 2 * code_units_read.value + + +class _WindowsConsoleWriter(_WindowsConsoleRawIOBase): + def writable(self): + return True + + @staticmethod + def _get_error_message(errno): + if errno == ERROR_SUCCESS: + return "ERROR_SUCCESS" + elif errno == ERROR_NOT_ENOUGH_MEMORY: + return "ERROR_NOT_ENOUGH_MEMORY" + return "Windows error %s" % errno + + def write(self, b): + bytes_to_be_written = len(b) + buf = get_buffer(b) + code_units_to_be_written = min(bytes_to_be_written, MAX_BYTES_WRITTEN) // 2 + code_units_written = c_ulong() + + WriteConsoleW( + self.handle, buf, code_units_to_be_written, byref(code_units_written), None + ) + bytes_written = 2 * code_units_written.value + + if bytes_written == 0 and bytes_to_be_written > 0: + raise OSError(self._get_error_message(GetLastError())) + return bytes_written + + +class ConsoleStream(object): + def __init__(self, text_stream, byte_stream): + self._text_stream = text_stream + self.buffer = byte_stream + + @property + def name(self): + return self.buffer.name + + @property + def fileno(self): + return self.buffer.fileno + + def write(self, x): + if isinstance(x, text_type): + return self._text_stream.write(x) + try: + self.flush() + except Exception: + pass + return self.buffer.write(x) + + def writelines(self, lines): + for line in lines: + self.write(line) + + def __getattr__(self, name): + try: + return getattr(self._text_stream, name) + except io.UnsupportedOperation: + return getattr(self.buffer, name) + + def isatty(self): + return self.buffer.isatty() + + def __repr__(self): + return "" % (self.name, self.encoding) + + +class WindowsChunkedWriter(object): + """ + Wraps a stream (such as stdout), acting as a transparent proxy for all + attribute access apart from method 'write()' which we wrap to write in + limited chunks due to a Windows limitation on binary console streams. + """ + + def __init__(self, wrapped): + # double-underscore everything to prevent clashes with names of + # attributes on the wrapped stream object. + self.__wrapped = wrapped + + def __getattr__(self, name): + return getattr(self.__wrapped, name) + + def write(self, text): + total_to_write = len(text) + written = 0 + + while written < total_to_write: + to_write = min(total_to_write - written, MAX_BYTES_WRITTEN) + self.__wrapped.write(text[written : written + to_write]) + written += to_write + + +_wrapped_std_streams = set() + + +def _wrap_std_stream(name): + # Python 2 & Windows 7 and below + if PY2 and sys.getwindowsversion()[:2] <= (6, 1) and name not in _wrapped_std_streams: + setattr(sys, name, WindowsChunkedWriter(getattr(sys, name))) + _wrapped_std_streams.add(name) + + +def _get_text_stdin(buffer_stream): + text_stream = StreamWrapper( + io.BufferedReader(_WindowsConsoleReader(STDIN_HANDLE)), + "utf-16-le", + "strict", + line_buffering=True, + ) + return ConsoleStream(text_stream, buffer_stream) + + +def _get_text_stdout(buffer_stream): + text_stream = StreamWrapper( + io.BufferedWriter(_WindowsConsoleWriter(STDOUT_HANDLE)), + "utf-16-le", + "strict", + line_buffering=True, + ) + return ConsoleStream(text_stream, buffer_stream) + + +def _get_text_stderr(buffer_stream): + text_stream = StreamWrapper( + io.BufferedWriter(_WindowsConsoleWriter(STDERR_HANDLE)), + "utf-16-le", + "strict", + line_buffering=True, + ) + return ConsoleStream(text_stream, buffer_stream) + + +if PY2: + + def _hash_py_argv(): + return zlib.crc32("\x00".join(sys.argv[1:])) + + _initial_argv_hash = _hash_py_argv() + + def _get_windows_argv(): + argc = c_int(0) + argv_unicode = CommandLineToArgvW(GetCommandLineW(), byref(argc)) + argv = [argv_unicode[i] for i in range(0, argc.value)] + + if not hasattr(sys, "frozen"): + argv = argv[1:] + while len(argv) > 0: + arg = argv[0] + if not arg.startswith("-") or arg == "-": + break + argv = argv[1:] + if arg.startswith(("-c", "-m")): + break + + return argv[1:] + + +_stream_factories = {0: _get_text_stdin, 1: _get_text_stdout, 2: _get_text_stderr} + + +def _get_windows_console_stream(f, encoding, errors): + if ( + get_buffer is not None + and encoding in ("utf-16-le", None) + and errors in ("strict", None) + and hasattr(f, "isatty") + and f.isatty() + ): + if isinstance(f, ConsoleStream): + return f + func = _stream_factories.get(f.fileno()) + if func is not None: + if not PY2: + f = getattr(f, "buffer", None) + if f is None: + return None + else: + # If we are on Python 2 we need to set the stream that we + # deal with to binary mode as otherwise the exercise if a + # bit moot. The same problems apply as for + # get_binary_stdin and friends from _compat. + msvcrt.setmode(f.fileno(), os.O_BINARY) + return func(f) + + +def hide_cursor(): + cursor_info = CONSOLE_CURSOR_INFO() + GetConsoleCursorInfo(STDOUT_HANDLE, ctypes.byref(cursor_info)) + cursor_info.visible = False + SetConsoleCursorInfo(STDOUT_HANDLE, ctypes.byref(cursor_info)) + + +def show_cursor(): + cursor_info = CONSOLE_CURSOR_INFO() + GetConsoleCursorInfo(STDOUT_HANDLE, ctypes.byref(cursor_info)) + cursor_info.visible = True + SetConsoleCursorInfo(STDOUT_HANDLE, ctypes.byref(cursor_info)) + + +def get_stream_handle(stream): + return STREAM_MAP.get(stream.fileno()) + + +def _walk_for_powershell(directory): + for path, dirs, files in os.walk(directory): + powershell = next( + iter(fn for fn in files if fn.lower() == "powershell.exe"), None + ) + if powershell is not None: + return os.path.join(directory, powershell) + for subdir in dirs: + powershell = _walk_for_powershell(os.path.join(directory, subdir)) + if powershell: + return powershell + return None + + +def _get_powershell_path(): + paths = [ + os.path.expandvars(r"%windir%\{0}\WindowsPowerShell").format(subdir) + for subdir in ("SysWOW64", "system32") + ] + powershell_path = next(iter(_walk_for_powershell(pth) for pth in paths), None) + if not powershell_path: + powershell_path, _ = run( + ["where", "powershell"], block=True, nospin=True, return_object=False + ) + if powershell_path: + return powershell_path.strip() + return None + + +def _get_sid_with_powershell(): + powershell_path = _get_powershell_path() + if not powershell_path: + return None + args = [ + powershell_path, + "-ExecutionPolicy", + "Bypass", + "-Command", + "Invoke-Expression '[System.Security.Principal.WindowsIdentity]::GetCurrent().user | Write-Host'", + ] + sid, _ = run(args, nospin=True) + return sid.strip() + + +def _get_sid_from_registry(): + try: + import winreg + except ImportError: + import _winreg as winreg + var_names = ("%USERPROFILE%", "%HOME%") + current_user_home = next(iter(os.path.expandvars(v) for v in var_names if v), None) + root, subkey = ( + winreg.HKEY_LOCAL_MACHINE, + r"Software\Microsoft\Windows NT\CurrentVersion\ProfileList", + ) + subkey_names = [] + value = None + matching_key = None + try: + with winreg.OpenKeyEx(root, subkey, 0, winreg.KEY_READ) as key: + for i in count(): + key_name = winreg.EnumKey(key, i) + subkey_names.append(key_name) + value = query_registry_value( + root, r"{0}\{1}".format(subkey, key_name), "ProfileImagePath" + ) + if value and value.lower() == current_user_home.lower(): + matching_key = key_name + break + except OSError: + pass + if matching_key is not None: + return matching_key + + +def get_value_from_tuple(value, value_type): + try: + import winreg + except ImportError: + import _winreg as winreg + if value_type in (winreg.REG_SZ, winreg.REG_EXPAND_SZ): + if "\0" in value: + return value[: value.index("\0")] + return value + return None + + +def query_registry_value(root, key_name, value): + try: + import winreg + except ImportError: + import _winreg as winreg + try: + with winreg.OpenKeyEx(root, key_name, 0, winreg.KEY_READ) as key: + return get_value_from_tuple(*winreg.QueryValueEx(key, value)) + except OSError: + return None + + +def get_current_user(): + fns = (_get_sid_from_registry, _get_sid_with_powershell) + for fn in fns: + result = fn() + if result: + return result + return None diff --git a/pipenv/vendor/vistir/backports/__init__.py b/pipenv/vendor/vistir/backports/__init__.py index 0bdac1ea..b5ed6562 100644 --- a/pipenv/vendor/vistir/backports/__init__.py +++ b/pipenv/vendor/vistir/backports/__init__.py @@ -2,10 +2,7 @@ from __future__ import absolute_import, unicode_literals from .functools import partialmethod +from .surrogateescape import register_surrogateescape from .tempfile import NamedTemporaryFile - -__all__ = [ - "NamedTemporaryFile", - "partialmethod" -] +__all__ = ["NamedTemporaryFile", "partialmethod", "register_surrogateescape"] diff --git a/pipenv/vendor/vistir/backports/functools.py b/pipenv/vendor/vistir/backports/functools.py index 8060d183..5d98f41a 100644 --- a/pipenv/vendor/vistir/backports/functools.py +++ b/pipenv/vendor/vistir/backports/functools.py @@ -3,8 +3,7 @@ from __future__ import absolute_import, unicode_literals from functools import partial - -__all__ = ["partialmethod",] +__all__ = ["partialmethod"] class partialmethod(object): @@ -16,8 +15,7 @@ class partialmethod(object): def __init__(self, func, *args, **keywords): if not callable(func) and not hasattr(func, "__get__"): - raise TypeError("{!r} is not callable or a descriptor" - .format(func)) + raise TypeError("{!r} is not callable or a descriptor".format(func)) # func could be a descriptor like classmethod which isn't callable, # so we can't inherit from partial (it verifies func is callable) @@ -36,26 +34,28 @@ class partialmethod(object): def __repr__(self): args = ", ".join(map(repr, self.args)) - keywords = ", ".join("{}={!r}".format(k, v) - for k, v in self.keywords.items()) + keywords = ", ".join("{}={!r}".format(k, v) for k, v in self.keywords.items()) format_string = "{module}.{cls}({func}, {args}, {keywords})" - return format_string.format(module=self.__class__.__module__, - cls=self.__class__.__qualname__, - func=self.func, - args=args, - keywords=keywords) + return format_string.format( + module=self.__class__.__module__, + cls=self.__class__.__qualname__, + func=self.func, + args=args, + keywords=keywords, + ) def _make_unbound_method(self): def _method(*args, **keywords): call_keywords = self.keywords.copy() call_keywords.update(keywords) if len(args) > 1: - cls_or_self, rest = args[0], tuple(args[1:],) + cls_or_self, rest = args[0], tuple(args[1:]) else: cls_or_self = args[0] rest = tuple() call_args = (cls_or_self,) + self.args + tuple(rest) return self.func(*call_args, **call_keywords) + _method.__isabstractmethod__ = self.__isabstractmethod__ _method._partialmethod = self return _method diff --git a/pipenv/vendor/vistir/backports/surrogateescape.py b/pipenv/vendor/vistir/backports/surrogateescape.py new file mode 100644 index 00000000..0532be08 --- /dev/null +++ b/pipenv/vendor/vistir/backports/surrogateescape.py @@ -0,0 +1,196 @@ +""" +This is Victor Stinner's pure-Python implementation of PEP 383: the "surrogateescape" error +handler of Python 3. +Source: misc/python/surrogateescape.py in https://bitbucket.org/haypo/misc +""" + +# This code is released under the Python license and the BSD 2-clause license + +import codecs +import sys + +import six + +FS_ERRORS = "surrogateescape" + +# # -- Python 2/3 compatibility ------------------------------------- +# FS_ERRORS = 'my_surrogateescape' + + +def u(text): + if six.PY3: + return text + else: + return text.decode("unicode_escape") + + +def b(data): + if six.PY3: + return data.encode("latin1") + else: + return data + + +if six.PY3: + _unichr = chr + bytes_chr = lambda code: bytes((code,)) +else: + _unichr = unichr + bytes_chr = chr + + +def surrogateescape_handler(exc): + """ + Pure Python implementation of the PEP 383: the "surrogateescape" error + handler of Python 3. Undecodable bytes will be replaced by a Unicode + character U+DCxx on decoding, and these are translated into the + original bytes on encoding. + """ + mystring = exc.object[exc.start : exc.end] + + try: + if isinstance(exc, UnicodeDecodeError): + # mystring is a byte-string in this case + decoded = replace_surrogate_decode(mystring) + elif isinstance(exc, UnicodeEncodeError): + # In the case of u'\udcc3'.encode('ascii', + # 'this_surrogateescape_handler'), both Python 2.x and 3.x raise an + # exception anyway after this function is called, even though I think + # it's doing what it should. It seems that the strict encoder is called + # to encode the unicode string that this function returns ... + decoded = replace_surrogate_encode(mystring) + else: + raise exc + except NotASurrogateError: + raise exc + return (decoded, exc.end) + + +class NotASurrogateError(Exception): + pass + + +def replace_surrogate_encode(mystring): + """ + Returns a (unicode) string, not the more logical bytes, because the codecs + register_error functionality expects this. + """ + decoded = [] + for ch in mystring: + # if utils.PY3: + # code = ch + # else: + code = ord(ch) + + # The following magic comes from Py3.3's Python/codecs.c file: + if not 0xD800 <= code <= 0xDCFF: + # Not a surrogate. Fail with the original exception. + raise NotASurrogateError + # mybytes = [0xe0 | (code >> 12), + # 0x80 | ((code >> 6) & 0x3f), + # 0x80 | (code & 0x3f)] + # Is this a good idea? + if 0xDC00 <= code <= 0xDC7F: + decoded.append(_unichr(code - 0xDC00)) + elif code <= 0xDCFF: + decoded.append(_unichr(code - 0xDC00)) + else: + raise NotASurrogateError + return str().join(decoded) + + +def replace_surrogate_decode(mybytes): + """ + Returns a (unicode) string + """ + decoded = [] + for ch in mybytes: + # We may be parsing newbytes (in which case ch is an int) or a native + # str on Py2 + if isinstance(ch, int): + code = ch + else: + code = ord(ch) + if 0x80 <= code <= 0xFF: + decoded.append(_unichr(0xDC00 + code)) + elif code <= 0x7F: + decoded.append(_unichr(code)) + else: + # # It may be a bad byte + # # Try swallowing it. + # continue + # print("RAISE!") + raise NotASurrogateError + return str().join(decoded) + + +def encodefilename(fn): + if FS_ENCODING == "ascii": + # ASCII encoder of Python 2 expects that the error handler returns a + # Unicode string encodable to ASCII, whereas our surrogateescape error + # handler has to return bytes in 0x80-0xFF range. + encoded = [] + for index, ch in enumerate(fn): + code = ord(ch) + if code < 128: + ch = bytes_chr(code) + elif 0xDC80 <= code <= 0xDCFF: + ch = bytes_chr(code - 0xDC00) + else: + raise UnicodeEncodeError( + FS_ENCODING, fn, index, index + 1, "ordinal not in range(128)" + ) + encoded.append(ch) + return bytes().join(encoded) + elif FS_ENCODING == "utf-8": + # UTF-8 encoder of Python 2 encodes surrogates, so U+DC80-U+DCFF + # doesn't go through our error handler + encoded = [] + for index, ch in enumerate(fn): + code = ord(ch) + if 0xD800 <= code <= 0xDFFF: + if 0xDC80 <= code <= 0xDCFF: + ch = bytes_chr(code - 0xDC00) + encoded.append(ch) + else: + raise UnicodeEncodeError( + FS_ENCODING, fn, index, index + 1, "surrogates not allowed" + ) + else: + ch_utf8 = ch.encode("utf-8") + encoded.append(ch_utf8) + return bytes().join(encoded) + else: + return fn.encode(FS_ENCODING, FS_ERRORS) + + +def decodefilename(fn): + return fn.decode(FS_ENCODING, FS_ERRORS) + + +FS_ENCODING = "ascii" +fn = b("[abc\xff]") +encoded = u("[abc\udcff]") +# FS_ENCODING = 'cp932'; fn = b('[abc\x81\x00]'); encoded = u('[abc\udc81\x00]') +# FS_ENCODING = 'UTF-8'; fn = b('[abc\xff]'); encoded = u('[abc\udcff]') + + +# normalize the filesystem encoding name. +# For example, we expect "utf-8", not "UTF8". +FS_ENCODING = codecs.lookup(FS_ENCODING).name + + +def register_surrogateescape(): + """ + Registers the surrogateescape error handler on Python 2 (only) + """ + if six.PY3: + return + try: + codecs.lookup_error(FS_ERRORS) + except LookupError: + codecs.register_error(FS_ERRORS, surrogateescape_handler) + + +if __name__ == "__main__": + pass diff --git a/pipenv/vendor/vistir/backports/tempfile.py b/pipenv/vendor/vistir/backports/tempfile.py index fb044acf..a3d7f3df 100644 --- a/pipenv/vendor/vistir/backports/tempfile.py +++ b/pipenv/vendor/vistir/backports/tempfile.py @@ -5,7 +5,6 @@ import functools import io import os import sys - from tempfile import _bin_openflags, _mkstemp_inner, gettempdir import six @@ -16,6 +15,24 @@ except ImportError: from pipenv.vendor.backports.weakref import finalize +def fs_encode(path): + try: + return os.fsencode(path) + except AttributeError: + from ..compat import fs_encode + + return fs_encode(path) + + +def fs_decode(path): + try: + return os.fsdecode(path) + except AttributeError: + from ..compat import fs_decode + + return fs_decode(path) + + __all__ = ["finalize", "NamedTemporaryFile"] @@ -49,7 +66,7 @@ def _sanitize_params(prefix, suffix, dir): if output_type is str: dir = gettempdir() else: - dir = os.fsencode(gettempdir()) + dir = fs_encode(gettempdir()) return prefix, suffix, dir, output_type @@ -175,7 +192,7 @@ def NamedTemporaryFile( prefix=None, dir=None, delete=True, - wrapper_class_override=None + wrapper_class_override=None, ): """Create and return a temporary file. Arguments: @@ -203,13 +220,11 @@ def NamedTemporaryFile( else: (fd, name) = _mkstemp_inner(dir, prefix, suffix, flags, output_type) try: - file = io.open( - fd, mode, buffering=buffering, newline=newline, encoding=encoding - ) + file = io.open(fd, mode, buffering=buffering, newline=newline, encoding=encoding) if wrapper_class_override is not None: - return type( - str("_TempFileWrapper"), (wrapper_class_override, object), {} - )(file, name, delete) + return type(str("_TempFileWrapper"), (wrapper_class_override, object), {})( + file, name, delete + ) else: return _TemporaryFileWrapper(file, name, delete) diff --git a/pipenv/vendor/vistir/cmdparse.py b/pipenv/vendor/vistir/cmdparse.py index 07326c93..664ae7df 100644 --- a/pipenv/vendor/vistir/cmdparse.py +++ b/pipenv/vendor/vistir/cmdparse.py @@ -1,12 +1,12 @@ # -*- coding=utf-8 -*- from __future__ import absolute_import, unicode_literals +import itertools import re import shlex import six - __all__ = ["ScriptEmptyError", "Script"] @@ -14,6 +14,12 @@ class ScriptEmptyError(ValueError): pass +def _quote_if_contains(value, pattern): + if next(re.finditer(pattern, value), None): + return '"{0}"'.format(re.sub(r'(\\*)"', r'\1\1\\"', value)) + return value + + class Script(object): """Parse a script line (in Pipfile's [scripts] section). @@ -72,7 +78,8 @@ class Script(object): See also: https://docs.python.org/3/library/subprocess.html#converting-argument-sequence """ return " ".join( - arg if not next(re.finditer(r'\s', arg), None) - else '"{0}"'.format(re.sub(r'(\\*)"', r'\1\1\\"', arg)) - for arg in self._parts + itertools.chain( + [_quote_if_contains(self.command, r"[\s^()]")], + (_quote_if_contains(arg, r"[\s^]") for arg in self.args), + ) ) diff --git a/pipenv/vendor/vistir/compat.py b/pipenv/vendor/vistir/compat.py index 83226481..ee96f761 100644 --- a/pipenv/vendor/vistir/compat.py +++ b/pipenv/vendor/vistir/compat.py @@ -1,15 +1,16 @@ # -*- coding=utf-8 -*- -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals +import codecs import errno import os import sys import warnings - from tempfile import mkdtemp import six +from .backports.tempfile import NamedTemporaryFile as _NamedTemporaryFile __all__ = [ "Path", @@ -19,47 +20,57 @@ __all__ = [ "JSONDecodeError", "FileNotFoundError", "ResourceWarning", - "FileNotFoundError", "PermissionError", + "is_type_checking", + "IS_TYPE_CHECKING", "IsADirectoryError", "fs_str", "lru_cache", "TemporaryDirectory", "NamedTemporaryFile", "to_native_string", + "Iterable", + "Mapping", + "Sequence", + "Set", + "ItemsView", + "fs_encode", + "fs_decode", + "_fs_encode_errors", + "_fs_decode_errors", ] -if sys.version_info >= (3, 5): +if sys.version_info >= (3, 5): # pragma: no cover from pathlib import Path - from functools import lru_cache -else: - from pathlib2 import Path - from pipenv.vendor.backports.functools_lru_cache import lru_cache +else: # pragma: no cover + from pipenv.vendor.pathlib2 import Path -from .backports.tempfile import NamedTemporaryFile as _NamedTemporaryFile -if sys.version_info < (3, 3): - from pipenv.vendor.backports.shutil_get_terminal_size import get_terminal_size - NamedTemporaryFile = _NamedTemporaryFile -else: +if six.PY3: # pragma: no cover + # Only Python 3.4+ is supported + from functools import lru_cache, partialmethod from tempfile import NamedTemporaryFile from shutil import get_terminal_size - -try: from weakref import finalize -except ImportError: - from pipenv.vendor.backports.weakref import finalize - -try: - from functools import partialmethod -except Exception: - from .backports.functools import partialmethod +else: # pragma: no cover + # Only Python 2.7 is supported + from pipenv.vendor.backports.functools_lru_cache import lru_cache + from .backports.functools import partialmethod # type: ignore + from pipenv.vendor.backports.shutil_get_terminal_size import get_terminal_size + from .backports.surrogateescape import register_surrogateescape + + register_surrogateescape() + NamedTemporaryFile = _NamedTemporaryFile + from pipenv.vendor.backports.weakref import finalize # type: ignore try: + # Introduced Python 3.5 from json import JSONDecodeError -except ImportError: # Old Pythons. - JSONDecodeError = ValueError +except ImportError: # pragma: no cover + JSONDecodeError = ValueError # type: ignore -if six.PY2: +if six.PY2: # pragma: no cover + + from io import BytesIO as StringIO class ResourceWarning(Warning): pass @@ -76,20 +87,75 @@ if six.PY2: self.errno = errno.EACCES super(PermissionError, self).__init__(*args, **kwargs) + class TimeoutError(OSError): + """Timeout expired.""" + + def __init__(self, *args, **kwargs): + self.errno = errno.ETIMEDOUT + super(TimeoutError, self).__init__(*args, **kwargs) + class IsADirectoryError(OSError): """The command does not work on directories""" - pass -else: - from builtins import ResourceWarning, FileNotFoundError, PermissionError, IsADirectoryError + def __init__(self, *args, **kwargs): + self.errno = errno.EISDIR + super(IsADirectoryError, self).__init__(*args, **kwargs) + + class FileExistsError(OSError): + def __init__(self, *args, **kwargs): + self.errno = errno.EEXIST + super(FileExistsError, self).__init__(*args, **kwargs) + + +else: # pragma: no cover + from builtins import ( + ResourceWarning, + FileNotFoundError, + PermissionError, + IsADirectoryError, + FileExistsError, + TimeoutError, + ) + from io import StringIO + +six.add_move( + six.MovedAttribute("Iterable", "collections", "collections.abc") +) # type: ignore +six.add_move( + six.MovedAttribute("Mapping", "collections", "collections.abc") +) # type: ignore +six.add_move( + six.MovedAttribute("Sequence", "collections", "collections.abc") +) # type: ignore +six.add_move(six.MovedAttribute("Set", "collections", "collections.abc")) # type: ignore +six.add_move( + six.MovedAttribute("ItemsView", "collections", "collections.abc") +) # type: ignore + +# fmt: off +from six.moves import ItemsView, Iterable, Mapping, Sequence, Set # type: ignore # noqa # isort:skip +# fmt: on if not sys.warnoptions: warnings.simplefilter("default", ResourceWarning) +def is_type_checking(): + try: + from typing import TYPE_CHECKING + except ImportError: + return False + return TYPE_CHECKING + + +IS_TYPE_CHECKING = os.environ.get("MYPY_RUNNING", is_type_checking()) + + class TemporaryDirectory(object): - """Create and return a temporary directory. This has the same + + """ + Create and return a temporary directory. This has the same behavior as mkdtemp but can be used as a context manager. For example: @@ -125,32 +191,7 @@ class TemporaryDirectory(object): def _rmtree(cls, name): from .path import rmtree - def onerror(func, path, exc_info): - if issubclass(exc_info[0], (PermissionError, OSError)): - try: - try: - if path != name: - os.chflags(os.path.dirname(path), 0) - os.chflags(path, 0) - except AttributeError: - pass - if path != name: - os.chmod(os.path.dirname(path), 0o70) - os.chmod(path, 0o700) - - try: - os.unlink(path) - # PermissionError is raised on FreeBSD for directories - except (IsADirectoryError, PermissionError, OSError): - cls._rmtree(path) - except FileNotFoundError: - pass - elif issubclass(exc_info[0], FileNotFoundError): - pass - else: - raise - - rmtree(name, onerror=onerror) + rmtree(name) @classmethod def _cleanup(cls, name, warn_message): @@ -171,22 +212,197 @@ class TemporaryDirectory(object): self._rmtree(self.name) +def is_bytes(string): + """Check if a string is a bytes instance + + :param Union[str, bytes] string: A string that may be string or bytes like + :return: Whether the provided string is a bytes type or not + :rtype: bool + """ + if six.PY3 and isinstance(string, (bytes, memoryview, bytearray)): # noqa + return True + elif six.PY2 and isinstance(string, (buffer, bytearray)): # noqa + return True + return False + + def fs_str(string): """Encodes a string into the proper filesystem encoding Borrowed from pip-tools """ + if isinstance(string, str): return string assert not isinstance(string, bytes) return string.encode(_fs_encoding) -_fs_encoding = sys.getfilesystemencoding() or sys.getdefaultencoding() +def _get_path(path): + """ + Fetch the string value from a path-like object + + Returns **None** if there is no string value. + """ + + if isinstance(path, (six.string_types, bytes)): + return path + path_type = type(path) + try: + path_repr = path_type.__fspath__(path) + except AttributeError: + return + if isinstance(path_repr, (six.string_types, bytes)): + return path_repr + return + + +# copied from the os backport which in turn copied this from +# the pyutf8 package -- +# URL: https://github.com/etrepum/pyutf8/blob/master/pyutf8/ref.py +# +def _invalid_utf8_indexes(bytes): + skips = [] + i = 0 + len_bytes = len(bytes) + while i < len_bytes: + c1 = bytes[i] + if c1 < 0x80: + # U+0000 - U+007F - 7 bits + i += 1 + continue + try: + c2 = bytes[i + 1] + if (c1 & 0xE0 == 0xC0) and (c2 & 0xC0 == 0x80): + # U+0080 - U+07FF - 11 bits + c = ((c1 & 0x1F) << 6) | (c2 & 0x3F) + if c < 0x80: # pragma: no cover + # Overlong encoding + skips.extend([i, i + 1]) # pragma: no cover + i += 2 + continue + c3 = bytes[i + 2] + if (c1 & 0xF0 == 0xE0) and (c2 & 0xC0 == 0x80) and (c3 & 0xC0 == 0x80): + # U+0800 - U+FFFF - 16 bits + c = ((((c1 & 0x0F) << 6) | (c2 & 0x3F)) << 6) | (c3 & 0x3F) + if (c < 0x800) or (0xD800 <= c <= 0xDFFF): + # Overlong encoding or surrogate. + skips.extend([i, i + 1, i + 2]) + i += 3 + continue + c4 = bytes[i + 3] + if ( + (c1 & 0xF8 == 0xF0) + and (c2 & 0xC0 == 0x80) + and (c3 & 0xC0 == 0x80) + and (c4 & 0xC0 == 0x80) + ): + # U+10000 - U+10FFFF - 21 bits + c = ((((((c1 & 0x0F) << 6) | (c2 & 0x3F)) << 6) | (c3 & 0x3F)) << 6) | ( + c4 & 0x3F + ) + if (c < 0x10000) or (c > 0x10FFFF): # pragma: no cover + # Overlong encoding or invalid code point. + skips.extend([i, i + 1, i + 2, i + 3]) + i += 4 + continue + except IndexError: + pass + skips.append(i) + i += 1 + return skips + + +# XXX backport: Another helper to support the Python 2 UTF-8 decoding hack. +def _chunks(b, indexes): + i = 0 + for j in indexes: + yield b[i:j] + yield b[j : j + 1] + i = j + 1 + yield b[i:] + + +def fs_encode(path): + """ + Encode a filesystem path to the proper filesystem encoding + + :param Union[str, bytes] path: A string-like path + :returns: A bytes-encoded filesystem path representation + """ + + path = _get_path(path) + if path is None: + raise TypeError("expected a valid path to encode") + if isinstance(path, six.text_type): + if six.PY2: + return b"".join( + ( + _byte(ord(c) - 0xDC00) + if 0xDC00 <= ord(c) <= 0xDCFF + else c.encode(_fs_encoding, _fs_encode_errors) + ) + for c in path + ) + return path.encode(_fs_encoding, _fs_encode_errors) + return path + + +def fs_decode(path): + """ + Decode a filesystem path using the proper filesystem encoding + + :param path: The filesystem path to decode from bytes or string + :return: The filesystem path, decoded with the determined encoding + :rtype: Text + """ + + path = _get_path(path) + if path is None: + raise TypeError("expected a valid path to decode") + if isinstance(path, six.binary_type): + import array + + indexes = _invalid_utf8_indexes(array.array(str("B"), path)) + if six.PY2: + return "".join( + chunk.decode(_fs_encoding, _fs_decode_errors) + for chunk in _chunks(path, indexes) + ) + if indexes and os.name == "nt": + return path.decode(_fs_encoding, "surrogateescape") + return path.decode(_fs_encoding, _fs_decode_errors) + return path + + +if sys.version_info[0] < 3: # pragma: no cover + _fs_encode_errors = "surrogateescape" + _fs_decode_errors = "surrogateescape" + _fs_encoding = "utf-8" +else: # pragma: no cover + _fs_encoding = "utf-8" + if sys.platform.startswith("win"): + _fs_error_fn = None + if sys.version_info[:2] > (3, 4): + alt_strategy = "surrogatepass" + else: + alt_strategy = "surrogateescape" + else: + if sys.version_info >= (3, 3): + _fs_encoding = sys.getfilesystemencoding() + if not _fs_encoding: + _fs_encoding = sys.getdefaultencoding() + alt_strategy = "surrogateescape" + _fs_error_fn = getattr(sys, "getfilesystemencodeerrors", None) + _fs_encode_errors = _fs_error_fn() if _fs_error_fn else alt_strategy + _fs_decode_errors = _fs_error_fn() if _fs_error_fn else alt_strategy + +_byte = chr if sys.version_info < (3,) else lambda i: bytes([i]) def to_native_string(string): from .misc import to_text, to_bytes + if six.PY2: return to_bytes(string) return to_text(string) diff --git a/pipenv/vendor/vistir/contextmanagers.py b/pipenv/vendor/vistir/contextmanagers.py index 77fbb9df..49ec964f 100644 --- a/pipenv/vendor/vistir/contextmanagers.py +++ b/pipenv/vendor/vistir/contextmanagers.py @@ -1,11 +1,10 @@ # -*- coding=utf-8 -*- -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals import io import os import stat import sys - from contextlib import contextmanager import six @@ -13,9 +12,16 @@ import six from .compat import NamedTemporaryFile, Path from .path import is_file_url, is_valid_url, path_to_url, url_to_path - __all__ = [ - "temp_environ", "temp_path", "cd", "atomic_open_for_write", "open_file", "spinner" + "temp_environ", + "temp_path", + "cd", + "atomic_open_for_write", + "open_file", + "spinner", + "dummy_spinner", + "replaced_stream", + "replaced_streams", ] @@ -103,7 +109,13 @@ def dummy_spinner(spin_type, text, **kwargs): @contextmanager -def spinner(spinner_name=None, start_text=None, handler_map=None, nospin=False, write_to_stdout=True): +def spinner( + spinner_name=None, + start_text=None, + handler_map=None, + nospin=False, + write_to_stdout=True, +): """Get a spinner object or a dummy spinner to wrap a context. :param str spinner_name: A spinner type e.g. "dots" or "bouncingBar" (default: {"bouncingBar"}) @@ -119,6 +131,7 @@ def spinner(spinner_name=None, start_text=None, handler_map=None, nospin=False, """ from .spin import create_spinner + has_yaspin = None try: import yaspin @@ -145,7 +158,7 @@ def spinner(spinner_name=None, start_text=None, handler_map=None, nospin=False, handler_map=handler_map, nospin=nospin, use_yaspin=use_yaspin, - write_to_stdout=write_to_stdout + write_to_stdout=write_to_stdout, ) as _spinner: yield _spinner @@ -266,8 +279,8 @@ def open_file(link, session=None, stream=True): if os.path.isdir(local_path): raise ValueError("Cannot open directory for read: {}".format(link)) else: - with io.open(local_path, "rb") as local_file: - yield local_file + with io.open(local_path, "rb") as local_file: + yield local_file else: # Remote URL headers = {"Accept-Encoding": "identity"} @@ -286,3 +299,57 @@ def open_file(link, session=None, stream=True): if conn is not None: conn.close() result.close() + + +@contextmanager +def replaced_stream(stream_name): + """ + Context manager to temporarily swap out *stream_name* with a stream wrapper. + + :param str stream_name: The name of a sys stream to wrap + :returns: A ``StreamWrapper`` replacement, temporarily + + >>> orig_stdout = sys.stdout + >>> with replaced_stream("stdout") as stdout: + ... sys.stdout.write("hello") + ... assert stdout.getvalue() == "hello" + + >>> sys.stdout.write("hello") + 'hello' + """ + + orig_stream = getattr(sys, stream_name) + new_stream = six.StringIO() + try: + setattr(sys, stream_name, new_stream) + yield getattr(sys, stream_name) + finally: + setattr(sys, stream_name, orig_stream) + + +@contextmanager +def replaced_streams(): + """ + Context manager to replace both ``sys.stdout`` and ``sys.stderr`` using + ``replaced_stream`` + + returns: *(stdout, stderr)* + + >>> import sys + >>> with vistir.contextmanagers.replaced_streams() as streams: + >>> stdout, stderr = streams + >>> sys.stderr.write("test") + >>> sys.stdout.write("hello") + >>> assert stdout.getvalue() == "hello" + >>> assert stderr.getvalue() == "test" + + >>> stdout.getvalue() + 'hello' + + >>> stderr.getvalue() + 'test' + """ + + with replaced_stream("stdout") as stdout: + with replaced_stream("stderr") as stderr: + yield (stdout, stderr) diff --git a/pipenv/vendor/vistir/cursor.py b/pipenv/vendor/vistir/cursor.py new file mode 100644 index 00000000..bdb281f6 --- /dev/null +++ b/pipenv/vendor/vistir/cursor.py @@ -0,0 +1,61 @@ +# -*- coding=utf-8 -*- +from __future__ import absolute_import, print_function + +import os +import sys + +__all__ = ["hide_cursor", "show_cursor", "get_stream_handle"] + + +def get_stream_handle(stream=sys.stdout): + """ + Get the OS appropriate handle for the corresponding output stream. + + :param str stream: The the stream to get the handle for + :return: A handle to the appropriate stream, either a ctypes buffer + or **sys.stdout** or **sys.stderr**. + """ + handle = stream + if os.name == "nt": + from ._winconsole import get_stream_handle as get_win_stream_handle + + return get_win_stream_handle(stream) + return handle + + +def hide_cursor(stream=sys.stdout): + """ + Hide the console cursor on the given stream + + :param stream: The name of the stream to get the handle for + :return: None + :rtype: None + """ + + handle = get_stream_handle(stream=stream) + if os.name == "nt": + from ._winconsole import hide_cursor + + hide_cursor() + else: + handle.write("\033[?25l") + handle.flush() + + +def show_cursor(stream=sys.stdout): + """ + Show the console cursor on the given stream + + :param stream: The name of the stream to get the handle for + :return: None + :rtype: None + """ + + handle = get_stream_handle(stream=stream) + if os.name == "nt": + from ._winconsole import show_cursor + + show_cursor() + else: + handle.write("\033[?25h") + handle.flush() diff --git a/pipenv/vendor/vistir/environment.py b/pipenv/vendor/vistir/environment.py new file mode 100644 index 00000000..b8490c00 --- /dev/null +++ b/pipenv/vendor/vistir/environment.py @@ -0,0 +1,6 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function + +from .compat import IS_TYPE_CHECKING + +MYPY_RUNNING = IS_TYPE_CHECKING diff --git a/pipenv/vendor/vistir/misc.py b/pipenv/vendor/vistir/misc.py index f7ed26b4..36218a50 100644 --- a/pipenv/vendor/vistir/misc.py +++ b/pipenv/vendor/vistir/misc.py @@ -1,24 +1,37 @@ # -*- coding=utf-8 -*- -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals +import io import json -import logging import locale +import logging import os import subprocess import sys - from collections import OrderedDict from functools import partial -from itertools import islice +from itertools import islice, tee +from weakref import WeakKeyDictionary import six from .cmdparse import Script -from .compat import Path, fs_str, partialmethod, to_native_string +from .compat import ( + Iterable, + Path, + StringIO, + TimeoutError, + fs_str, + is_bytes, + partialmethod, + to_native_string, +) from .contextmanagers import spinner as spinner +from .environment import MYPY_RUNNING +from .termcolors import ANSI_REMOVAL_RE, colorize if os.name != "nt": + class WindowsError(OSError): pass @@ -38,10 +51,19 @@ __all__ = [ "divide", "getpreferredencoding", "decode_for_output", + "get_canonical_encoding_name", + "get_wrapped_stream", + "StreamWrapper", ] +if MYPY_RUNNING: + from typing import Any, Dict, List, Optional, Union + from .spin import VistirSpinner + + def _get_logger(name=None, level="ERROR"): + # type: (Optional[str], str) -> logging.Logger if not name: name = __name__ if isinstance(level, six.string_types): @@ -58,6 +80,7 @@ def _get_logger(name=None, level="ERROR"): def shell_escape(cmd): + # type: (Union[str, List[str]]) -> str """Escape strings for use in :func:`~subprocess.Popen` and :func:`run`. This is a passthrough method for instantiating a :class:`~vistir.cmdparse.Script` @@ -68,6 +91,7 @@ def shell_escape(cmd): def unnest(elem): + # type: (Iterable) -> Any """Flatten an arbitrarily nested iterable :param elem: An iterable to flatten @@ -78,24 +102,31 @@ def unnest(elem): [1234, 3456, 4398345, 234234, 2396, 23895750, 9283798, 29384, 289375983275, 293759, 2347, 2098, 7987, 27599] """ - if _is_iterable(elem): - for item in elem: - if _is_iterable(item): - for sub_item in unnest(item): - yield sub_item - else: - yield item + if isinstance(elem, Iterable) and not isinstance(elem, six.string_types): + elem, target = tee(elem, 2) else: - raise ValueError("Expecting an iterable, got %r" % elem) + target = elem + if not target or not _is_iterable(target): + yield target + else: + for el in target: + if isinstance(el, Iterable) and not isinstance(el, six.string_types): + el, el_copy = tee(el, 2) + for sub in unnest(el_copy): + yield sub + else: + yield el def _is_iterable(elem): - if getattr(elem, "__iter__", False): + # type: (Any) -> bool + if getattr(elem, "__iter__", False) or isinstance(elem, Iterable): return True return False def dedup(iterable): + # type: (Iterable) -> Iterable """Deduplicate an iterable object like iter(set(iterable)) but order-reserved. """ @@ -103,6 +134,7 @@ def dedup(iterable): def _spawn_subprocess(script, env=None, block=True, cwd=None, combine_stderr=True): + # type: (Union[str, List[str]], Optional[Dict[str, str], bool, Optional[str], bool]) -> subprocess.Popen from distutils.spawn import find_executable if not env: @@ -130,7 +162,7 @@ def _spawn_subprocess(script, env=None, block=True, cwd=None, combine_stderr=Tru # a "command" that is non-executable. See pypa/pipenv#2727. try: return subprocess.Popen(cmd, **options) - except WindowsError as e: + except WindowsError as e: # pragma: no cover if getattr(e, "winerror", 9999) != 193: raise options["shell"] = True @@ -138,6 +170,86 @@ def _spawn_subprocess(script, env=None, block=True, cwd=None, combine_stderr=Tru return subprocess.Popen(script.cmdify(), **options) +def _read_streams(stream_dict): + results = {} + for outstream in stream_dict.keys(): + stream = stream_dict[outstream] + if not stream: + results[outstream] = None + continue + line = to_text(stream.readline()) + if not line: + results[outstream] = None + continue + line = to_text("{0}".format(line.rstrip())) + results[outstream] = line + return results + + + +def get_stream_results(cmd_instance, verbose, maxlen, spinner=None, stdout_allowed=False): + stream_results = {"stdout": [], "stderr": []} + streams = {"stderr": cmd_instance.stderr, "stdout": cmd_instance.stdout} + while True: + stream_contents = _read_streams(streams) + stdout_line = stream_contents["stdout"] + stderr_line = stream_contents["stderr"] + if not (stdout_line or stderr_line): + break + last_changed = 0 + display_line = "" + for stream_name in stream_contents.keys(): + if stream_contents[stream_name] and stream_name in stream_results: + line = stream_contents[stream_name] + stream_results[stream_name].append(line) + display_line = ( + fs_str("{0}".format(line)) + if stream_name == "stderr" + else display_line + ) + if display_line and last_changed < 100: + last_changed = 0 + display_line = "" + elif display_line: + last_changed += 1 + if len(display_line) > maxlen: + display_line = "{0}...".format(display_line[:maxlen]) + if verbose: + use_stderr = not stdout_allowed or stream_name != "stdout" + if spinner: + target = spinner.stderr if use_stderr else spinner.stdout + spinner.hide_and_write(display_line, target=target) + else: + target = sys.stderr if use_stderr else sys.stdout + target.write(display_line) + target.flush() + if spinner: + spinner.text = to_native_string( + "{0} {1}".format(spinner.text, display_line) + ) + continue + return stream_results + + +def _handle_nonblocking_subprocess(c, spinner=None): + # type: (subprocess.Popen, VistirSpinner) -> subprocess.Popen + try: + c.wait() + finally: + if c.stdout: + c.stdout.close() + if c.stderr: + c.stderr.close() + if spinner: + if c.returncode > 0: + spinner.fail(to_native_string("Failed...cleaning up...")) + if not os.name == "nt": + spinner.ok(to_native_string("✔ Complete")) + else: + spinner.ok(to_native_string("Complete")) + return c + + def _create_subprocess( cmd, env=None, @@ -149,89 +261,52 @@ def _create_subprocess( combine_stderr=False, display_limit=200, start_text="", - write_to_stdout=True + write_to_stdout=True, ): if not env: env = os.environ.copy() try: - c = _spawn_subprocess(cmd, env=env, block=block, cwd=cwd, - combine_stderr=combine_stderr) + c = _spawn_subprocess( + cmd, env=env, block=block, cwd=cwd, combine_stderr=combine_stderr + ) except Exception as exc: - sys.stderr.write("Error %s while executing command %s", exc, " ".join(cmd._parts)) - raise + import traceback + + formatted_tb = "".join( + traceback.format_exception(*sys.exc_info()) + ) # pragma: no cover + sys.stderr.write( # pragma: no cover + "Error while executing command %s:" + % to_native_string(" ".join(cmd._parts)) # pragma: no cover + ) # pragma: no cover + sys.stderr.write(formatted_tb) # pragma: no cover + raise exc # pragma: no cover if not block: c.stdin.close() - output = [] - err = [] - spinner_orig_text = None - if spinner: - spinner_orig_text = getattr(spinner, "text", None) - if spinner_orig_text is None: - spinner_orig_text = start_text if start_text is not None else "" - streams = { - "stdout": c.stdout, - "stderr": c.stderr - } - while True: - stdout_line = None - stderr_line = None - for outstream in streams.keys(): - stream = streams[outstream] - if not stream: - continue - line = to_text(stream.readline()) - if not line: - continue - line = to_text("{0}".format(line.rstrip())) - if outstream == "stderr": - stderr_line = line - else: - stdout_line = line - if not (stdout_line or stderr_line): - break - if stderr_line is not None: - err.append(stderr_line) - err_line = fs_str("{0}".format(stderr_line)) - if verbose and err_line is not None: - if spinner: - spinner.hide_and_write(err_line, target=spinner.stderr) - else: - sys.stderr.write(err_line) - sys.stderr.flush() - if stdout_line is not None: - output.append(stdout_line) - display_line = fs_str("{0}".format(stdout_line)) - if len(stdout_line) > display_limit: - display_line = "{0}...".format(stdout_line[:display_limit]) - if verbose and display_line is not None: - if spinner: - target = spinner.stdout if write_to_stdout else spinner.stderr - spinner.hide_and_write(display_line, target=target) - else: - target = sys.stdout if write_to_stdout else sys.stderr - target.write(display_line) - target.flush() - if spinner: - spinner.text = to_native_string("{0} {1}".format(spinner_orig_text, display_line)) - continue - try: - c.wait() - finally: - if c.stdout: - c.stdout.close() - if c.stderr: - c.stderr.close() - if spinner: - if c.returncode > 0: - spinner.fail(to_native_string("Failed...cleaning up...")) - if not os.name == "nt": - spinner.ok(to_native_string("✔ Complete")) - else: - spinner.ok(to_native_string("Complete")) + spinner_orig_text = "" + if spinner and getattr(spinner, "text", None) is not None: + spinner_orig_text = spinner.text + if not spinner_orig_text and start_text is not None: + spinner_orig_text = start_text + stream_results = get_stream_results( + c, + verbose=verbose, + maxlen=display_limit, + spinner=spinner, + stdout_allowed=write_to_stdout, + ) + _handle_nonblocking_subprocess(c, spinner) + output = stream_results["stdout"] + err = stream_results["stderr"] c.out = "\n".join(output) if output else "" c.err = "\n".join(err) if err else "" else: - c.out, c.err = c.communicate() + try: + c.out, c.err = c.communicate() + except (SystemExit, KeyboardInterrupt, TimeoutError): + c.terminate() + c.out, c.err = c.communicate() + raise if not block: c.wait() c.out = to_text("{0}".format(c.out)) if c.out else fs_str("") @@ -252,7 +327,7 @@ def run( spinner_name=None, combine_stderr=True, display_limit=200, - write_to_stdout=True + write_to_stdout=True, ): """Use `subprocess.Popen` to get the output of a command and decode it. @@ -277,14 +352,11 @@ def run( _env = os.environ.copy() if env: _env.update(env) - env = _env if six.PY2: fs_encode = partial(to_bytes, encoding=locale_encoding) - _env = {fs_encode(k): fs_encode(v) for k, v in os.environ.items()} - for key, val in env.items(): - _env[fs_encode(key)] = fs_encode(val) + _env = {fs_encode(k): fs_encode(v) for k, v in _env.items()} else: - _env = {k: fs_str(v) for k, v in os.environ.items()} + _env = {k: fs_str(v) for k, v in _env.items()} if not spinner_name: spinner_name = "bouncingBar" if six.PY2: @@ -297,8 +369,12 @@ def run( if block or not return_object: combine_stderr = False start_text = "" - with spinner(spinner_name=spinner_name, start_text=start_text, nospin=nospin, - write_to_stdout=write_to_stdout) as sp: + with spinner( + spinner_name=spinner_name, + start_text=start_text, + nospin=nospin, + write_to_stdout=write_to_stdout, + ) as sp: return _create_subprocess( cmd, env=_env, @@ -309,11 +385,10 @@ def run( spinner=sp, combine_stderr=combine_stderr, start_text=start_text, - write_to_stdout=True + write_to_stdout=write_to_stdout, ) - def load_path(python): """Load the :mod:`sys.path` from the given python executable's environment as json @@ -326,8 +401,9 @@ def load_path(python): """ python = Path(python).as_posix() - out, err = run([python, "-c", "import json, sys; print(json.dumps(sys.path))"], - nospin=True) + out, err = run( + [python, "-c", "import json, sys; print(json.dumps(sys.path))"], nospin=True + ) if out: return json.loads(out) else: @@ -365,14 +441,14 @@ def partialclass(cls, *args, **kwargs): # Swiped from attrs.make_class try: type_.__module__ = sys._getframe(1).f_globals.get("__name__", "__main__") - except (AttributeError, ValueError): - pass + except (AttributeError, ValueError): # pragma: no cover + pass # pragma: no cover return type_ # Borrowed from django -- force bytes and decode -- see link for details: # https://github.com/django/django/blob/fc6b90b/django/utils/encoding.py#L112 -def to_bytes(string, encoding="utf-8", errors="ignore"): +def to_bytes(string, encoding="utf-8", errors=None): """Force a value to bytes. :param string: Some input that can be converted to a bytes. @@ -383,19 +459,23 @@ def to_bytes(string, encoding="utf-8", errors="ignore"): :rtype: bytes """ + unicode_name = get_canonical_encoding_name("utf-8") if not errors: - if encoding.lower() == "utf-8": - errors = "surrogateescape" if six.PY3 else "ignore" + if get_canonical_encoding_name(encoding) == unicode_name: + if six.PY3 and os.name == "nt": + errors = "surrogatepass" + else: + errors = "surrogateescape" if six.PY3 else "ignore" else: errors = "strict" if isinstance(string, bytes): - if encoding.lower() == "utf-8": + if get_canonical_encoding_name(encoding) == unicode_name: return string else: - return string.decode("utf-8").encode(encoding, errors) + return string.decode(unicode_name).encode(encoding, errors) elif isinstance(string, memoryview): - return bytes(string) - elif not isinstance(string, six.string_types): + return string.tobytes() + elif not isinstance(string, six.string_types): # pragma: no cover try: if six.PY3: return six.text_type(string).encode(encoding, errors) @@ -420,9 +500,13 @@ def to_text(string, encoding="utf-8", errors=None): :rtype: str """ + unicode_name = get_canonical_encoding_name("utf-8") if not errors: - if encoding.lower() == "utf-8": - errors = "surrogateescape" if six.PY3 else "ignore" + if get_canonical_encoding_name(encoding) == unicode_name: + if six.PY3 and os.name == "nt": + errors = "surrogatepass" + else: + errors = "surrogateescape" if six.PY3 else "ignore" else: errors = "strict" if issubclass(type(string), six.text_type): @@ -434,13 +518,13 @@ def to_text(string, encoding="utf-8", errors=None): string = six.text_type(string, encoding, errors) else: string = six.text_type(string) - elif hasattr(string, "__unicode__"): + elif hasattr(string, "__unicode__"): # pragma: no cover string = six.text_type(string) else: string = six.text_type(bytes(string), encoding, errors) else: string = string.decode(encoding, errors) - except UnicodeDecodeError as e: + except UnicodeDecodeError: # pragma: no cover string = " ".join(to_text(arg, encoding, errors) for arg in string) return string @@ -492,7 +576,7 @@ def chunked(n, iterable): try: - locale_encoding = locale.getdefaultencoding()[1] or "ascii" + locale_encoding = locale.getdefaultlocale()[1] or "ascii" except Exception: locale_encoding = "ascii" @@ -513,19 +597,441 @@ def getpreferredencoding(): PREFERRED_ENCODING = getpreferredencoding() -def decode_for_output(output): +def get_output_encoding(source_encoding): + """ + Given a source encoding, determine the preferred output encoding. + + :param str source_encoding: The encoding of the source material. + :returns: The output encoding to decode to. + :rtype: str + """ + + if source_encoding is not None: + if get_canonical_encoding_name(source_encoding) == "ascii": + return "utf-8" + return get_canonical_encoding_name(source_encoding) + return get_canonical_encoding_name(PREFERRED_ENCODING) + + +def _encode(output, encoding=None, errors=None, translation_map=None): + if encoding is None: + encoding = PREFERRED_ENCODING + try: + output = output.encode(encoding) + except (UnicodeDecodeError, UnicodeEncodeError): + if translation_map is not None: + if six.PY2: + output = unicode.translate( # noqa: F821 + to_text(output, encoding=encoding, errors=errors), translation_map + ) + else: + output = output.translate(translation_map) + else: + output = to_text(output, encoding=encoding, errors=errors) + except AttributeError: + pass + return output + + +def decode_for_output(output, target_stream=None, translation_map=None): """Given a string, decode it for output to a terminal :param str output: A string to print to a terminal + :param target_stream: A stream to write to, we will encode to target this stream if possible. + :param dict translation_map: A mapping of unicode character ordinals to replacement strings. :return: A re-encoded string using the preferred encoding :rtype: str """ if not isinstance(output, six.string_types): return output + encoding = None + if target_stream is not None: + encoding = getattr(target_stream, "encoding", None) + encoding = get_output_encoding(encoding) try: - output = output.encode(PREFERRED_ENCODING) - except AttributeError: - pass - output = output.decode(PREFERRED_ENCODING) - return output + output = _encode(output, encoding=encoding, translation_map=translation_map) + except (UnicodeDecodeError, UnicodeEncodeError): + output = to_native_string(output) + output = _encode( + output, encoding=encoding, errors="replace", translation_map=translation_map + ) + return to_text(output, encoding=encoding, errors="replace") + + +def get_canonical_encoding_name(name): + # type: (str) -> str + """ + Given an encoding name, get the canonical name from a codec lookup. + + :param str name: The name of the codec to lookup + :return: The canonical version of the codec name + :rtype: str + """ + + import codecs + + try: + codec = codecs.lookup(name) + except LookupError: + return name + else: + return codec.name + + +def _is_binary_buffer(stream): + try: + stream.write(b"") + except Exception: + try: + stream.write("") + except Exception: + pass + return False + return True + + +def _get_binary_buffer(stream): + if six.PY3 and not _is_binary_buffer(stream): + stream = getattr(stream, "buffer", None) + if stream is not None and _is_binary_buffer(stream): + return stream + return stream + + +def get_wrapped_stream(stream, encoding=None, errors="replace"): + """ + Given a stream, wrap it in a `StreamWrapper` instance and return the wrapped stream. + + :param stream: A stream instance to wrap + :param str encoding: The encoding to use for the stream + :param str errors: The error handler to use, default "replace" + :returns: A new, wrapped stream + :rtype: :class:`StreamWrapper` + """ + + if stream is None: + raise TypeError("must provide a stream to wrap") + stream = _get_binary_buffer(stream) + if stream is not None and encoding is None: + encoding = "utf-8" + if not encoding: + encoding = get_output_encoding(stream) + else: + encoding = get_canonical_encoding_name(encoding) + return StreamWrapper(stream, encoding, errors, line_buffering=True) + + +class StreamWrapper(io.TextIOWrapper): + + """ + This wrapper class will wrap a provided stream and supply an interface + for compatibility. + """ + + def __init__(self, stream, encoding, errors, line_buffering=True, **kwargs): + self._stream = stream = _StreamProvider(stream) + io.TextIOWrapper.__init__( + self, stream, encoding, errors, line_buffering=line_buffering, **kwargs + ) + + # borrowed from click's implementation of stream wrappers, see + # https://github.com/pallets/click/blob/6cafd32/click/_compat.py#L64 + if six.PY2: + + def write(self, x): + if isinstance(x, (str, buffer, bytearray)): # noqa: F821 + try: + self.flush() + except Exception: + pass + # This is modified from the initial implementation to rely on + # our own decoding functionality to preserve unicode strings where + # possible + return self.buffer.write(str(x)) + return io.TextIOWrapper.write(self, x) + + else: + + def write(self, x): + # try to use backslash and surrogate escape strategies before failing + self._errors = ( + "backslashescape" if self.encoding != "mbcs" else "surrogateescape" + ) + try: + return io.TextIOWrapper.write(self, to_text(x, errors=self._errors)) + except UnicodeDecodeError: + if self._errors != "surrogateescape": + self._errors = "surrogateescape" + else: + self._errors = "replace" + return io.TextIOWrapper.write(self, to_text(x, errors=self._errors)) + + def writelines(self, lines): + for line in lines: + self.write(line) + + def __del__(self): + try: + self.detach() + except Exception: + pass + + def isatty(self): + return self._stream.isatty() + + +# More things borrowed from click, this is because we are using `TextIOWrapper` instead of +# just a normal StringIO +class _StreamProvider(object): + def __init__(self, stream): + self._stream = stream + super(_StreamProvider, self).__init__() + + def __getattr__(self, name): + return getattr(self._stream, name) + + def read1(self, size): + fn = getattr(self._stream, "read1", None) + if fn is not None: + return fn(size) + if six.PY2: + return self._stream.readline(size) + return self._stream.read(size) + + def readable(self): + fn = getattr(self._stream, "readable", None) + if fn is not None: + return fn() + try: + self._stream.read(0) + except Exception: + return False + return True + + def writable(self): + fn = getattr(self._stream, "writable", None) + if fn is not None: + return fn() + try: + self._stream.write(b"") + except Exception: + return False + return True + + def seekable(self): + fn = getattr(self._stream, "seekable", None) + if fn is not None: + return fn() + try: + self._stream.seek(self._stream.tell()) + except Exception: + return False + return True + + +# XXX: The approach here is inspired somewhat by click with details taken from various +# XXX: other sources. Specifically we are using a stream cache and stream wrapping +# XXX: techniques from click (loosely inspired for the most part, with many details) +# XXX: heavily modified to suit our needs + + +def _isatty(stream): + try: + is_a_tty = stream.isatty() + except Exception: # pragma: no cover + is_a_tty = False + return is_a_tty + + +_wrap_for_color = None + +try: + import colorama +except ImportError: + colorama = None + +_color_stream_cache = WeakKeyDictionary() + +if os.name == "nt" or sys.platform.startswith("win"): + + if colorama is not None: + + def _is_wrapped_for_color(stream): + return isinstance(stream, (colorama.AnsiToWin32, colorama.ansitowin32.StreamWrapper)) + + def _wrap_for_color(stream, color=None): + try: + cached = _color_stream_cache.get(stream) + except KeyError: + cached = None + if cached is not None: + return cached + strip = not _can_use_color(stream, color) + _color_wrapper = colorama.AnsiToWin32(stream, strip=strip) + result = _color_wrapper.stream + _write = result.write + + def _write_with_color(s): + try: + return _write(s) + except Exception: + _color_wrapper.reset_all() + raise + + result.write = _write_with_color + try: + _color_stream_cache[stream] = result + except Exception: + pass + return result + + +def _cached_stream_lookup(stream_lookup_func, stream_resolution_func): + stream_cache = WeakKeyDictionary() + + def lookup(): + stream = stream_lookup_func() + result = None + if stream in stream_cache: + result = stream_cache.get(stream, None) + if result is not None: + return result + result = stream_resolution_func() + try: + stream = stream_lookup_func() + stream_cache[stream] = result + except Exception: + pass + return result + + return lookup + + +def get_text_stream(stream="stdout", encoding=None): + """Retrieve a unicode stream wrapper around **sys.stdout** or **sys.stderr**. + + :param str stream: The name of the stream to wrap from the :mod:`sys` module. + :param str encoding: An optional encoding to use. + :return: A new :class:`~vistir.misc.StreamWrapper` instance around the stream + :rtype: `vistir.misc.StreamWrapper` + """ + + stream_map = {"stdin": sys.stdin, "stdout": sys.stdout, "stderr": sys.stderr} + if os.name == "nt" or sys.platform.startswith("win"): + from ._winconsole import _get_windows_console_stream, _wrap_std_stream + + else: + _get_windows_console_stream = lambda *args: None # noqa + _wrap_std_stream = lambda *args: None # noqa + + if six.PY2 and stream != "stdin": + _wrap_std_stream(stream) + sys_stream = stream_map[stream] + windows_console = _get_windows_console_stream(sys_stream, encoding, None) + if windows_console is not None: + if _can_use_color(windows_console): + return _wrap_for_color(windows_console) + return windows_console + return get_wrapped_stream(sys_stream, encoding) + + +def get_text_stdout(): + return get_text_stream("stdout") + + +def get_text_stderr(): + return get_text_stream("stderr") + + +def get_text_stdin(): + return get_text_stream("stdin") + + +_text_stdin = _cached_stream_lookup(lambda: sys.stdin, get_text_stdin) +_text_stdout = _cached_stream_lookup(lambda: sys.stdout, get_text_stdout) +_text_stderr = _cached_stream_lookup(lambda: sys.stderr, get_text_stderr) + + +TEXT_STREAMS = { + "stdin": get_text_stdin, + "stdout": get_text_stdout, + "stderr": get_text_stderr, +} + + +def replace_with_text_stream(stream_name): + """Given a stream name, replace the target stream with a text-converted equivalent + + :param str stream_name: The name of a target stream, such as **stdout** or **stderr** + :return: None + """ + new_stream = TEXT_STREAMS.get(stream_name) + if new_stream is not None: + new_stream = new_stream() + setattr(sys, stream_name, new_stream) + return None + + +def _can_use_color(stream=None, color=None): + from .termcolors import DISABLE_COLORS + + if DISABLE_COLORS: + return False + if not color: + if not stream: + stream = sys.stdin + return _isatty(stream) + return bool(color) + + +def echo(text, fg=None, bg=None, style=None, file=None, err=False, color=None): + """Write the given text to the provided stream or **sys.stdout** by default. + + Provides optional foreground and background colors from the ansi defaults: + **grey**, **red**, **green**, **yellow**, **blue**, **magenta**, **cyan** + or **white**. + + Available styles include **bold**, **dark**, **underline**, **blink**, **reverse**, + **concealed** + + :param str text: Text to write + :param str fg: Foreground color to use (default: None) + :param str bg: Foreground color to use (default: None) + :param str style: Style to use (default: None) + :param stream file: File to write to (default: None) + :param bool color: Whether to force color (i.e. ANSI codes are in the text) + """ + + if file and not hasattr(file, "write"): + raise TypeError("Expected a writable stream, received {0!r}".format(file)) + if not file: + if err: + file = _text_stderr() + else: + file = _text_stdout() + if text and not isinstance(text, (six.string_types, bytes, bytearray)): + text = six.text_type(text) + text = "" if not text else text + if isinstance(text, six.text_type): + text += "\n" + else: + text += b"\n" + if text and six.PY3 and is_bytes(text): + buffer = _get_binary_buffer(file) + if buffer is not None: + file.flush() + buffer.write(text) + buffer.flush() + return + if text and not is_bytes(text): + can_use_color = _can_use_color(file, color=color) + if any([fg, bg, style]): + text = colorize(text, fg=fg, bg=bg, attrs=style) + if not can_use_color or (os.name == "nt" and not _wrap_for_color): + text = ANSI_REMOVAL_RE.sub("", text) + elif os.name == "nt" and _wrap_for_color and not _is_wrapped_for_color(file): + file = _wrap_for_color(file, color=color) + if text: + file.write(text) + file.flush() diff --git a/pipenv/vendor/vistir/path.py b/pipenv/vendor/vistir/path.py index febaddbc..d5b02f64 100644 --- a/pipenv/vendor/vistir/path.py +++ b/pipenv/vendor/vistir/path.py @@ -1,5 +1,5 @@ # -*- coding=utf-8 -*- -from __future__ import absolute_import, unicode_literals, print_function +from __future__ import absolute_import, print_function, unicode_literals import atexit import errno @@ -8,28 +8,37 @@ import os import posixpath import shutil import stat +import time import warnings import six - from six.moves import urllib_parse from six.moves.urllib import request as urllib_request from .backports.tempfile import _TemporaryFileWrapper from .compat import ( - _NamedTemporaryFile, + IS_TYPE_CHECKING, + FileNotFoundError, Path, + PermissionError, ResourceWarning, TemporaryDirectory, _fs_encoding, + _NamedTemporaryFile, finalize, + fs_decode, + fs_encode, ) +if IS_TYPE_CHECKING: + from typing import Optional, Callable, Text, ByteString, AnyStr __all__ = [ "check_for_unc_path", "get_converted_relative_path", "handle_remove_readonly", + "normalize_path", + "is_in_path", "is_file_url", "is_readonly_path", "is_valid_url", @@ -47,7 +56,11 @@ __all__ = [ if os.name == "nt": - warnings.filterwarnings("ignore", category=DeprecationWarning, message="The Windows bytes API has been deprecated.*") + warnings.filterwarnings( + "ignore", + category=DeprecationWarning, + message="The Windows bytes API has been deprecated.*", + ) def unicode_path(path): @@ -80,7 +93,41 @@ else: return os.path.normpath(path) +def normalize_path(path): + # type: (AnyStr) -> AnyStr + """ + Return a case-normalized absolute variable-expanded path. + + :param str path: The non-normalized path + :return: A normalized, expanded, case-normalized path + :rtype: str + """ + + path = os.path.abspath(os.path.expandvars(os.path.expanduser(str(path)))) + if os.name == "nt" and os.path.exists(path): + from ._winconsole import get_long_path + + path = get_long_path(path) + + return os.path.normpath(os.path.normcase(path)) + + +def is_in_path(path, parent): + # type: (AnyStr, AnyStr) -> bool + """ + Determine if the provided full path is in the given parent root. + + :param str path: The full path to check the location of. + :param str parent: The parent path to check for membership in + :return: Whether the full path is a member of the provided parent. + :rtype: bool + """ + + return normalize_path(str(path)).startswith(normalize_path(str(parent))) + + def normalize_drive(path): + # type: (str) -> Text """Normalize drive in path so they stay consistent. This currently only affects local drives on Windows, which can be @@ -101,6 +148,7 @@ def normalize_drive(path): def path_to_url(path): + # type: (str) -> Text """Convert the supplied local path to a file uri. :param str path: A string pointing to or representing a local path @@ -120,7 +168,9 @@ def path_to_url(path): def url_to_path(url): - """Convert a valid file url to a local filesystem path + # type: (str) -> ByteString + """ + Convert a valid file url to a local filesystem path Follows logic taken from pip's equivalent function """ @@ -166,9 +216,8 @@ def is_readonly_path(fn): Permissions check is `bool(path.stat & stat.S_IREAD)` or `not os.access(path, os.W_OK)` """ - from .compat import to_native_string - fn = to_native_string(fn) + fn = fs_encode(fn) if os.path.exists(fn): file_stat = os.stat(fn).st_mode return not bool(file_stat & stat.S_IWRITE) or not os.access(fn, os.W_OK) @@ -183,20 +232,19 @@ def mkdir_p(newdir, mode=0o777): :raises: OSError if a file is encountered along the way """ # http://code.activestate.com/recipes/82465-a-friendly-mkdir/ - from .misc import to_bytes, to_text - newdir = to_bytes(newdir, "utf-8") + newdir = fs_encode(newdir) if os.path.exists(newdir): if not os.path.isdir(newdir): raise OSError( "a file with the same name as the desired dir, '{0}', already exists.".format( - newdir + fs_decode(newdir) ) ) else: - head, tail = os.path.split(to_bytes(newdir, encoding="utf-8")) + head, tail = os.path.split(newdir) # Make sure the tail doesn't point to the asame place as the head - curdir = to_bytes(".", encoding="utf-8") + curdir = fs_encode(".") tail_and_head_match = ( os.path.relpath(tail, start=os.path.basename(head)) == curdir ) @@ -204,8 +252,8 @@ def mkdir_p(newdir, mode=0o777): target = os.path.join(head, tail) if os.path.exists(target) and os.path.isfile(target): raise OSError( - "A file with the same name as the desired dir, '{0}', already exists.".format( - to_text(newdir, encoding="utf-8") + "A file with the same name as the desired dir, '{0}', already exists.".format( + fs_decode(newdir) ) ) os.makedirs(os.path.join(head, tail), mode) @@ -260,34 +308,77 @@ def create_tracked_tempfile(*args, **kwargs): return _NamedTemporaryFile(*args, **kwargs) +def _find_icacls_exe(): + if os.name == "nt": + paths = [ + os.path.expandvars(r"%windir%\{0}").format(subdir) + for subdir in ("system32", "SysWOW64") + ] + for path in paths: + icacls_path = next( + iter(fn for fn in os.listdir(path) if fn.lower() == "icacls.exe"), None + ) + if icacls_path is not None: + icacls_path = os.path.join(path, icacls_path) + return icacls_path + return None + + def set_write_bit(fn): - """Set read-write permissions for the current user on the target path. Fail silently + # type: (str) -> None + """ + Set read-write permissions for the current user on the target path. Fail silently if the path doesn't exist. :param str fn: The target filename or path + :return: None """ - from .compat import to_native_string - - fn = to_native_string(fn) + fn = fs_encode(fn) if not os.path.exists(fn): return file_stat = os.stat(fn).st_mode os.chmod(fn, file_stat | stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + if os.name == "nt": + from ._winconsole import get_current_user + + user_sid = get_current_user() + icacls_exe = _find_icacls_exe() or "icacls" + from .misc import run + + if user_sid: + c = run( + [ + icacls_exe, + "''{0}''".format(fn), + "/grant", + "{0}:WD".format(user_sid), + "/T", + "/C", + "/Q", + ], nospin=True, return_object=True + ) + if not c.err and c.returncode == 0: + return + if not os.path.isdir(fn): - try: - os.chflags(fn, 0) - except AttributeError: - pass + for path in [fn, os.path.dirname(fn)]: + try: + os.chflags(path, 0) + except AttributeError: + pass + return None for root, dirs, files in os.walk(fn, topdown=False): - for dir_ in [os.path.join(root,d) for d in dirs]: + for dir_ in [os.path.join(root, d) for d in dirs]: set_write_bit(dir_) for file_ in [os.path.join(root, f) for f in files]: set_write_bit(file_) def rmtree(directory, ignore_errors=False, onerror=None): - """Stand-in for :func:`~shutil.rmtree` with additional error-handling. + # type: (str, bool, Optional[Callable]) -> None + """ + Stand-in for :func:`~shutil.rmtree` with additional error-handling. This version of `rmtree` handles read-only paths, especially in the case of index files written by certain source control systems. @@ -301,20 +392,48 @@ def rmtree(directory, ignore_errors=False, onerror=None): Setting `ignore_errors=True` may cause this to silently fail to delete the path """ - from .compat import to_native_string - - directory = to_native_string(directory) + directory = fs_encode(directory) if onerror is None: onerror = handle_remove_readonly try: - shutil.rmtree( - directory, ignore_errors=ignore_errors, onerror=onerror - ) - except (IOError, OSError, FileNotFoundError) as exc: + shutil.rmtree(directory, ignore_errors=ignore_errors, onerror=onerror) + except (IOError, OSError, FileNotFoundError, PermissionError) as exc: # Ignore removal failures where the file doesn't exist - if exc.errno == errno.ENOENT: - pass - raise + if exc.errno != errno.ENOENT: + raise + + +def _wait_for_files(path): # pragma: no cover + """ + Retry with backoff up to 1 second to delete files from a directory. + + :param str path: The path to crawl to delete files from + :return: A list of remaining paths or None + :rtype: Optional[List[str]] + """ + timeout = 0.001 + remaining = [] + while timeout < 1.0: + remaining = [] + if os.path.isdir(path): + L = os.listdir(path) + for target in L: + _remaining = _wait_for_files(target) + if _remaining: + remaining.extend(_remaining) + continue + try: + os.unlink(path) + except FileNotFoundError as e: + if e.errno == errno.ENOENT: + return + except (OSError, IOError, PermissionError): + time.sleep(timeout) + timeout *= 2 + remaining.append(path) + else: + return + return remaining def handle_remove_readonly(func, path, exc): @@ -331,45 +450,45 @@ def handle_remove_readonly(func, path, exc): :func:`set_write_bit` on the target path and try again. """ # Check for read-only attribute - from .compat import ( - ResourceWarning, FileNotFoundError, PermissionError, to_native_string - ) + from .compat import ResourceWarning, FileNotFoundError, PermissionError PERM_ERRORS = (errno.EACCES, errno.EPERM, errno.ENOENT) - default_warning_message = ( - "Unable to remove file due to permissions restriction: {!r}" - ) + default_warning_message = "Unable to remove file due to permissions restriction: {!r}" # split the initial exception out into its type, exception, and traceback exc_type, exc_exception, exc_tb = exc - path = to_native_string(path) if is_readonly_path(path): # Apply write permission and call original function set_write_bit(path) try: func(path) - except (OSError, IOError, FileNotFoundError, PermissionError) as e: - if e.errno == errno.ENOENT: - return - elif e.errno in PERM_ERRORS: - warnings.warn(default_warning_message.format(path), ResourceWarning) + except ( + OSError, + IOError, + FileNotFoundError, + PermissionError, + ) as e: # pragma: no cover + if e.errno in PERM_ERRORS: + if e.errno == errno.ENOENT: + return + remaining = None + if os.path.isdir(path): + remaining = _wait_for_files(path) + if remaining: + warnings.warn(default_warning_message.format(path), ResourceWarning) + else: + func(path, ignore_errors=True) return if exc_exception.errno in PERM_ERRORS: set_write_bit(path) + remaining = _wait_for_files(path) try: func(path) except (OSError, IOError, FileNotFoundError, PermissionError) as e: if e.errno in PERM_ERRORS: - warnings.warn(default_warning_message.format(path), ResourceWarning) - pass - elif e.errno == errno.ENOENT: # File already gone - pass - else: - raise - else: + if e.errno != errno.ENOENT: # File still exists + warnings.warn(default_warning_message.format(path), ResourceWarning) return - elif exc_exception.errno == errno.ENOENT: - pass else: raise exc_exception @@ -465,8 +584,8 @@ def get_converted_relative_path(path, relative_to=None): raise ValueError("The path argument does not currently accept UNC paths") relpath_s = to_text(posixpath.normpath(path.as_posix())) - if not (relpath_s == u"." or relpath_s.startswith(u"./")): - relpath_s = posixpath.join(u".", relpath_s) + if not (relpath_s == "." or relpath_s.startswith("./")): + relpath_s = posixpath.join(".", relpath_s) return relpath_s diff --git a/pipenv/vendor/vistir/spin.py b/pipenv/vendor/vistir/spin.py index 5e975b6f..64b615de 100644 --- a/pipenv/vendor/vistir/spin.py +++ b/pipenv/vendor/vistir/spin.py @@ -1,4 +1,5 @@ # -*- coding=utf-8 -*- +from __future__ import absolute_import, print_function import functools import os @@ -6,41 +7,73 @@ import signal import sys import threading import time +from io import StringIO import colorama -import cursor import six from .compat import to_native_string -from .termcolors import COLOR_MAP, COLORS, colored -from io import StringIO +from .cursor import hide_cursor, show_cursor +from .misc import decode_for_output, to_text +from .termcolors import COLOR_MAP, COLORS, DISABLE_COLORS, colored try: import yaspin -except ImportError: +except ImportError: # pragma: no cover yaspin = None Spinners = None -else: - from yaspin.spinners import Spinners + SpinBase = None +else: # pragma: no cover + import yaspin.spinners + import yaspin.core + + Spinners = yaspin.spinners.Spinners + SpinBase = yaspin.core.Yaspin + +if os.name == "nt": # pragma: no cover + + def handler(signum, frame, spinner): + """Signal handler, used to gracefully shut down the ``spinner`` instance + when specified signal is received by the process running the ``spinner``. + + ``signum`` and ``frame`` are mandatory arguments. Check ``signal.signal`` + function for more details. + """ + spinner.fail() + spinner.stop() + + +else: # pragma: no cover + + def handler(signum, frame, spinner): + """Signal handler, used to gracefully shut down the ``spinner`` instance + when specified signal is received by the process running the ``spinner``. + + ``signum`` and ``frame`` are mandatory arguments. Check ``signal.signal`` + function for more details. + """ + spinner.red.fail("✘") + spinner.stop() -handler = None -if yaspin and os.name == "nt": - handler = yaspin.signal_handlers.default_handler -elif yaspin and os.name != "nt": - handler = yaspin.signal_handlers.fancy_handler CLEAR_LINE = chr(27) + "[K" +TRANSLATION_MAP = {10004: u"OK", 10008: u"x"} + + +decode_output = functools.partial(decode_for_output, translation_map=TRANSLATION_MAP) + class DummySpinner(object): def __init__(self, text="", **kwargs): - colorama.init() - from .misc import decode_for_output - self.text = to_native_string(decode_for_output(text)) if text else "" + if DISABLE_COLORS: + colorama.init() + self.text = to_native_string(decode_output(text)) if text else "" self.stdout = kwargs.get("stdout", sys.stdout) self.stderr = kwargs.get("stderr", sys.stderr) self.out_buff = StringIO() self.write_to_stdout = kwargs.get("write_to_stdout", False) + super(DummySpinner, self).__init__() def __enter__(self): if self.text and self.text != "None": @@ -48,15 +81,16 @@ class DummySpinner(object): self.write(self.text) return self - def __exit__(self, exc_type, exc_val, traceback): + def __exit__(self, exc_type, exc_val, tb): if exc_type: import traceback - from .misc import decode_for_output - self.write_err(decode_for_output(traceback.format_exception(*sys.exc_info()))) + + formatted_tb = traceback.format_exception(exc_type, exc_val, tb) + self.write_err("".join(formatted_tb)) self._close_output_buffer() return False - def __getattr__(self, k): + def __getattr__(self, k): # pragma: no cover try: retval = super(DummySpinner, self).__getattribute__(k) except AttributeError: @@ -74,56 +108,63 @@ class DummySpinner(object): pass def fail(self, exitcode=1, text="FAIL"): - from .misc import decode_for_output - if text and text != "None": + if text is not None and text != "None": if self.write_to_stdout: - self.write(decode_for_output(text)) + self.write(text) else: - self.write_err(decode_for_output(text)) + self.write_err(text) self._close_output_buffer() def ok(self, text="OK"): - if text and text != "None": + if text is not None and text != "None": if self.write_to_stdout: - self.stdout.write(self.text) + self.write(text) else: - self.stderr.write(self.text) + self.write_err(text) self._close_output_buffer() return 0 def hide_and_write(self, text, target=None): if not target: target = self.stdout - from .misc import decode_for_output if text is None or isinstance(text, six.string_types) and text == "None": pass - target.write(decode_for_output("\r")) + target.write(decode_output(u"\r", target_stream=target)) self._hide_cursor(target=target) - target.write(decode_for_output("{0}\n".format(text))) + target.write(decode_output(u"{0}\n".format(text), target_stream=target)) target.write(CLEAR_LINE) self._show_cursor(target=target) def write(self, text=None): if not self.write_to_stdout: return self.write_err(text) - from .misc import decode_for_output if text is None or isinstance(text, six.string_types) and text == "None": pass - text = decode_for_output(text) - self.stdout.write(decode_for_output("\r")) - line = decode_for_output("{0}\n".format(text)) - self.stdout.write(line) - self.stdout.write(CLEAR_LINE) + if not self.stdout.closed: + stdout = self.stdout + else: + stdout = sys.stdout + stdout.write(decode_output(u"\r", target_stream=stdout)) + text = to_text(text) + line = decode_output(u"{0}\n".format(text), target_stream=stdout) + stdout.write(line) + stdout.write(CLEAR_LINE) def write_err(self, text=None): - from .misc import decode_for_output if text is None or isinstance(text, six.string_types) and text == "None": pass - text = decode_for_output(text) - self.stderr.write(decode_for_output("\r")) - line = decode_for_output("{0}\n".format(text)) - self.stderr.write(line) - self.stderr.write(CLEAR_LINE) + text = to_text(text) + if not self.stderr.closed: + stderr = self.stderr + else: + if sys.stderr.closed: + print(text) + return + stderr = sys.stderr + stderr.write(decode_output(u"\r", target_stream=stderr)) + line = decode_output(u"{0}\n".format(text), target_stream=stderr) + stderr.write(line) + stderr.write(CLEAR_LINE) @staticmethod def _hide_cursor(target=None): @@ -134,10 +175,11 @@ class DummySpinner(object): pass -base_obj = yaspin.core.Yaspin if yaspin is not None else DummySpinner +if SpinBase is None: + SpinBase = DummySpinner -class VistirSpinner(base_obj): +class VistirSpinner(SpinBase): "A spinner class for handling spinners on windows and posix." def __init__(self, *args, **kwargs): @@ -156,10 +198,7 @@ class VistirSpinner(base_obj): colorama.init() sigmap = {} if handler: - sigmap.update({ - signal.SIGINT: handler, - signal.SIGTERM: handler - }) + sigmap.update({signal.SIGINT: handler, signal.SIGTERM: handler}) handler_map = kwargs.pop("handler_map", {}) if os.name == "nt": sigmap[signal.SIGBREAK] = handler @@ -180,60 +219,64 @@ class VistirSpinner(base_obj): self.write_to_stdout = write_to_stdout self.is_dummy = bool(yaspin is None) super(VistirSpinner, self).__init__(*args, **kwargs) + if DISABLE_COLORS: + colorama.deinit() - def ok(self, text="OK", err=False): + def ok(self, text=u"OK", err=False): """Set Ok (success) finalizer to a spinner.""" # Do not display spin text for ok state self._text = None - _text = text if text else "OK" + _text = to_text(text) if text else u"OK" err = err or not self.write_to_stdout self._freeze(_text, err=err) - def fail(self, text="FAIL", err=False): + def fail(self, text=u"FAIL", err=False): """Set fail finalizer to a spinner.""" # Do not display spin text for fail state self._text = None - _text = text if text else "FAIL" + _text = text if text else u"FAIL" err = err or not self.write_to_stdout self._freeze(_text, err=err) def hide_and_write(self, text, target=None): if not target: target = self.stdout - from .misc import decode_for_output - if text is None or isinstance(text, six.string_types) and text == "None": + if text is None or isinstance(text, six.string_types) and text == u"None": pass - target.write(decode_for_output("\r")) + target.write(decode_output(u"\r")) self._hide_cursor(target=target) - target.write(decode_for_output("{0}\n".format(text))) + target.write(decode_output(u"{0}\n".format(text))) target.write(CLEAR_LINE) self._show_cursor(target=target) - def write(self, text): + def write(self, text): # pragma: no cover if not self.write_to_stdout: return self.write_err(text) - from .misc import to_text - sys.stdout.write("\r") - self.stdout.write(CLEAR_LINE) + stdout = self.stdout + if self.stdout.closed: + stdout = sys.stdout + stdout.write(decode_output(u"\r", target_stream=stdout)) + stdout.write(decode_output(CLEAR_LINE, target_stream=stdout)) if text is None: text = "" - text = to_native_string("{0}\n".format(text)) - self.stdout.write(text) - self.out_buff.write(to_text(text)) + text = decode_output(u"{0}\n".format(text), target_stream=stdout) + stdout.write(text) + self.out_buff.write(text) - def write_err(self, text): + def write_err(self, text): # pragma: no cover """Write error text in the terminal without breaking the spinner.""" - from .misc import to_text - - self.stderr.write("\r") - self.stderr.write(CLEAR_LINE) + stderr = self.stderr + if self.stderr.closed: + stderr = sys.stderr + stderr.write(decode_output(u"\r", target_stream=stderr)) + stderr.write(decode_output(CLEAR_LINE, target_stream=stderr)) if text is None: text = "" - text = to_native_string("{0}\n".format(text)) + text = decode_output(u"{0}\n".format(text), target_stream=stderr) self.stderr.write(text) - self.out_buff.write(to_text(text)) + self.out_buff.write(decode_output(text, target_stream=self.out_buff)) def start(self): if self._sigmap: @@ -268,52 +311,47 @@ class VistirSpinner(base_obj): if target.isatty(): self._show_cursor(target=target) - if self.stderr and self.stderr != sys.stderr: - self.stderr.close() - if self.stdout and self.stdout != sys.stdout: - self.stdout.close() self.out_buff.close() def _freeze(self, final_text, err=False): """Stop spinner, compose last frame and 'freeze' it.""" if not final_text: final_text = "" - text = to_native_string(final_text) - self._last_frame = self._compose_out(text, mode="last") + target = self.stderr if err else self.stdout + if target.closed: + target = sys.stderr if err else sys.stdout + text = to_text(final_text) + last_frame = self._compose_out(text, mode="last") + self._last_frame = decode_output(last_frame, target_stream=target) # Should be stopped here, otherwise prints after # self._freeze call will mess up the spinner self.stop() - if err or not self.write_to_stdout: - self.stderr.write(self._last_frame) - else: - self.stdout.write(self._last_frame) + target.write(self._last_frame) def _compose_color_func(self): fn = functools.partial( - colored, - color=self._color, - on_color=self._on_color, - attrs=list(self._attrs), + colored, color=self._color, on_color=self._on_color, attrs=list(self._attrs) ) return fn def _compose_out(self, frame, mode=None): # Ensure Unicode input - frame = to_native_string(frame) + frame = to_text(frame) if self._text is None: - self._text = "" - text = to_native_string(self._text) + self._text = u"" + text = to_text(self._text) if self._color_func is not None: frame = self._color_func(frame) if self._side == "right": frame, text = text, frame # Mode + frame = to_text(frame) if not mode: - out = to_native_string("\r{0} {1}".format(frame, text)) + out = u"\r{0} {1}".format(frame, text) else: - out = to_native_string("{0} {1}\n".format(frame, text)) + out = u"{0} {1}\n".format(frame, text) return out def _spin(self): @@ -329,6 +367,7 @@ class VistirSpinner(base_obj): # Compose output spin_phase = next(self._cycle) out = self._compose_out(spin_phase) + out = decode_output(out, target) # Write target.write(out) @@ -379,13 +418,13 @@ class VistirSpinner(base_obj): def _hide_cursor(target=None): if not target: target = sys.stdout - cursor.hide(stream=target) + hide_cursor(stream=target) @staticmethod def _show_cursor(target=None): if not target: target = sys.stdout - cursor.show(stream=target) + show_cursor(stream=target) @staticmethod def _clear_err(): diff --git a/pipenv/vendor/vistir/termcolors.py b/pipenv/vendor/vistir/termcolors.py index 8395d97d..27b5ff44 100644 --- a/pipenv/vendor/vistir/termcolors.py +++ b/pipenv/vendor/vistir/termcolors.py @@ -1,57 +1,27 @@ # -*- coding=utf-8 -*- from __future__ import absolute_import, print_function, unicode_literals -import colorama + import os +import re + +import colorama +import six + from .compat import to_native_string - -ATTRIBUTES = dict( - list(zip([ - 'bold', - 'dark', - '', - 'underline', - 'blink', - '', - 'reverse', - 'concealed' - ], - list(range(1, 9)) - )) - ) -del ATTRIBUTES[''] +DISABLE_COLORS = os.getenv("CI", False) or os.getenv( + "ANSI_COLORS_DISABLED", os.getenv("VISTIR_DISABLE_COLORS", False) +) -HIGHLIGHTS = dict( - list(zip([ - 'on_grey', - 'on_red', - 'on_green', - 'on_yellow', - 'on_blue', - 'on_magenta', - 'on_cyan', - 'on_white' - ], - list(range(40, 48)) - )) - ) +ATTRIBUTE_NAMES = ["bold", "dark", "", "underline", "blink", "", "reverse", "concealed"] +ATTRIBUTES = dict(zip(ATTRIBUTE_NAMES, range(1, 9))) +del ATTRIBUTES[""] - -COLORS = dict( - list(zip([ - 'grey', - 'red', - 'green', - 'yellow', - 'blue', - 'magenta', - 'cyan', - 'white', - ], - list(range(30, 38)) - )) - ) +colors = ["grey", "red", "green", "yellow", "blue", "magenta", "cyan", "white"] +COLORS = dict(zip(colors, range(30, 38))) +HIGHLIGHTS = dict(zip(["on_{0}".format(c) for c in colors], range(40, 48))) +ANSI_REMOVAL_RE = re.compile(r"\033\[((?:\d|;)*)([a-zA-Z])") COLOR_MAP = { @@ -101,39 +71,49 @@ def colored(text, color=None, on_color=None, attrs=None): colored('Hello, World!', 'red', 'on_grey', ['blue', 'blink']) colored('Hello, World!', 'green') """ - if os.getenv('ANSI_COLORS_DISABLED') is None: + return colorize(text, fg=color, bg=on_color, attrs=attrs) + + +def colorize(text, fg=None, bg=None, attrs=None): + if os.getenv("ANSI_COLORS_DISABLED") is None: style = "NORMAL" - if 'bold' in attrs: + if attrs is not None and not isinstance(attrs, list): + _attrs = [] + if isinstance(attrs, six.string_types): + _attrs.append(attrs) + else: + _attrs = list(attrs) + attrs = _attrs + if attrs and "bold" in attrs: style = "BRIGHT" - attrs.remove('bold') - if color is not None: - color = color.upper() + attrs.remove("bold") + if fg is not None: + fg = fg.upper() text = to_native_string("%s%s%s%s%s") % ( - to_native_string(getattr(colorama.Fore, color)), + to_native_string(getattr(colorama.Fore, fg)), to_native_string(getattr(colorama.Style, style)), to_native_string(text), to_native_string(colorama.Fore.RESET), to_native_string(colorama.Style.NORMAL), ) - if on_color is not None: - on_color = on_color.upper() + if bg is not None: + bg = bg.upper() text = to_native_string("%s%s%s%s") % ( - to_native_string(getattr(colorama.Back, on_color)), + to_native_string(getattr(colorama.Back, bg)), to_native_string(text), to_native_string(colorama.Back.RESET), to_native_string(colorama.Style.NORMAL), ) if attrs is not None: - fmt_str = to_native_string("%s[%%dm%%s%s[9m") % ( - chr(27), - chr(27) - ) + fmt_str = to_native_string("%s[%%dm%%s%s[9m") % (chr(27), chr(27)) for attr in attrs: text = fmt_str % (ATTRIBUTES[attr], text) text += RESET + else: + text = ANSI_REMOVAL_RE.sub("", text) return text diff --git a/pipenv/vendor/yaspin/__version__.py b/pipenv/vendor/yaspin/__version__.py index 9e78220f..23f00709 100644 --- a/pipenv/vendor/yaspin/__version__.py +++ b/pipenv/vendor/yaspin/__version__.py @@ -1 +1 @@ -__version__ = "0.14.0" +__version__ = "0.14.3" diff --git a/pipenv/vendor/yaspin/api.py b/pipenv/vendor/yaspin/api.py index 156630db..f59ce002 100644 --- a/pipenv/vendor/yaspin/api.py +++ b/pipenv/vendor/yaspin/api.py @@ -84,5 +84,7 @@ def kbi_safe_yaspin(*args, **kwargs): return Yaspin(*args, **kwargs) -_kbi_safe_doc = yaspin.__doc__.replace("yaspin", "kbi_safe_yaspin") -kbi_safe_yaspin.__doc__ = _kbi_safe_doc +# Handle PYTHONOPTIMIZE=2 case, when docstrings are set to None. +if yaspin.__doc__: + _kbi_safe_doc = yaspin.__doc__.replace("yaspin", "kbi_safe_yaspin") + kbi_safe_yaspin.__doc__ = _kbi_safe_doc diff --git a/pipenv/vendor/yaspin/core.py b/pipenv/vendor/yaspin/core.py index 06b8b621..12960b3b 100644 --- a/pipenv/vendor/yaspin/core.py +++ b/pipenv/vendor/yaspin/core.py @@ -17,7 +17,7 @@ import threading import time import colorama -import cursor +from pipenv.vendor.vistir import cursor from .base_spinner import default_spinner from .compat import PY2, basestring, builtin_str, bytes, iteritems, str @@ -82,6 +82,7 @@ class Yaspin(object): self._hide_spin = None self._spin_thread = None self._last_frame = None + self._stdout_lock = threading.Lock() # Signals @@ -253,43 +254,47 @@ class Yaspin(object): thr_is_alive = self._spin_thread and self._spin_thread.is_alive() if thr_is_alive and not self._hide_spin.is_set(): - # set the hidden spinner flag - self._hide_spin.set() + with self._stdout_lock: + # set the hidden spinner flag + self._hide_spin.set() - # clear the current line - sys.stdout.write("\r") - self._clear_line() + # clear the current line + sys.stdout.write("\r") + self._clear_line() - # flush the stdout buffer so the current line can be rewritten to - sys.stdout.flush() + # flush the stdout buffer so the current line + # can be rewritten to + sys.stdout.flush() def show(self): """Show the hidden spinner.""" thr_is_alive = self._spin_thread and self._spin_thread.is_alive() if thr_is_alive and self._hide_spin.is_set(): - # clear the hidden spinner flag - self._hide_spin.clear() + with self._stdout_lock: + # clear the hidden spinner flag + self._hide_spin.clear() - # clear the current line so the spinner is not appended to it - sys.stdout.write("\r") - self._clear_line() + # clear the current line so the spinner is not appended to it + sys.stdout.write("\r") + self._clear_line() def write(self, text): """Write text in the terminal without breaking the spinner.""" # similar to tqdm.write() # https://pypi.python.org/pypi/tqdm#writing-messages - sys.stdout.write("\r") - self._clear_line() + with self._stdout_lock: + sys.stdout.write("\r") + self._clear_line() - _text = to_unicode(text) - if PY2: - _text = _text.encode(ENCODING) + _text = to_unicode(text) + if PY2: + _text = _text.encode(ENCODING) - # Ensure output is bytes for Py2 and Unicode for Py3 - assert isinstance(_text, builtin_str) + # Ensure output is bytes for Py2 and Unicode for Py3 + assert isinstance(_text, builtin_str) - sys.stdout.write("{0}\n".format(_text)) + sys.stdout.write("{0}\n".format(_text)) def ok(self, text="OK"): """Set Ok (success) finalizer to a spinner.""" @@ -312,7 +317,8 @@ class Yaspin(object): # Should be stopped here, otherwise prints after # self._freeze call will mess up the spinner self.stop() - sys.stdout.write(self._last_frame) + with self._stdout_lock: + sys.stdout.write(self._last_frame) def _spin(self): while not self._stop_spin.is_set(): @@ -327,13 +333,13 @@ class Yaspin(object): out = self._compose_out(spin_phase) # Write - sys.stdout.write(out) - self._clear_line() - sys.stdout.flush() + with self._stdout_lock: + sys.stdout.write(out) + self._clear_line() + sys.stdout.flush() # Wait time.sleep(self._interval) - sys.stdout.write("\b") def _compose_color_func(self): fn = functools.partial( @@ -530,11 +536,11 @@ class Yaspin(object): @staticmethod def _hide_cursor(): - cursor.hide() + cursor.hide_cursor() @staticmethod def _show_cursor(): - cursor.show() + cursor.show_cursor() @staticmethod def _clear_line(): diff --git a/pyproject.toml b/pyproject.toml index a799764c..71c94f3d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,32 @@ [build-system] requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.black] +line-length = 90 +include = '\.pyi?$' +exclude = ''' +/( + \.eggs + | \.git + | \.github + | \.hg + | \.mypy_cache + | \.tox + | \.pyre_configuration + | \.venv + | _build + | buck-out + | build + | dist + | pipenv/vendor + | pipenv/patched + | tests/pypi + | tests/pytest-pypi + | tests/test_artifacts + | get-pipenv.py +) +''' [tool.towncrier] package = "pipenv" diff --git a/pytest.ini b/pytest.ini index ff9847d3..da966ec9 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,8 +1,25 @@ [pytest] addopts = -ra -n auto +plugins = xdist testpaths = tests ; Add vendor and patched in addition to the default list of ignored dirs -norecursedirs = .* build dist CVS _darcs {arch} *.egg vendor patched news tasks docs tests/test_artifacts +; Additionally, ignore tasks, news, test subdirectories and peeps directory +norecursedirs = + .* build + dist + CVS + _darcs + {arch} + *.egg + vendor + patched + news + tasks + docs + tests/test_artifacts + tests/pytest-pypi + tests/pypi + peeps filterwarnings = ignore::DeprecationWarning ignore::PendingDeprecationWarning diff --git a/run-tests.bat b/run-tests.bat index d20af35f..973425af 100644 --- a/run-tests.bat +++ b/run-tests.bat @@ -1,7 +1,15 @@ -rem imdisk -a -s 964515b -m R: -p "/FS:NTFS /Y" +rem If you want to use a ramdisk, use this section: -virtualenv R:\.venv -R:\.venv\Scripts\pip install -e . --upgrade --upgrade-strategy=only-if-needed +rem imdisk -a -s 4G -m R: -p "FS:NTFS /y" +rem if you are using a ram disk, you should comment the following substitution line out +subst R: %TEMP% + +set TMP=R:\\ +set TEMP=R:\\ +set WORKON_HOME=R:\\ +set RAM_DISK=R:\\ + +R:\.venv\Scripts\pip install -e .[test] --upgrade --upgrade-strategy=only-if-needed R:\.venv\Scripts\pipenv install --dev git submodule sync && git submodule update --init --recursive -SET RAM_DISK=R: && R:\.venv\Scripts\pipenv run pytest -n auto -v tests --tap-stream > report.tap +R:\.venv\Scripts\pipenv run pytest -n auto -v tests diff --git a/run-tests.sh b/run-tests.sh index 991c0df7..aef44f63 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -7,6 +7,8 @@ set -eo pipefail export PYTHONIOENCODING="utf-8" export LANG=C.UTF-8 export PIP_PROCESS_DEPENDENCY_LINKS="1" +# Let's use a temporary cache directory +export PIPENV_CACHE_DIR=`mktemp -d 2>/dev/null || mktemp -d -t 'pipenv_cache'` prefix() { sed "s/^/ $1: /" @@ -25,49 +27,27 @@ fi if [[ ! -z "$HOME" ]]; then export PATH="${HOME}/.local/bin:${PATH}" fi -# pip uninstall -y pipenv pip install certifi export GIT_SSL_CAINFO=$(python -m certifi) echo "Path: $PATH" echo "Installing Pipenv…" -PIP_USER="1" python -m pip install --upgrade setuptools -PIP_USER="1" python3 -m pip install --upgrade setuptools -python -m pip install -e "$(pwd)" --upgrade && python3 -m pip install -e "$(pwd)" --upgrade -python3 -m pipenv install --deploy --dev --system - -# Otherwise, we're on a development machine. -# First, try MacOS… -if [[ $(python -c "import sys; print(sys.platform)") == "darwin" ]]; then - - echo "Clearing Caches…" - rm -fr ~/Library/Caches/pip - rm -fr ~/Library/Caches/pipenv - -# Otherwise, assume Linux… -else - echo "Clearing Caches…" - rm -fr ~/.cache/pip - rm -fr ~/.cache/pipenv -fi +python -m pip install --upgrade -e "$(pwd)" setuptools wheel pip +VENV_CMD="python -m pipenv --venv" +RM_CMD="pipenv --rm" +echo "$ PIPENV_PYTHON=2.7 $VENV_CMD && PIPENV_PYTHON=2.7 $RM_CMD" +echo "$ PIPENV_PYTHON=3.7 $VENV_CMD && PIPENV_PYTHON=3.7 $RM_CMD" +{ PIPENV_PYTHON=2.7 $VENV_CMD && PIPENV_PYTHON=2.7 $RM_CMD ; PIPENV_PYTHON=3.7 $VENV_CMD && PIPENV_PYTHON=3.7 $RM_CMD ; } echo "Installing dependencies…" -PIPENV_PYTHON=2.7 python3 -m pipenv --venv && pipenv --rm && pipenv install --dev -PIPENV_PYTHON=3.7 python3 -m pipenv --venv && pipenv --rm && pipenv install --dev -PIPENV_PYTHON=2.7 python3 -m pipenv run pip install --upgrade -e . -PIPENV_PYTHON=3.7 python3 -m pipenv run pip install --upgrade -e . +INSTALL_CMD="python -m pipenv install --deploy --dev" +echo "$ PIPENV_PYTHON=2.7 $INSTALL_CMD" +echo "$ PIPENV_PYTHON=3.7 $INSTALL_CMD" + +{ ( PIPENV_PYTHON=2.7 $INSTALL_CMD & ); PIPENV_PYTHON=3.7 $INSTALL_CMD ; } echo "$ git submodule sync && git submodule update --init --recursive" git submodule sync && git submodule update --init --recursive -echo "$ pipenv run time pytest -v -n auto tests -m \"$TEST_SUITE\"" -# PIPENV_PYTHON=2.7 pipenv run time pytest -v -n auto tests -m "$TEST_SUITE" | prefix 2.7 & -# PIPENV_PYTHON=3.6 pipenv run time pytest -v -n auto tests -m "$TEST_SUITE" | prefix 3.6 -# Better to run them sequentially. -PIPENV_PYTHON=2.7 python3 -m pipenv run time pytest -PIPENV_PYTHON=3.7 python3 -m pipenv run time pytest - -# test revendoring -pip3 install --upgrade invoke requests parver vistir -python3 -m invoke vendoring.update -# Cleanup junk. -rm -fr .venv +echo "$ pipenv run time pytest" +PIPENV_PYTHON=2.7 python -m pipenv run time pytest +PIPENV_PYTHON=3.7 python -m pipenv run time pytest diff --git a/setup.cfg b/setup.cfg index 78140fa0..896a5a91 100644 --- a/setup.cfg +++ b/setup.cfg @@ -17,7 +17,7 @@ ignore = # E402: module level import not at top of file # E501: line too long # W503: line break before binary operator - E127,E128,E129,E222,E231,E402,E501,W503 + E402,E501,W503 [isort] atomic=true @@ -29,3 +29,10 @@ known_first_party = pipenv tests ignore_trailing_comma=true + +[mypy] +ignore_missing_imports=true +follow_imports=skip +html_report=mypyhtml +python_version=3.6 +mypy_path=typeshed/pyi:typeshed/imports diff --git a/setup.py b/setup.py index 4a6b9389..d86d85e0 100644 --- a/setup.py +++ b/setup.py @@ -22,17 +22,28 @@ if sys.argv[-1] == "publish": sys.exit() required = [ - "pip>=9.0.1", + "pip>=18.0", "certifi", "setuptools>=36.2.1", "virtualenv-clone>=0.2.5", "virtualenv", - 'requests[security];python_version<"2.7"', - 'ordereddict;python_version<"2.7"', 'enum34; python_version<"3"', + # LEAVE THIS HERE!!! we have vendored dependencies that require it 'typing; python_version<"3.5"' ] - +extras = { + "dev": [ + "towncrier", + "bs4", + "twine", + "sphinx<2", + "flake8>=3.3.0,<4.0", + "black;python_version>='3.6'", + "parver", + "invoke", + ], + "tests": ["pytest<5.0", "pytest-tap", "pytest-xdist", "flaky", "mock"], +} # https://pypi.python.org/pypi/stdeb/0.8.5#quickstart-2-just-tell-me-the-fastest-way-to-make-a-deb class DebCommand(Command): @@ -68,7 +79,7 @@ class DebCommand(Command): class UploadCommand(Command): - """Support setup.py publish.""" + """Support setup.py upload.""" description = "Build and publish the package." user_options = [] @@ -132,9 +143,9 @@ setup( ], }, python_requires=">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*", - setup_requires=["invoke", "parver"], + setup_requires=[], install_requires=required, - extras_require={}, + extras_require=extras, include_package_data=True, license="MIT", classifiers=[ diff --git a/tasks/__init__.py b/tasks/__init__.py index 6896edb6..63fe1388 100644 --- a/tasks/__init__.py +++ b/tasks/__init__.py @@ -1,22 +1,16 @@ # -*- coding=utf-8 -*- -# Copyied from pip's vendoring process +# Copied from pip's vendoring process # see https://github.com/pypa/pip/blob/95bcf8c5f6394298035a7332c441868f3b0169f4/tasks/__init__.py -import invoke import re -from . import vendoring, release -from .vendoring import vendor_passa from pathlib import Path +import invoke + +from . import release, vendoring +from .vendoring import vendor_passa + + ROOT = Path(".").parent.parent.absolute() - -@invoke.task -def clean_mdchangelog(ctx): - changelog = ROOT / "CHANGELOG.md" - content = changelog.read_text() - content = re.sub(r"([^\n]+)\n?\s+\[[\\]+(#\d+)\]\(https://github\.com/pypa/[\w\-]+/issues/\d+\)", r"\1 \2", content, flags=re.MULTILINE) - changelog.write_text(content) - - -ns = invoke.Collection(vendoring, release, clean_mdchangelog, vendor_passa.vendor_passa) +ns = invoke.Collection(vendoring, release, release.clean_mdchangelog, vendor_passa.vendor_passa) diff --git a/tasks/release.py b/tasks/release.py index d85c6920..02f694ac 100644 --- a/tasks/release.py +++ b/tasks/release.py @@ -1,147 +1,265 @@ # -*- coding=utf-8 -*- import datetime -import invoke +import os +import pathlib +import re import sys -from pipenv.__version__ import __version__ + +import invoke + from parver import Version +from towncrier._builder import find_fragments, render_fragments, split_fragments +from towncrier._settings import load_config + +from pipenv.__version__ import __version__ +from pipenv.vendor.vistir.contextmanagers import temp_environ + from .vendoring import _get_git_root, drop_dir -VERSION_FILE = 'pipenv/__version__.py' +VERSION_FILE = "pipenv/__version__.py" +ROOT = pathlib.Path(".").parent.parent.absolute() +PACKAGE_NAME = "pipenv" def log(msg): - print('[release] %s' % msg) + print("[release] %s" % msg) def get_version_file(ctx): return _get_git_root(ctx).joinpath(VERSION_FILE) +def find_version(ctx): + version_file = get_version_file(ctx).read_text() + version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", version_file, re.M) + if version_match: + return version_match.group(1) + raise RuntimeError("Unable to find version string.") + + def get_history_file(ctx): - return _get_git_root(ctx).joinpath('HISTORY.txt') + return _get_git_root(ctx).joinpath("HISTORY.txt") def get_dist_dir(ctx): - return _get_git_root(ctx) / 'dist' + return _get_git_root(ctx) / "dist" def get_build_dir(ctx): - return _get_git_root(ctx) / 'build' + return _get_git_root(ctx) / "build" + + +def _render_log(): + """Totally tap into Towncrier internals to get an in-memory result. + """ + config = load_config(ROOT) + definitions = config["types"] + fragments, fragment_filenames = find_fragments( + pathlib.Path(config["directory"]).absolute(), + config["sections"], + None, + definitions, + ) + rendered = render_fragments( + pathlib.Path(config["template"]).read_text(encoding="utf-8"), + config["issue_format"], + split_fragments(fragments, definitions), + definitions, + config["underlines"][1:], + False, # Don't add newlines to wrapped text. + ) + return rendered + + +@invoke.task +def release(ctx, dry_run=False): + drop_dist_dirs(ctx) + bump_version(ctx, dry_run=dry_run) + version = find_version(ctx) + tag_content = _render_log() + if dry_run: + ctx.run("towncrier --draft > CHANGELOG.draft.rst") + log("would remove: news/*") + log("would remove: CHANGELOG.draft.rst") + log(f'Would commit with message: "Release v{version}"') + else: + ctx.run("towncrier") + ctx.run( + "git add CHANGELOG.rst news/ {0}".format(get_version_file(ctx).as_posix()) + ) + ctx.run("git rm CHANGELOG.draft.rst") + generate_manual(ctx) + ctx.run(f'git commit -m "Release v{version}"') + + tag_content = tag_content.replace('"', '\\"') + if dry_run: + log(f"Generated tag content: {tag_content}") + markdown = ctx.run( + "pandoc CHANGELOG.draft.rst -f rst -t markdown", hide=True + ).stdout.strip() + content = clean_mdchangelog(ctx, markdown) + log(f"would generate markdown: {content}") + else: + generate_markdown(ctx) + clean_mdchangelog(ctx) + ctx.run(f'git tag -a v{version} -m "Version v{version}\n\n{tag_content}"') + build_dists(ctx) + if dry_run: + dist_pattern = f'{PACKAGE_NAME.replace("-", "[-_]")}-*' + artifacts = list(ROOT.joinpath("dist").glob(dist_pattern)) + filename_display = "\n".join(f" {a}" for a in artifacts) + log(f"Would upload dists: {filename_display}") + else: + upload_dists(ctx) + bump_version(ctx, dev=True) def drop_dist_dirs(ctx): - log('Dropping Dist dir...') + log("Dropping Dist dir...") drop_dir(get_dist_dir(ctx)) - log('Dropping build dir...') + log("Dropping build dir...") drop_dir(get_build_dir(ctx)) @invoke.task def build_dists(ctx): drop_dist_dirs(ctx) - log('Building sdist using %s ....' % sys.executable) - for py_version in ['2.7', '3.6', '3.7']: - env = {'PIPENV_PYTHON': py_version} - ctx.run('pipenv install --dev', env=env) - if py_version == '3.6': - ctx.run('pipenv run python setup.py sdist', env=env) - log('Building wheel using python %s ....' % py_version) - ctx.run('pipenv run python setup.py bdist_wheel', env=env) + for py_version in ["3.6", "2.7"]: + env = {"PIPENV_PYTHON": py_version} + with ctx.cd(ROOT.as_posix()), temp_environ(): + executable = ctx.run( + "python -c 'import sys; print(sys.executable)'", hide=True + ).stdout.strip() + log("Building sdist using %s ...." % executable) + os.environ["PIPENV_PYTHON"] = py_version + ctx.run("pipenv install --dev", env=env) + ctx.run( + "pipenv run pip install -e . --upgrade --upgrade-strategy=eager", env=env + ) + log("Building wheel using python %s ...." % py_version) + if py_version == "3.6": + ctx.run("pipenv run python setup.py sdist bdist_wheel", env=env) + else: + ctx.run("pipenv run python setup.py bdist_wheel", env=env) @invoke.task(build_dists) -def upload_dists(ctx): - log('Uploading distributions to pypi...') - ctx.run('twine upload dist/*') +def upload_dists(ctx, repo="pypi"): + dist_pattern = f'{PACKAGE_NAME.replace("-", "[-_]")}-*' + artifacts = list(ROOT.joinpath("dist").glob(dist_pattern)) + filename_display = "\n".join(f" {a}" for a in artifacts) + print(f"[release] Will upload:\n{filename_display}") + try: + input("[release] Release ready. ENTER to upload, CTRL-C to abort: ") + except KeyboardInterrupt: + print("\nAborted!") + return + + arg_display = " ".join(f'"{n}"' for n in artifacts) + ctx.run(f'twine upload --repository="{repo}" {arg_display}') @invoke.task def generate_markdown(ctx): - log('Generating markdown from changelog...') - ctx.run('pandoc CHANGELOG.rst -f rst -t markdown -o CHANGELOG.md') + log("Generating markdown from changelog...") + ctx.run("pandoc CHANGELOG.rst -f rst -t markdown -o CHANGELOG.md") @invoke.task def generate_manual(ctx, commit=False): - log('Generating manual from reStructuredText source...') - ctx.run('make man -C docs') - ctx.run('cp docs/_build/man/pipenv.1 pipenv/') + log("Generating manual from reStructuredText source...") + ctx.run("make man -C docs") + ctx.run("cp docs/_build/man/pipenv.1 pipenv/") if commit: - log('Commiting...') - ctx.run('git add pipenv/pipenv.1') + log("Commiting...") + ctx.run("git add pipenv/pipenv.1") ctx.run('git commit -m "Update manual page."') @invoke.task def generate_changelog(ctx, commit=False, draft=False): - log('Generating changelog...') + log("Generating changelog...") if draft: commit = False - log('Writing draft to file...') - ctx.run('towncrier --draft > CHANGELOG.draft.rst') + log("Writing draft to file...") + ctx.run("towncrier --draft > CHANGELOG.draft.rst") + else: + ctx.run("towncrier") if commit: - ctx.run('towncrier') - log('Committing...') - ctx.run('git add CHANGELOG.rst') - ctx.run('git rm CHANGELOG.draft.rst') + log("Committing...") + ctx.run("git add CHANGELOG.rst") + ctx.run("git rm CHANGELOG.draft.rst") ctx.run('git commit -m "Update changelog."') @invoke.task -def tag_version(ctx, push=False): - version = Version.parse(__version__) - log('Tagging revision: v%s' % version) - ctx.run('git tag v%s' % version) - if push: - log('Pushing tags...') - ctx.run('git push --tags') +def clean_mdchangelog(ctx, content=None): + changelog = None + if not content: + changelog = _get_git_root(ctx) / "CHANGELOG.md" + content = changelog.read_text() + content = re.sub( + r"([^\n]+)\n?\s+\[[\\]+(#\d+)\]\(https://github\.com/pypa/[\w\-]+/issues/\d+\)", + r"\1 \2", + content, + flags=re.MULTILINE, + ) + if changelog: + changelog.write_text(content) + else: + return content @invoke.task -def bump_version(ctx, dry_run=False, increment=True, release=False, dev=False, pre=False, tag=None, clear=False, commit=False,): +def tag_version(ctx, push=False): + version = find_version(ctx) + version = Version.parse(version) + log("Tagging revision: v%s" % version.normalize()) + ctx.run("git tag v%s" % version.normalize()) + if push: + log("Pushing tags...") + ctx.run("git push origin master") + ctx.run("git push --tags") + + +@invoke.task +def bump_version(ctx, dry_run=False, dev=False, pre=False, tag=None, commit=False): current_version = Version.parse(__version__) today = datetime.date.today() - next_month_number = today.month + 1 if today.month != 12 else 1 - next_year_number = today.year if next_month_number != 1 else today.year+1 - next_month = (next_year_number, next_month_number, 0) + tomorrow = today + datetime.timedelta(days=1) + next_month = datetime.date.today().replace(month=today.month + 1, day=1) + next_year = datetime.date.today().replace(year=today.year + 1, month=1, day=1) if pre and not tag: print('Using "pre" requires a corresponding tag.') return - if release and not dev and not pre and increment: - new_version = current_version.replace(release=today.timetuple()[:3]).clear(pre=True, dev=True) - elif release and (dev or pre): - if increment: - new_version = current_version.replace(release=today.timetuple()[:3]) - else: - new_version = current_version + if not (dev or pre or tag): + new_version = current_version.replace(release=today.timetuple()[:3]).clear( + pre=True, dev=True + ) + if pre and dev: + raise RuntimeError("Can't use 'pre' and 'dev' together!") + if dev or pre: + new_version = current_version.replace(release=tomorrow.timetuple()[:3]).clear( + pre=True, dev=True + ) if dev: new_version = new_version.bump_dev() - elif pre: - new_version = new_version.bump_pre(tag=tag) - else: - if not release: - increment = False - if increment: - new_version = current_version.replace(release=next_month) else: - new_version = current_version - if dev: - new_version = new_version.bump_dev() - elif pre: new_version = new_version.bump_pre(tag=tag) - if clear: - new_version = new_version.clear(dev=True, pre=True, post=True) - log('Updating version to %s' % new_version.normalize()) - version_file = get_version_file(ctx) - file_contents = version_file.read_text() - log('Found current version: %s' % __version__) + log("Updating version to %s" % new_version.normalize()) + version = find_version(ctx) + log("Found current version: %s" % version) if dry_run: - log('Would update to: %s' % new_version.normalize()) + log("Would update to: %s" % new_version.normalize()) else: - log('Updating to: %s' % new_version.normalize()) - version_file.write_text(file_contents.replace(__version__, str(new_version.normalize()))) + log("Updating to: %s" % new_version.normalize()) + version_file = get_version_file(ctx) + file_contents = version_file.read_text() + version_file.write_text( + file_contents.replace(version, str(new_version.normalize())) + ) if commit: - ctx.run('git add {0}'.format(version_file)) - log('Committing...') + ctx.run("git add {0}".format(version_file.as_posix())) + log("Committing...") ctx.run('git commit -s -m "Bumped version."') diff --git a/tasks/vendoring/__init__.py b/tasks/vendoring/__init__.py index ea0d038c..69398450 100644 --- a/tasks/vendoring/__init__.py +++ b/tasks/vendoring/__init__.py @@ -1,19 +1,30 @@ # -*- coding=utf-8 -*- -""""Vendoring script, python 3.5 needed""" # Taken from pip # see https://github.com/pypa/pip/blob/95bcf8c5f6394298035a7332c441868f3b0169f4/tasks/vendoring/__init__.py -from pipenv._compat import NamedTemporaryFile, TemporaryDirectory -from pathlib import Path -from pipenv.utils import mkdir_p -# from tempfile import TemporaryDirectory -import tarfile -import zipfile +""""Vendoring script, python 3.5 needed""" + +import io import re import shutil import sys +# from tempfile import TemporaryDirectory +import tarfile +import zipfile + +from pathlib import Path + +import bs4 import invoke import requests +from urllib3.util import parse_url as urllib3_parse + +from pipenv.utils import mkdir_p +from pipenv.vendor.vistir.compat import NamedTemporaryFile, TemporaryDirectory +from pipenv.vendor.vistir.contextmanagers import open_file +import pipenv.vendor.parse as parse + + TASK_NAME = 'update' LIBRARY_DIRNAMES = { @@ -39,8 +50,6 @@ HARDCODED_LICENSE_URLS = { 'delegator.py': 'https://raw.githubusercontent.com/kennethreitz/delegator.py/master/LICENSE', 'click-didyoumean': 'https://raw.githubusercontent.com/click-contrib/click-didyoumean/master/LICENSE', 'click-completion': 'https://raw.githubusercontent.com/click-contrib/click-completion/master/LICENSE', - 'blindspin': 'https://raw.githubusercontent.com/kennethreitz/delegator.py/master/LICENSE', - 'shutilwhich': 'https://raw.githubusercontent.com/mbr/shutilwhich/master/LICENSE', 'parse': 'https://raw.githubusercontent.com/techalchemy/parse/master/LICENSE', 'semver': 'https://raw.githubusercontent.com/k-bx/python-semver/master/LICENSE.txt', 'crayons': 'https://raw.githubusercontent.com/kennethreitz/crayons/master/LICENSE', @@ -79,6 +88,12 @@ LIBRARY_RENAMES = { } +LICENSE_RENAMES = { + "pythonfinder/LICENSE": "pythonfinder/pep514tools.LICENSE" +} + + + def drop_dir(path): if path.exists() and path.is_dir(): shutil.rmtree(str(path), ignore_errors=True) @@ -336,6 +351,26 @@ def post_install_cleanup(ctx, vendor_dir): remove_all(vendor_dir.glob('toml.py')) +@invoke.task +def apply_patches(ctx, patched=False, pre=False): + if patched: + vendor_dir = _get_patched_dir(ctx) + else: + vendor_dir = _get_vendor_dir(ctx) + log("Applying pre-patches...") + patch_dir = Path(__file__).parent / 'patches' / vendor_dir.name + if pre: + if not patched: + pass + for patch in patch_dir.glob('*.patch'): + if not patch.name.startswith('_post'): + apply_patch(ctx, patch) + else: + patches = patch_dir.glob('*.patch' if not patched else '_post*.patch') + for patch in patches: + apply_patch(ctx, patch) + + def vendor(ctx, vendor_dir, package=None, rewrite=True): log('Reinstalling vendored libraries') is_patched = vendor_dir.name == 'patched' @@ -349,12 +384,8 @@ def vendor(ctx, vendor_dir, package=None, rewrite=True): # Apply pre-patches log("Applying pre-patches...") - patch_dir = Path(__file__).parent / 'patches' / vendor_dir.name if is_patched: - for patch in patch_dir.glob('*.patch'): - if not patch.name.startswith('_post'): - apply_patch(ctx, patch) - + apply_patches(ctx, patched=is_patched, pre=True) log("Removing scandir library files...") remove_all(vendor_dir.glob('*.so')) drop_dir(vendor_dir / 'setuptools') @@ -375,10 +406,7 @@ def vendor(ctx, vendor_dir, package=None, rewrite=True): rewrite_file_imports(item, vendored_libs, vendor_dir) write_backport_imports(ctx, vendor_dir) if not package: - log('Applying post-patches...') - patches = patch_dir.glob('*.patch' if not is_patched else '_post*.patch') - for patch in patches: - apply_patch(ctx, patch) + apply_patches(ctx, patched=is_patched, pre=False) if is_patched: piptools_vendor = vendor_dir / 'piptools' / '_vendored' if piptools_vendor.exists(): @@ -435,19 +463,21 @@ def packages_missing_licenses(ctx, vendor_dir=None, requirements_file='vendor.tx possible_pkgs.append(LIBRARY_DIRNAMES[pkg]) for pkgpath in possible_pkgs: pkgpath = vendor_dir.joinpath(pkgpath) + py_path = pkgpath.parent / "{0}.py".format(pkgpath.stem) if pkgpath.exists() and pkgpath.is_dir(): - for licensepath in LICENSES: - licensepath = pkgpath.joinpath(licensepath) - if licensepath.exists(): + for license_path in LICENSES: + license_path = pkgpath.joinpath(license_path) + if license_path.exists(): match_found = True - # log("%s: Trying path %s... FOUND" % (pkg, licensepath)) + # log("%s: Trying path %s... FOUND" % (pkg, license_path)) break - elif (pkgpath.exists() or pkgpath.parent.joinpath("{0}.py".format(pkgpath.stem)).exists()): - for licensepath in LICENSES: - licensepath = pkgpath.parent.joinpath("{0}.{1}".format(pkgpath.stem, licensepath)) - if licensepath.exists(): + elif pkgpath.exists() or py_path.exists(): + for license_path in LICENSES: + license_name = "{0}.{1}".format(pkgpath.stem, license_path) + license_path = pkgpath.parent / license_name + if license_path.exists(): match_found = True - # log("%s: Trying path %s... FOUND" % (pkg, licensepath)) + # log("%s: Trying path %s... FOUND" % (pkg, license_path)) break if match_found: break @@ -460,7 +490,10 @@ def packages_missing_licenses(ctx, vendor_dir=None, requirements_file='vendor.tx @invoke.task -def download_licenses(ctx, vendor_dir=None, requirements_file='vendor.txt', package=None, only=False, patched=False): +def download_licenses( + ctx, vendor_dir=None, requirements_file='vendor.txt', package=None, only=False, + patched=False +): log('Downloading licenses') if not vendor_dir: if patched: @@ -486,13 +519,37 @@ def download_licenses(ctx, vendor_dir=None, requirements_file='vendor.txt', pack requirement = package tmp_dir = vendor_dir / '__tmp__' # TODO: Fix this whenever it gets sorted out (see https://github.com/pypa/pip/issues/5739) + cmd = "pip download --no-binary :all: --only-binary requests_download --no-deps" + enum_cmd = "pip download --no-deps" ctx.run('pip install flit') # needed for the next step - ctx.run( - 'pip download --no-binary :all: --only-binary requests_download --no-build-isolation --no-deps -d {0} {1}'.format( - tmp_dir.as_posix(), - requirement, - ) - ) + for req in requirements_file.read_text().splitlines(): + if req.startswith("enum34"): + exe_cmd = "{0} -d {1} {2}".format(enum_cmd, tmp_dir.as_posix(), req) + else: + exe_cmd = "{0} --no-build-isolation -d {1} {2}".format( + cmd, tmp_dir.as_posix(), req + ) + try: + ctx.run(exe_cmd) + except invoke.exceptions.UnexpectedExit as e: + if "Disabling PEP 517 processing is invalid" not in e.result.stderr: + log("WARNING: Failed to download license for {0}".format(req)) + continue + parse_target = ( + "Disabling PEP 517 processing is invalid: project specifies a build " + "backend of {backend} in pyproject.toml" + ) + target = parse.parse(parse_target, e.result.stderr.strip()) + backend = target.named.get("backend") + if backend is not None: + if "." in backend: + backend, _, _ = backend.partition(".") + ctx.run("pip install {0}".format(backend)) + ctx.run( + "{0} --no-build-isolation -d {1} {2}".format( + cmd, tmp_dir.as_posix(), req + ) + ) for sdist in tmp_dir.iterdir(): extract_license(vendor_dir, sdist) new_requirements_file.unlink() @@ -504,7 +561,7 @@ def extract_license(vendor_dir, sdist): ext = sdist.suffix[1:] with tarfile.open(sdist, mode='r:{}'.format(ext)) as tar: found = find_and_extract_license(vendor_dir, tar, tar.getmembers()) - elif sdist.suffix == '.zip': + elif sdist.suffix in ('.zip', '.whl'): with zipfile.ZipFile(sdist) as zip: found = find_and_extract_license(vendor_dir, zip, zip.infolist()) else: @@ -576,6 +633,9 @@ def license_destination(vendor_dir, libname, filename): return ( vendor_dir / override.parent ) / '{0}.{1}'.format(override.name, filename) + license_path = Path(LIBRARY_DIRNAMES[libname]) / filename + if license_path.as_posix() in LICENSE_RENAMES: + return vendor_dir / LICENSE_RENAMES[license_path.as_posix()] return vendor_dir / LIBRARY_DIRNAMES[libname] / filename # fallback to libname.LICENSE (used for nondirs) return vendor_dir / '{}.{}'.format(libname, filename) @@ -613,6 +673,18 @@ def generate_patch(ctx, package_path, patch_description, base='HEAD'): ctx.run(command) +@invoke.task() +def update_pip_deps(ctx): + patched_dir = _get_patched_dir(ctx) + base_vendor_dir = _get_vendor_dir(ctx) + base_vendor_file = base_vendor_dir / "vendor_pip.txt" + pip_dir = patched_dir / "notpip" + vendor_dir = pip_dir / "_vendor" + vendor_file = vendor_dir / "vendor.txt" + vendor_file.write_bytes(base_vendor_file.read_bytes()) + download_licenses(ctx, vendor_dir) + + @invoke.task(name=TASK_NAME) def main(ctx, package=None): vendor_dir = _get_vendor_dir(ctx) @@ -640,3 +712,22 @@ def main(ctx, package=None): # vendor_passa(ctx) # update_safety(ctx) log('Revendoring complete') + + +@invoke.task +def vendor_artifact(ctx, package, version=None): + simple = requests.get("https://pypi.org/simple/{0}/".format(package)) + pkg_str = "{0}-{1}".format(package, version) + soup = bs4.BeautifulSoup(simple.content) + links = [ + a.attrs["href"] for a in soup.find_all("a") if a.getText().startswith(pkg_str) + ] + for link in links: + dest_dir = _get_git_root(ctx) / "tests" / "pypi" / package + if not dest_dir.exists(): + dest_dir.mkdir() + _, _, dest_path = urllib3_parse(link).path.rpartition("/") + dest_file = dest_dir / dest_path + with io.open(dest_file.as_posix(), "wb") as target_handle: + with open_file(link) as fp: + shutil.copyfileobj(fp, target_handle) diff --git a/tasks/vendoring/patches/patched/_post-pip-update-pep425tags.patch b/tasks/vendoring/patches/patched/_post-pip-update-pep425tags.patch index fce6ae89..b4ffbc9f 100644 --- a/tasks/vendoring/patches/patched/_post-pip-update-pep425tags.patch +++ b/tasks/vendoring/patches/patched/_post-pip-update-pep425tags.patch @@ -1,8 +1,8 @@ diff --git a/pipenv/patched/notpip/_internal/pep425tags.py b/pipenv/patched/notpip/_internal/pep425tags.py -index f3b9b5b4..182c1c88 100644 +index 3c760ca3..3b11b965 100644 --- a/pipenv/patched/notpip/_internal/pep425tags.py +++ b/pipenv/patched/notpip/_internal/pep425tags.py -@@ -159,7 +159,7 @@ def is_manylinux1_compatible(): +@@ -178,7 +178,7 @@ def is_manylinux1_compatible(): pass # Check glibc version. CentOS 5 uses glibc 2.5. @@ -10,4 +10,13 @@ index f3b9b5b4..182c1c88 100644 + return pipenv.patched.notpip._internal.utils.glibc.have_compatible_glibc(2, 5) + def is_manylinux2010_compatible(): +@@ -196,7 +196,7 @@ def is_manylinux2010_compatible(): + pass + + # Check glibc version. CentOS 6 uses glibc 2.12. +- return pip._internal.utils.glibc.have_compatible_glibc(2, 12) ++ return pipenv.patched.notpip._internal.utils.glibc.have_compatible_glibc(2, 12) + + def get_darwin_arches(major, minor, machine): diff --git a/tasks/vendoring/patches/patched/_post-pip-update-pypi-uri.patch b/tasks/vendoring/patches/patched/_post-pip-update-pypi-uri.patch index 99269a90..93f7ccbc 100644 --- a/tasks/vendoring/patches/patched/_post-pip-update-pypi-uri.patch +++ b/tasks/vendoring/patches/patched/_post-pip-update-pypi-uri.patch @@ -12,7 +12,7 @@ index 2406be21..7a87cdcf 100644 class PackageIndex(object): diff --git a/pipenv/patched/notpip/_vendor/distlib/locators.py b/pipenv/patched/notpip/_vendor/distlib/locators.py -index 11d26361..cb05b184 100644 +index 5c655c3e..a7ed9469 100644 --- a/pipenv/patched/notpip/_vendor/distlib/locators.py +++ b/pipenv/patched/notpip/_vendor/distlib/locators.py @@ -36,7 +36,7 @@ logger = logging.getLogger(__name__) @@ -33,7 +33,7 @@ index 11d26361..cb05b184 100644 is_downloadable, is_wheel, compatible, basename) def prefer_url(self, url1, url2): -@@ -1046,7 +1046,7 @@ class AggregatingLocator(Locator): +@@ -1049,7 +1049,7 @@ class AggregatingLocator(Locator): # versions which don't conform to PEP 426 / PEP 440. default_locator = AggregatingLocator( JSONLocator(), diff --git a/tasks/vendoring/patches/patched/_post-pip-update-requests-imports.patch b/tasks/vendoring/patches/patched/_post-pip-update-requests-imports.patch index e2f4f7de..79e12659 100644 --- a/tasks/vendoring/patches/patched/_post-pip-update-requests-imports.patch +++ b/tasks/vendoring/patches/patched/_post-pip-update-requests-imports.patch @@ -1,5 +1,5 @@ diff --git a/pipenv/patched/notpip/_vendor/requests/packages.py b/pipenv/patched/notpip/_vendor/requests/packages.py -index 9582fa7..928d1bb 100644 +index 9582fa73..258c89ed 100644 --- a/pipenv/patched/notpip/_vendor/requests/packages.py +++ b/pipenv/patched/notpip/_vendor/requests/packages.py @@ -4,13 +4,13 @@ import sys diff --git a/tasks/vendoring/patches/patched/pip18.patch b/tasks/vendoring/patches/patched/pip19.patch similarity index 70% rename from tasks/vendoring/patches/patched/pip18.patch rename to tasks/vendoring/patches/patched/pip19.patch index f4e607c1..74b98f50 100644 --- a/tasks/vendoring/patches/patched/pip18.patch +++ b/tasks/vendoring/patches/patched/pip19.patch @@ -1,16 +1,8 @@ diff --git a/pipenv/patched/pip/_internal/download.py b/pipenv/patched/pip/_internal/download.py -index 96f3b65c..cc5b3d15 100644 +index 2bbe1762..872af328 100644 --- a/pipenv/patched/pip/_internal/download.py +++ b/pipenv/patched/pip/_internal/download.py -@@ -19,6 +19,7 @@ from pip._vendor.lockfile import LockError - from pip._vendor.requests.adapters import BaseAdapter, HTTPAdapter - from pip._vendor.requests.auth import AuthBase, HTTPBasicAuth - from pip._vendor.requests.models import CONTENT_CHUNK_SIZE, Response -+from pip._vendor.requests.sessions import Session - from pip._vendor.requests.structures import CaseInsensitiveDict - from pip._vendor.requests.utils import get_netrc_auth - # NOTE: XMLRPC Client is not annotated in typeshed as on 2017-07-17, which is -@@ -69,7 +70,7 @@ def user_agent(): +@@ -77,7 +77,7 @@ def user_agent(): Return a string representing the user agent. """ data = { @@ -19,11 +11,415 @@ index 96f3b65c..cc5b3d15 100644 "python": platform.python_version(), "implementation": { "name": platform.python_implementation(), +diff --git a/pipenv/patched/pip/_internal/index.py b/pipenv/patched/pip/_internal/index.py +index 9eda3a35..67dd952c 100644 +--- a/pipenv/patched/pip/_internal/index.py ++++ b/pipenv/patched/pip/_internal/index.py +@@ -331,6 +331,9 @@ class PackageFinder(object): + # The Session we'll use to make requests + self.session = session + ++ # Kenneth's Hack ++ self.extra = None ++ + # The valid tags to check potential found wheel candidates against + self.valid_tags = get_supported( + versions=versions, +@@ -369,6 +372,23 @@ class PackageFinder(object): + ) + return "\n".join(lines) + ++ @staticmethod ++ def get_extras_links(links): ++ requires = [] ++ extras = {} ++ ++ current_list = requires ++ ++ for link in links: ++ if not link: ++ current_list = requires ++ if link.startswith('['): ++ current_list = [] ++ extras[link[1:-1]] = current_list ++ else: ++ current_list.append(link) ++ return extras ++ + @staticmethod + def _sort_locations(locations, expand_dir=False): + # type: (Sequence[str], bool) -> Tuple[List[str], List[str]] +@@ -427,8 +447,8 @@ class PackageFinder(object): + + return files, urls + +- def _candidate_sort_key(self, candidate): +- # type: (InstallationCandidate) -> CandidateSortingKey ++ def _candidate_sort_key(self, candidate, ignore_compatibility=True): ++ # type: (InstallationCandidate, bool) -> CandidateSortingKey + """ + Function used to generate link sort key for link tuples. + The greater the return value, the more preferred it is. +@@ -448,14 +468,18 @@ class PackageFinder(object): + if candidate.location.is_wheel: + # can raise InvalidWheelFilename + wheel = Wheel(candidate.location.filename) +- if not wheel.supported(self.valid_tags): ++ if not wheel.supported(self.valid_tags) and not ignore_compatibility: + raise UnsupportedWheel( + "%s is not a supported wheel for this platform. It " + "can't be sorted." % wheel.filename + ) + if self.prefer_binary: + binary_preference = 1 +- pri = -(wheel.support_index_min(self.valid_tags)) ++ tags = self.valid_tags if not ignore_compatibility else None ++ try: ++ pri = -(wheel.support_index_min(tags=tags)) ++ except TypeError: ++ pri = -(support_num) + if wheel.build_tag is not None: + match = re.match(r'^(\d+)(.*)$', wheel.build_tag) + build_tag_groups = match.groups() +@@ -608,7 +632,10 @@ class PackageFinder(object): + + page_versions = [] + for page in self._get_pages(url_locations, project_name): +- logger.debug('Analyzing links from page %s', page.url) ++ try: ++ logger.debug('Analyzing links from page %s', page.url) ++ except AttributeError: ++ continue + with indent_log(): + page_versions.extend( + self._package_versions(page.iter_links(), search) +@@ -628,8 +655,8 @@ class PackageFinder(object): + # This is an intentional priority ordering + return file_versions + find_links_versions + page_versions + +- def find_requirement(self, req, upgrade): +- # type: (InstallRequirement, bool) -> Optional[Link] ++ def find_requirement(self, req, upgrade, ignore_compatibility=False): ++ # type: (InstallRequirement, bool, bool) -> Optional[Link] + """Try to find a Link matching req + + Expects req, an InstallRequirement and upgrade, a boolean +@@ -784,8 +811,8 @@ class PackageFinder(object): + logger.debug('Skipping link %s; %s', link, reason) + self.logged_links.add(link) + +- def _link_package_versions(self, link, search): +- # type: (Link, Search) -> Optional[InstallationCandidate] ++ def _link_package_versions(self, link, search, ignore_compatibility=True): ++ # type: (Link, Search, bool) -> Optional[InstallationCandidate] + """Return an InstallationCandidate or None""" + version = None + if link.egg_fragment: +@@ -801,12 +828,12 @@ class PackageFinder(object): + link, 'unsupported archive format: %s' % ext, + ) + return None +- if "binary" not in search.formats and ext == WHEEL_EXTENSION: ++ if "binary" not in search.formats and ext == WHEEL_EXTENSION and not ignore_compatibility: + self._log_skipped_link( + link, 'No binaries permitted for %s' % search.supplied, + ) + return None +- if "macosx10" in link.path and ext == '.zip': ++ if "macosx10" in link.path and ext == '.zip' and not ignore_compatibility: + self._log_skipped_link(link, 'macosx10 one') + return None + if ext == WHEEL_EXTENSION: +@@ -820,7 +847,7 @@ class PackageFinder(object): + link, 'wrong project name (not %s)' % search.supplied) + return None + +- if not wheel.supported(self.valid_tags): ++ if not wheel.supported(self.valid_tags) and not ignore_compatibility: + self._log_skipped_link( + link, 'it is not compatible with this Python') + return None +@@ -856,14 +883,14 @@ class PackageFinder(object): + link.filename, link.requires_python) + support_this_python = True + +- if not support_this_python: ++ if not support_this_python and not ignore_compatibility: + logger.debug("The package %s is incompatible with the python " + "version in use. Acceptable python versions are: %s", + link, link.requires_python) + return None + logger.debug('Found link %s, version: %s', link, version) + +- return InstallationCandidate(search.supplied, version, link) ++ return InstallationCandidate(search.supplied, version, link, link.requires_python) + + + def _find_name_version_sep(egg_info, canonical_name): +diff --git a/pipenv/patched/pip/_internal/models/candidate.py b/pipenv/patched/pip/_internal/models/candidate.py +index 4475458a..6748957d 100644 +--- a/pipenv/patched/pip/_internal/models/candidate.py ++++ b/pipenv/patched/pip/_internal/models/candidate.py +@@ -13,11 +13,12 @@ class InstallationCandidate(KeyBasedCompareMixin): + """Represents a potential "candidate" for installation. + """ + +- def __init__(self, project, version, location): +- # type: (Any, str, Link) -> None ++ def __init__(self, project, version, location, requires_python=None): ++ # type: (Any, str, Link, Any) -> None + self.project = project + self.version = parse_version(version) # type: _BaseVersion + self.location = location ++ self.requires_python = requires_python + + super(InstallationCandidate, self).__init__( + key=(self.project, self.version, self.location), +diff --git a/pipenv/patched/pip/_internal/operations/prepare.py b/pipenv/patched/pip/_internal/operations/prepare.py +index 4f31dd5a..ed0c86b2 100644 +--- a/pipenv/patched/pip/_internal/operations/prepare.py ++++ b/pipenv/patched/pip/_internal/operations/prepare.py +@@ -17,7 +17,7 @@ from pip._internal.exceptions import ( + from pip._internal.utils.compat import expanduser + from pip._internal.utils.hashes import MissingHashes + from pip._internal.utils.logging import indent_log +-from pip._internal.utils.misc import display_path, normalize_path ++from pip._internal.utils.misc import display_path, normalize_path, rmtree + from pip._internal.utils.typing import MYPY_CHECK_RUNNING + from pip._internal.vcs import vcs + +@@ -258,14 +258,7 @@ class RequirementPreparer(object): + # package unpacked in `req.source_dir` + # package unpacked in `req.source_dir` + if os.path.exists(os.path.join(req.source_dir, 'setup.py')): +- raise PreviousBuildDirError( +- "pip can't proceed with requirements '%s' due to a" +- " pre-existing build directory (%s). This is " +- "likely due to a previous installation that failed" +- ". pip is being responsible and not assuming it " +- "can delete this. Please delete it and try again." +- % (req, req.source_dir) +- ) ++ rmtree(req.source_dir) + req.populate_link(finder, upgrade_allowed, require_hashes) + + # We can't hit this spot and have populate_link return None. +diff --git a/pipenv/patched/pip/_internal/pep425tags.py b/pipenv/patched/pip/_internal/pep425tags.py +index 1e782d1a..3c760ca3 100644 +--- a/pipenv/patched/pip/_internal/pep425tags.py ++++ b/pipenv/patched/pip/_internal/pep425tags.py +@@ -10,7 +10,10 @@ import sysconfig + import warnings + from collections import OrderedDict + +-import pip._internal.utils.glibc ++try: ++ import pip._internal.utils.glibc ++except ImportError: ++ import pip.utils.glibc + from pip._internal.utils.compat import get_extension_suffixes + from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +diff --git a/pipenv/patched/pip/_internal/req/req_install.py b/pipenv/patched/pip/_internal/req/req_install.py +index a4834b00..2c22e141 100644 +--- a/pipenv/patched/pip/_internal/req/req_install.py ++++ b/pipenv/patched/pip/_internal/req/req_install.py +@@ -588,7 +588,8 @@ class InstallRequirement(object): + self.setup_py, self.link, + ) + script = SETUPTOOLS_SHIM % self.setup_py +- base_cmd = [sys.executable, '-c', script] ++ sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) ++ base_cmd = [sys_executable, '-c', script] + if self.isolated: + base_cmd += ["--no-user-cfg"] + egg_info_cmd = base_cmd + ['egg_info'] +@@ -746,9 +747,10 @@ class InstallRequirement(object): + with indent_log(): + # FIXME: should we do --install-headers here too? + with self.build_env: ++ sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) + call_subprocess( + [ +- sys.executable, ++ sys_executable, + '-c', + SETUPTOOLS_SHIM % self.setup_py + ] + +@@ -995,7 +997,8 @@ class InstallRequirement(object): + pycompile # type: bool + ): + # type: (...) -> List[str] +- install_args = [sys.executable, "-u"] ++ sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) ++ install_args = [sys_executable, "-u"] + install_args.append('-c') + install_args.append(SETUPTOOLS_SHIM % self.setup_py) + install_args += list(global_options) + \ +diff --git a/pipenv/patched/pip/_internal/req/req_set.py b/pipenv/patched/pip/_internal/req/req_set.py +index d1410e93..69a53bf2 100644 +--- a/pipenv/patched/pip/_internal/req/req_set.py ++++ b/pipenv/patched/pip/_internal/req/req_set.py +@@ -18,7 +18,7 @@ logger = logging.getLogger(__name__) + + class RequirementSet(object): + +- def __init__(self, require_hashes=False, check_supported_wheels=True): ++ def __init__(self, require_hashes=False, check_supported_wheels=True, ignore_compatibility=True): + # type: (bool, bool) -> None + """Create a RequirementSet. + """ +@@ -26,6 +26,9 @@ class RequirementSet(object): + self.requirements = OrderedDict() # type: Dict[str, InstallRequirement] # noqa: E501 + self.require_hashes = require_hashes + self.check_supported_wheels = check_supported_wheels ++ if ignore_compatibility: ++ self.check_supported_wheels = False ++ self.ignore_compatibility = (check_supported_wheels is False or ignore_compatibility is True) + + # Mapping of alias: real_name + self.requirement_aliases = {} # type: Dict[str, str] +@@ -186,7 +189,7 @@ class RequirementSet(object): + return self.requirements[name] + if name in self.requirement_aliases: + return self.requirements[self.requirement_aliases[name]] +- raise KeyError("No project with the name %r" % project_name) ++ pass + + def cleanup_files(self): + # type: () -> None +diff --git a/pipenv/patched/pip/_internal/resolve.py b/pipenv/patched/pip/_internal/resolve.py +index 33f572f1..dfe149ad 100644 +--- a/pipenv/patched/pip/_internal/resolve.py ++++ b/pipenv/patched/pip/_internal/resolve.py +@@ -19,6 +19,7 @@ from pip._internal.exceptions import ( + UnsupportedPythonVersion, + ) + from pip._internal.req.constructors import install_req_from_req_string ++from pip._internal.req.req_install import InstallRequirement + from pip._internal.utils.logging import indent_log + from pip._internal.utils.misc import dist_in_usersite, ensure_dir + from pip._internal.utils.packaging import check_dist_requires_python +@@ -58,7 +59,8 @@ class Resolver(object): + force_reinstall, # type: bool + isolated, # type: bool + upgrade_strategy, # type: str +- use_pep517=None # type: Optional[bool] ++ use_pep517=None, # type: Optional[bool] ++ ignore_compatibility=False, # type: bool + ): + # type: (...) -> None + super(Resolver, self).__init__() +@@ -81,8 +83,12 @@ class Resolver(object): + self.ignore_dependencies = ignore_dependencies + self.ignore_installed = ignore_installed + self.ignore_requires_python = ignore_requires_python ++ self.ignore_compatibility = ignore_compatibility + self.use_user_site = use_user_site + self.use_pep517 = use_pep517 ++ self.requires_python = None ++ if self.ignore_compatibility: ++ self.ignore_requires_python = True + + self._discovered_dependencies = \ + defaultdict(list) # type: DefaultDict[str, List] +@@ -273,7 +279,8 @@ class Resolver(object): + def _resolve_one( + self, + requirement_set, # type: RequirementSet +- req_to_install # type: InstallRequirement ++ req_to_install, # type: InstallRequirement ++ ignore_requires_python=False # type: bool + ): + # type: (...) -> List[InstallRequirement] + """Prepare a single requirements file. +@@ -298,11 +305,18 @@ class Resolver(object): + try: + check_dist_requires_python(dist) + except UnsupportedPythonVersion as err: +- if self.ignore_requires_python: ++ if self.ignore_requires_python or ignore_requires_python or self.ignore_compatibility: + logger.warning(err.args[0]) + else: + raise + ++ # A huge hack, by Kenneth Reitz. ++ try: ++ self.requires_python = check_dist_requires_python(dist, absorb=False) ++ except TypeError: ++ self.requires_python = None ++ ++ + more_reqs = [] # type: List[InstallRequirement] + + def add_req(subreq, extras_requested): +@@ -329,10 +343,14 @@ class Resolver(object): + # We add req_to_install before its dependencies, so that we + # can refer to it when adding dependencies. + if not requirement_set.has_requirement(req_to_install.name): ++ available_requested = sorted( ++ set(dist.extras) & set(req_to_install.extras) ++ ) + # 'unnamed' requirements will get added here + req_to_install.is_direct = True + requirement_set.add_requirement( + req_to_install, parent_req_name=None, ++ extras_requested=available_requested, + ) + + if not self.ignore_dependencies: +@@ -356,6 +374,20 @@ class Resolver(object): + for subreq in dist.requires(available_requested): + add_req(subreq, extras_requested=available_requested) + ++ # Hack for deep-resolving extras. ++ for available in available_requested: ++ if hasattr(dist, '_DistInfoDistribution__dep_map'): ++ for req in dist._DistInfoDistribution__dep_map[available]: ++ req = InstallRequirement( ++ req, ++ req_to_install, ++ isolated=self.isolated, ++ wheel_cache=self.wheel_cache, ++ use_pep517=None ++ ) ++ ++ more_reqs.append(req) ++ + if not req_to_install.editable and not req_to_install.satisfied_by: + # XXX: --no-install leads this to report 'Successfully + # downloaded' for only non-editable reqs, even though we took +diff --git a/pipenv/patched/pip/_internal/utils/packaging.py b/pipenv/patched/pip/_internal/utils/packaging.py +index 7aaf7b5e..d56f0512 100644 +--- a/pipenv/patched/pip/_internal/utils/packaging.py ++++ b/pipenv/patched/pip/_internal/utils/packaging.py +@@ -37,7 +37,7 @@ def check_requires_python(requires_python): + requires_python_specifier = specifiers.SpecifierSet(requires_python) + + # We only use major.minor.micro +- python_version = version.parse('.'.join(map(str, sys.version_info[:3]))) ++ python_version = version.parse('{0}.{1}.{2}'.format(*sys.version_info[:3])) + return python_version in requires_python_specifier + + +@@ -57,9 +57,11 @@ def get_metadata(dist): + return feed_parser.close() + + +-def check_dist_requires_python(dist): ++def check_dist_requires_python(dist, absorb=False): + pkg_info_dict = get_metadata(dist) + requires_python = pkg_info_dict.get('Requires-Python') ++ if absorb: ++ return requires_python + try: + if not check_requires_python(requires_python): + raise exceptions.UnsupportedPythonVersion( diff --git a/pipenv/patched/pip/_internal/utils/temp_dir.py b/pipenv/patched/pip/_internal/utils/temp_dir.py -index edc506bf..84d57dac 100644 +index 2c81ad55..ff2ccc5a 100644 --- a/pipenv/patched/pip/_internal/utils/temp_dir.py +++ b/pipenv/patched/pip/_internal/utils/temp_dir.py -@@ -3,8 +3,10 @@ from __future__ import absolute_import +@@ -5,8 +5,10 @@ import itertools import logging import os.path import tempfile @@ -34,7 +430,7 @@ index edc506bf..84d57dac 100644 logger = logging.getLogger(__name__) -@@ -45,6 +47,20 @@ class TempDirectory(object): +@@ -47,6 +49,20 @@ class TempDirectory(object): self.path = path self.delete = delete self.kind = kind @@ -48,14 +444,14 @@ index edc506bf..84d57dac 100644 + self, + self._cleanup, + self.path, -+ warn_message=None ++ warn_message = None + ) + else: + self._finalizer = None def __repr__(self): return "<{} {!r}>".format(self.__class__.__name__, self.path) -@@ -72,11 +88,27 @@ class TempDirectory(object): +@@ -74,14 +90,30 @@ class TempDirectory(object): self.path = os.path.realpath( tempfile.mkdtemp(prefix="pip-{}-".format(self.kind)) ) @@ -86,424 +482,20 @@ index edc506bf..84d57dac 100644 + pass + else: + self.path = None -diff --git a/pipenv/patched/pip/_internal/index.py b/pipenv/patched/pip/_internal/index.py -index 8c2f24f1..cdd48874 100644 ---- a/pipenv/patched/pip/_internal/index.py -+++ b/pipenv/patched/pip/_internal/index.py -@@ -246,6 +246,9 @@ class PackageFinder(object): - # The Session we'll use to make requests - self.session = session -+ # Kenneth's Hack -+ self.extra = None -+ - # The valid tags to check potential found wheel candidates against - self.valid_tags = get_supported( - versions=versions, -@@ -298,6 +301,25 @@ class PackageFinder(object): + + class AdjacentTempDirectory(TempDirectory): +@@ -152,4 +184,5 @@ class AdjacentTempDirectory(TempDirectory): + self.path = os.path.realpath( + tempfile.mkdtemp(prefix="pip-{}-".format(self.kind)) ) - self.dependency_links.extend(links) - -+ @staticmethod -+ def get_extras_links(links): -+ requires = [] -+ extras = {} -+ -+ current_list = requires -+ -+ for link in links: -+ if not link: -+ current_list = requires -+ if link.startswith('['): -+ current_list = [] -+ extras[link[1:-1]] = current_list -+ else: -+ current_list.append(link) -+ -+ return extras -+ -+ - @staticmethod - def _sort_locations(locations, expand_dir=False): - """ -@@ -350,7 +372,7 @@ class PackageFinder(object): - - return files, urls - -- def _candidate_sort_key(self, candidate): -+ def _candidate_sort_key(self, candidate, ignore_compatibility=True): - """ - Function used to generate link sort key for link tuples. - The greater the return value, the more preferred it is. -@@ -370,14 +392,19 @@ class PackageFinder(object): - if candidate.location.is_wheel: - # can raise InvalidWheelFilename - wheel = Wheel(candidate.location.filename) -- if not wheel.supported(self.valid_tags): -+ if not wheel.supported(self.valid_tags) and not ignore_compatibility: - raise UnsupportedWheel( - "%s is not a supported wheel for this platform. It " - "can't be sorted." % wheel.filename - ) - if self.prefer_binary: - binary_preference = 1 -- pri = -(wheel.support_index_min(self.valid_tags)) -+ tags = self.valid_tags if not ignore_compatibility else None -+ try: -+ pri = -(wheel.support_index_min(tags=tags)) -+ except TypeError: -+ pri = -(support_num) -+ - if wheel.build_tag is not None: - match = re.match(r'^(\d+)(.*)$', wheel.build_tag) - build_tag_groups = match.groups() -@@ -528,7 +555,10 @@ class PackageFinder(object): - - page_versions = [] - for page in self._get_pages(url_locations, project_name): -- logger.debug('Analyzing links from page %s', page.url) -+ try: -+ logger.debug('Analyzing links from page %s', page.url) -+ except AttributeError: -+ continue - with indent_log(): - page_versions.extend( - self._package_versions(page.iter_links(), search) -@@ -562,7 +592,7 @@ class PackageFinder(object): - dependency_versions - ) - -- def find_requirement(self, req, upgrade): -+ def find_requirement(self, req, upgrade, ignore_compatibility=False): - """Try to find a Link matching req - - Expects req, an InstallRequirement and upgrade, a boolean -@@ -672,7 +702,10 @@ class PackageFinder(object): - continue - seen.add(location) - -- page = self._get_page(location) -+ try: -+ page = self._get_page(location) -+ except requests.HTTPError: -+ continue - if page is None: - continue - -@@ -709,7 +742,7 @@ class PackageFinder(object): - logger.debug('Skipping link %s; %s', link, reason) - self.logged_links.add(link) - -- def _link_package_versions(self, link, search): -+ def _link_package_versions(self, link, search, ignore_compatibility=True): - """Return an InstallationCandidate or None""" - version = None - if link.egg_fragment: -@@ -725,12 +758,12 @@ class PackageFinder(object): - link, 'unsupported archive format: %s' % ext, - ) - return -- if "binary" not in search.formats and ext == wheel_ext: -+ if "binary" not in search.formats and ext == wheel_ext and not ignore_compatibility: - self._log_skipped_link( - link, 'No binaries permitted for %s' % search.supplied, - ) - return -- if "macosx10" in link.path and ext == '.zip': -+ if "macosx10" in link.path and ext == '.zip' and not ignore_compatibility: - self._log_skipped_link(link, 'macosx10 one') - return - if ext == wheel_ext: -@@ -744,7 +777,7 @@ class PackageFinder(object): - link, 'wrong project name (not %s)' % search.supplied) - return - -- if not wheel.supported(self.valid_tags): -+ if not wheel.supported(self.valid_tags) and not ignore_compatibility: - self._log_skipped_link( - link, 'it is not compatible with this Python') - return -@@ -780,14 +813,14 @@ class PackageFinder(object): - link.filename, link.requires_python) - support_this_python = True - -- if not support_this_python: -+ if not support_this_python and not ignore_compatibility: - logger.debug("The package %s is incompatible with the python" - "version in use. Acceptable python versions are:%s", - link, link.requires_python) - return - logger.debug('Found link %s, version: %s', link, version) - -- return InstallationCandidate(search.supplied, version, link) -+ return InstallationCandidate(search.supplied, version, link, link.requires_python) - - def _get_page(self, link): - return _get_html_page(link, session=self.session) -diff --git a/pipenv/patched/pip/_internal/models/candidate.py b/pipenv/patched/pip/_internal/models/candidate.py -index c736de6c..a78566c1 100644 ---- a/pipenv/patched/pip/_internal/models/candidate.py -+++ b/pipenv/patched/pip/_internal/models/candidate.py -@@ -7,10 +7,11 @@ class InstallationCandidate(KeyBasedCompareMixin): - """Represents a potential "candidate" for installation. - """ - -- def __init__(self, project, version, location): -+ def __init__(self, project, version, location, requires_python=None): - self.project = project - self.version = parse_version(version) - self.location = location -+ self.requires_python = requires_python - - super(InstallationCandidate, self).__init__( - key=(self.project, self.version, self.location), -diff --git a/pipenv/patched/pip/_internal/operations/prepare.py b/pipenv/patched/pip/_internal/operations/prepare.py -index 104bea33..ecf78b9a 100644 ---- a/pipenv/patched/pip/_internal/operations/prepare.py -+++ b/pipenv/patched/pip/_internal/operations/prepare.py -@@ -17,7 +17,7 @@ from pip._internal.exceptions import ( - from pip._internal.utils.compat import expanduser - from pip._internal.utils.hashes import MissingHashes - from pip._internal.utils.logging import indent_log --from pip._internal.utils.misc import display_path, normalize_path -+from pip._internal.utils.misc import display_path, normalize_path, rmtree - from pip._internal.vcs import vcs - - logger = logging.getLogger(__name__) -@@ -123,7 +123,11 @@ class IsSDist(DistAbstraction): - " and ".join(map(repr, sorted(missing))) - ) - -- self.req.run_egg_info() -+ try: -+ self.req.run_egg_info() -+ except (OSError, TypeError): -+ self.req._correct_build_location() -+ self.req.run_egg_info() - self.req.assert_source_matches_version() - - -@@ -205,16 +209,8 @@ class RequirementPreparer(object): - # installation. - # FIXME: this won't upgrade when there's an existing - # package unpacked in `req.source_dir` -- # package unpacked in `req.source_dir` - if os.path.exists(os.path.join(req.source_dir, 'setup.py')): -- raise PreviousBuildDirError( -- "pip can't proceed with requirements '%s' due to a" -- " pre-existing build directory (%s). This is " -- "likely due to a previous installation that failed" -- ". pip is being responsible and not assuming it " -- "can delete this. Please delete it and try again." -- % (req, req.source_dir) -- ) -+ rmtree(req.source_dir) - req.populate_link(finder, upgrade_allowed, require_hashes) - - # We can't hit this spot and have populate_link return None. -diff --git a/pipenv/patched/pip/_internal/pep425tags.py b/pipenv/patched/pip/_internal/pep425tags.py -index ab1a0298..763c0a24 100644 ---- a/pipenv/patched/pip/_internal/pep425tags.py -+++ b/pipenv/patched/pip/_internal/pep425tags.py -@@ -10,7 +10,11 @@ import sysconfig - import warnings - from collections import OrderedDict - --import pip._internal.utils.glibc -+try: -+ import pip._internal.utils.glibc -+except ImportError: -+ import pip.utils.glibc -+ - from pip._internal.utils.compat import get_extension_suffixes - - logger = logging.getLogger(__name__) -diff --git a/pipenv/patched/pip/_internal/req/req_install.py b/pipenv/patched/pip/_internal/req/req_install.py -index c2624fee..ee75acd6 100644 ---- a/pipenv/patched/pip/_internal/req/req_install.py -+++ b/pipenv/patched/pip/_internal/req/req_install.py -@@ -452,7 +452,8 @@ class InstallRequirement(object): - - with indent_log(): - script = SETUPTOOLS_SHIM % self.setup_py -- base_cmd = [sys.executable, '-c', script] -+ sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) -+ base_cmd = [sys_executable, '-c', script] - if self.isolated: - base_cmd += ["--no-user-cfg"] - egg_info_cmd = base_cmd + ['egg_info'] -@@ -613,10 +614,11 @@ class InstallRequirement(object): - - with indent_log(): - # FIXME: should we do --install-headers here too? -+ sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) - with self.build_env: - call_subprocess( - [ -- sys.executable, -+ sys_executable, - '-c', - SETUPTOOLS_SHIM % self.setup_py - ] + -@@ -834,7 +836,8 @@ class InstallRequirement(object): - - def get_install_args(self, global_options, record_filename, root, prefix, - pycompile): -- install_args = [sys.executable, "-u"] -+ sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) -+ install_args = [sys_executable, "-u"] - install_args.append('-c') - install_args.append(SETUPTOOLS_SHIM % self.setup_py) - install_args += list(global_options) + \ -diff --git a/pipenv/patched/pip/_internal/req/req_set.py b/pipenv/patched/pip/_internal/req/req_set.py -index b1983171..0bab231d 100644 ---- a/pipenv/patched/pip/_internal/req/req_set.py -+++ b/pipenv/patched/pip/_internal/req/req_set.py -@@ -12,13 +12,16 @@ logger = logging.getLogger(__name__) - - class RequirementSet(object): - -- def __init__(self, require_hashes=False, check_supported_wheels=True): -+ def __init__(self, require_hashes=False, check_supported_wheels=True, ignore_compatibility=True): - """Create a RequirementSet. - """ - - self.requirements = OrderedDict() - self.require_hashes = require_hashes - self.check_supported_wheels = check_supported_wheels -+ if ignore_compatibility: -+ self.check_supported_wheels = False -+ self.ignore_compatibility = True if (check_supported_wheels is False or ignore_compatibility is True) else False - - # Mapping of alias: real_name - self.requirement_aliases = {} -@@ -171,7 +174,7 @@ class RequirementSet(object): - return self.requirements[name] - if name in self.requirement_aliases: - return self.requirements[self.requirement_aliases[name]] -- raise KeyError("No project with the name %r" % project_name) -+ pass - - def cleanup_files(self): - """Clean up files, remove builds.""" -diff --git a/pipenv/patched/pip/_internal/resolve.py b/pipenv/patched/pip/_internal/resolve.py -index 2d9f1c56..bedc2582 100644 ---- a/pipenv/patched/pip/_internal/resolve.py -+++ b/pipenv/patched/pip/_internal/resolve.py -@@ -35,7 +35,7 @@ class Resolver(object): - - def __init__(self, preparer, session, finder, wheel_cache, use_user_site, - ignore_dependencies, ignore_installed, ignore_requires_python, -- force_reinstall, isolated, upgrade_strategy): -+ force_reinstall, isolated, upgrade_strategy, ignore_compatibility=False): - super(Resolver, self).__init__() - assert upgrade_strategy in self._allowed_strategies - -@@ -55,7 +55,11 @@ class Resolver(object): - self.ignore_dependencies = ignore_dependencies - self.ignore_installed = ignore_installed - self.ignore_requires_python = ignore_requires_python -+ self.ignore_compatibility = ignore_compatibility - self.use_user_site = use_user_site -+ self.requires_python = None -+ if self.ignore_compatibility: -+ self.ignore_requires_python = True - - self._discovered_dependencies = defaultdict(list) - -@@ -237,7 +241,7 @@ class Resolver(object): - - return abstract_dist - -- def _resolve_one(self, requirement_set, req_to_install): -+ def _resolve_one(self, requirement_set, req_to_install, ignore_requires_python=False): - """Prepare a single requirements file. - - :return: A list of additional InstallRequirements to also install. -@@ -260,11 +264,17 @@ class Resolver(object): - try: - check_dist_requires_python(dist) - except UnsupportedPythonVersion as err: -- if self.ignore_requires_python: -+ if self.ignore_requires_python or self.ignore_compatibility: - logger.warning(err.args[0]) - else: - raise - -+ # A huge hack, by Kenneth Reitz. -+ try: -+ self.requires_python = check_dist_requires_python(dist, absorb=False) -+ except TypeError: -+ self.requires_python = None -+ - more_reqs = [] - - def add_req(subreq, extras_requested): -@@ -291,9 +301,13 @@ class Resolver(object): - # can refer to it when adding dependencies. - if not requirement_set.has_requirement(req_to_install.name): - # 'unnamed' requirements will get added here -+ available_requested = sorted( -+ set(dist.extras) & set(req_to_install.extras) -+ ) - req_to_install.is_direct = True - requirement_set.add_requirement( - req_to_install, parent_req_name=None, -+ extras_requested=available_requested, - ) - - if not self.ignore_dependencies: -@@ -317,6 +331,19 @@ class Resolver(object): - for subreq in dist.requires(available_requested): - add_req(subreq, extras_requested=available_requested) - -+ # Hack for deep-resolving extras. -+ for available in available_requested: -+ if hasattr(dist, '_DistInfoDistribution__dep_map'): -+ for req in dist._DistInfoDistribution__dep_map[available]: -+ req = install_req_from_req( -+ str(req), -+ req_to_install, -+ isolated=self.isolated, -+ wheel_cache=self.wheel_cache, -+ ) -+ -+ more_reqs.append(req) -+ - if not req_to_install.editable and not req_to_install.satisfied_by: - # XXX: --no-install leads this to report 'Successfully - # downloaded' for only non-editable reqs, even though we took -diff --git a/pipenv/patched/pip/_internal/utils/packaging.py b/pipenv/patched/pip/_internal/utils/packaging.py -index c43142f0..f241cce0 100644 ---- a/pipenv/patched/pip/_internal/utils/packaging.py -+++ b/pipenv/patched/pip/_internal/utils/packaging.py -@@ -29,7 +29,7 @@ def check_requires_python(requires_python): - requires_python_specifier = specifiers.SpecifierSet(requires_python) - - # We only use major.minor.micro -- python_version = version.parse('.'.join(map(str, sys.version_info[:3]))) -+ python_version = version.parse('{0}.{1}.{2}'.format(*sys.version_info[:3])) - return python_version in requires_python_specifier - - -@@ -48,9 +48,11 @@ def get_metadata(dist): - return feed_parser.close() - - --def check_dist_requires_python(dist): -+def check_dist_requires_python(dist, absorb=True): - pkg_info_dict = get_metadata(dist) - requires_python = pkg_info_dict.get('Requires-Python') -+ if absorb: -+ return requires_python - try: - if not check_requires_python(requires_python): - raise exceptions.UnsupportedPythonVersion( ++ self._register_finalizer() + logger.debug("Created temporary directory: {}".format(self.path)) diff --git a/pipenv/patched/pip/_internal/wheel.py b/pipenv/patched/pip/_internal/wheel.py -index 5ce890eb..46c0181c 100644 +index 67bcc7f7..968cdff9 100644 --- a/pipenv/patched/pip/_internal/wheel.py +++ b/pipenv/patched/pip/_internal/wheel.py -@@ -83,7 +83,7 @@ def fix_script(path): +@@ -114,7 +114,7 @@ def fix_script(path): firstline = script.readline() if not firstline.startswith(b'#!python'): return False @@ -512,7 +504,7 @@ index 5ce890eb..46c0181c 100644 firstline = b'#!' + exename + os.linesep.encode("ascii") rest = script.read() with open(path, 'wb') as script: -@@ -167,7 +167,8 @@ def message_about_scripts_not_on_PATH(scripts): +@@ -201,7 +201,8 @@ def message_about_scripts_not_on_PATH(scripts): ] # If an executable sits with sys.executable, we don't warn for it. # This covers the case of venv invocations without activating the venv. @@ -522,7 +514,7 @@ index 5ce890eb..46c0181c 100644 warn_for = { parent_dir: scripts for parent_dir, scripts in grouped_by_dir.items() if os.path.normcase(parent_dir) not in not_warn_dirs -@@ -667,8 +668,9 @@ class WheelBuilder(object): +@@ -901,8 +902,9 @@ class WheelBuilder(object): # isolating. Currently, it breaks Python in virtualenvs, because it # relies on site.py to find parts of the standard library outside the # virtualenv. @@ -533,3 +525,18 @@ index 5ce890eb..46c0181c 100644 SETUPTOOLS_SHIM % req.setup_py ] + list(self.global_options) +diff --git a/pipenv/patched/pip/_internal/utils/misc.py b/pipenv/patched/pip/_internal/utils/misc.py +index 84605ee3..649311c0 100644 +--- a/pipenv/patched/pip/_internal/utils/misc.py ++++ b/pipenv/patched/pip/_internal/utils/misc.py +@@ -117,8 +117,8 @@ def get_prog(): + @retry(stop_max_delay=3000, wait_fixed=500) + def rmtree(dir, ignore_errors=False): + # type: (str, bool) -> None +- shutil.rmtree(dir, ignore_errors=ignore_errors, +- onerror=rmtree_errorhandler) ++ from pipenv.vendor.vistir.path import rmtree as vistir_rmtree, handle_remove_readonly ++ vistir_rmtree(dir, onerror=handle_remove_readonly, ignore_errors=ignore_errors) + + + def rmtree_errorhandler(func, path, exc_info): diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index 3799ccf4..52d66f51 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -1,33 +1,28 @@ diff --git a/pipenv/patched/piptools/_compat/__init__.py b/pipenv/patched/piptools/_compat/__init__.py -index 1fa3805..c0ecec8 100644 +index e4ac717..19adcbc 100644 --- a/pipenv/patched/piptools/_compat/__init__.py +++ b/pipenv/patched/piptools/_compat/__init__.py -@@ -27,6 +27,8 @@ from .pip_compat import ( - cmdoptions, - get_installed_distributions, - PyPI, -+ SafeFileCache, -+ InstallationError, - install_req_from_line, +@@ -31,4 +31,6 @@ from .pip_compat import ( install_req_from_editable, + stdlib_pkgs, + DEV_PKGS, ++ SafeFileCache, ++ InstallationError ) diff --git a/pipenv/patched/piptools/_compat/pip_compat.py b/pipenv/patched/piptools/_compat/pip_compat.py -index 28da51f..c466ef0 100644 +index 82ccb8b..715144a 100644 --- a/pipenv/patched/piptools/_compat/pip_compat.py +++ b/pipenv/patched/piptools/_compat/pip_compat.py -@@ -1,45 +1,55 @@ +@@ -1,48 +1,51 @@ # -*- coding=utf-8 -*- -import importlib - --import pip --import pkg_resources +__all__ = [ + "InstallRequirement", + "parse_requirements", + "RequirementSet", -+ "user_cache_dir", + "FAVORITE_HASH", + "is_file_url", ++ "path_to_url", + "url_to_path", + "PackageFinder", + "FormatControl", @@ -36,15 +31,20 @@ index 28da51f..c466ef0 100644 + "cmdoptions", + "get_installed_distributions", + "PyPI", -+ "SafeFileCache", -+ "InstallationError", -+ "parse_version", -+ "pip_version", -+ "install_req_from_editable", ++ "stdlib_pkgs", ++ "DEV_PKGS", + "install_req_from_line", -+ "user_cache_dir" ++ "install_req_from_editable", ++ "user_cache_dir", ++ "SafeFileCache", ++ "InstallationError" +] +-import pip +-import pkg_resources ++import os ++os.environ["PIP_SHIMS_BASE_MODULE"] = str("pipenv.patched.notpip") + -def do_import(module_path, subimport=None, old_path=None): - old_path = old_path or module_path - prefixes = ["pip._internal", "pip"] @@ -68,6 +68,7 @@ index 28da51f..c466ef0 100644 -user_cache_dir = do_import('utils.appdirs', 'user_cache_dir') -FAVORITE_HASH = do_import('utils.hashes', 'FAVORITE_HASH') -is_file_url = do_import('download', 'is_file_url') +-path_to_url = do_import('download', 'path_to_url') -url_to_path = do_import('download', 'url_to_path') -PackageFinder = do_import('index', 'PackageFinder') -FormatControl = do_import('index', 'FormatControl') @@ -76,13 +77,23 @@ index 28da51f..c466ef0 100644 -cmdoptions = do_import('cli.cmdoptions', old_path='cmdoptions') -get_installed_distributions = do_import('utils.misc', 'get_installed_distributions', old_path='utils') -PyPI = do_import('models.index', 'PyPI') -+from pipenv.vendor.appdirs import user_cache_dir +-stdlib_pkgs = do_import('utils.compat', 'stdlib_pkgs', old_path='compat') +-DEV_PKGS = do_import('commands.freeze', 'DEV_PKGS') +- +-# pip 18.1 has refactored InstallRequirement constructors use by pip-tools. +-if pkg_resources.parse_version(pip.__version__) < pkg_resources.parse_version('18.1'): +- install_req_from_line = InstallRequirement.from_line +- install_req_from_editable = InstallRequirement.from_editable +-else: +- install_req_from_line = do_import('req.constructors', 'install_req_from_line') +- install_req_from_editable = do_import('req.constructors', 'install_req_from_editable') +from pip_shims.shims import ( + InstallRequirement, + parse_requirements, + RequirementSet, + FAVORITE_HASH, + is_file_url, ++ path_to_url, + url_to_path, + PackageFinder, + FormatControl, @@ -91,50 +102,74 @@ index 28da51f..c466ef0 100644 + cmdoptions, + get_installed_distributions, + PyPI, ++ stdlib_pkgs, ++ DEV_PKGS, ++ install_req_from_line, ++ install_req_from_editable, ++ USER_CACHE_DIR as user_cache_dir, + SafeFileCache, -+ InstallationError, -+ parse_version, -+ pip_version, ++ InstallationError +) +diff --git a/pipenv/patched/piptools/locations.py b/pipenv/patched/piptools/locations.py +index 4e6174c..9e0c6f1 100644 +--- a/pipenv/patched/piptools/locations.py ++++ b/pipenv/patched/piptools/locations.py +@@ -5,7 +5,11 @@ from .click import secho + from ._compat import user_cache_dir - # pip 18.1 has refactored InstallRequirement constructors use by pip-tools. --if pkg_resources.parse_version(pip.__version__) < pkg_resources.parse_version('18.1'): -+if parse_version(pip_version) < parse_version('18.1'): - install_req_from_line = InstallRequirement.from_line - install_req_from_editable = InstallRequirement.from_editable - else: -- install_req_from_line = do_import('req.constructors', 'install_req_from_line') -- install_req_from_editable = do_import('req.constructors', 'install_req_from_editable') -+ from pip_shims.shims import ( -+ install_req_from_editable, install_req_from_line -+ ) + # The user_cache_dir helper comes straight from pip itself +-CACHE_DIR = user_cache_dir('pip-tools') ++try: ++ from pipenv.environments import PIPENV_CACHE_DIR ++ CACHE_DIR = PIPENV_CACHE_DIR ++except ImportError: ++ CACHE_DIR = user_cache_dir('pipenv') + + # NOTE + # We used to store the cache dir under ~/.pip-tools, which is not the diff --git a/pipenv/patched/piptools/repositories/local.py b/pipenv/patched/piptools/repositories/local.py -index 08dabe1..480ad1e 100644 +index 08dabe1..36bafdb 100644 --- a/pipenv/patched/piptools/repositories/local.py +++ b/pipenv/patched/piptools/repositories/local.py -@@ -56,7 +56,7 @@ class LocalRequirementsRepository(BaseRepository): +@@ -56,7 +56,8 @@ class LocalRequirementsRepository(BaseRepository): if existing_pin and ireq_satisfied_by_existing_pin(ireq, existing_pin): project, version, _ = as_tuple(existing_pin) return make_install_requirement( - project, version, ireq.extras, constraint=ireq.constraint -+ project, version, ireq.extras, constraint=ireq.constraint, markers=ireq.markers ++ project, version, ireq.extras, constraint=ireq.constraint, ++ markers=ireq.markers ) else: return self.repository.find_best_match(ireq, prereleases) diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py -index bf69803..31b85b9 100644 +index e54ae08..75b8208 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py -@@ -1,7 +1,7 @@ - # coding: utf-8 +@@ -2,14 +2,22 @@ from __future__ import (absolute_import, division, print_function, unicode_literals) -- + +import copy import hashlib import os from contextlib import contextmanager -@@ -15,13 +15,22 @@ from .._compat import ( + from shutil import rmtree + +-import pip + import pkg_resources + ++from packaging.requirements import Requirement ++from packaging.specifiers import SpecifierSet, Specifier ++ ++os.environ["PIP_SHIMS_BASE_MODULE"] = str("pipenv.patched.notpip") ++import pip_shims ++from pip_shims.shims import VcsSupport, WheelCache, InstallationError, pip_version ++ ++ + from .._compat import ( + is_file_url, + url_to_path, +@@ -18,13 +26,15 @@ from .._compat import ( Wheel, FAVORITE_HASH, TemporaryDirectory, @@ -143,25 +178,17 @@ index bf69803..31b85b9 100644 + InstallRequirement, + SafeFileCache ) -+os.environ["PIP_SHIMS_BASE_MODULE"] = str("pip") -+from pip_shims.shims import do_import, VcsSupport, WheelCache -+from packaging.requirements import Requirement -+from packaging.specifiers import SpecifierSet, Specifier -+InstallationError = do_import(("exceptions.InstallationError", "7.0", "9999")) -+from pip._internal.resolve import Resolver as PipResolver -+ --from ..cache import CACHE_DIR -+from pipenv.environments import PIPENV_CACHE_DIR as CACHE_DIR + from ..cache import CACHE_DIR from ..exceptions import NoCandidateFound -from ..utils import (fs_str, is_pinned_requirement, lookup_table, - make_install_requirement) +from ..utils import (fs_str, is_pinned_requirement, lookup_table, dedup, -+ make_install_requirement, clean_requires_python) ++ make_install_requirement, clean_requires_python) from .base import BaseRepository try: -@@ -31,10 +40,44 @@ except ImportError: +@@ -34,10 +44,44 @@ except ImportError: def RequirementTracker(): yield @@ -210,18 +237,29 @@ index bf69803..31b85b9 100644 class PyPIRepository(BaseRepository): -@@ -46,8 +89,9 @@ class PyPIRepository(BaseRepository): +@@ -49,10 +93,11 @@ class PyPIRepository(BaseRepository): config), but any other PyPI mirror can be used if index_urls is changed/configured on the Finder. """ -- def __init__(self, pip_options, session): -+ def __init__(self, pip_options, session, use_json=False): +- def __init__(self, pip_options, session, build_isolation=False): ++ def __init__(self, pip_options, session, build_isolation=False, use_json=False): self.session = session -+ self.use_json = use_json self.pip_options = pip_options + self.build_isolation = build_isolation ++ self.use_json = use_json index_urls = [pip_options.index_url] + pip_options.extra_index_urls -@@ -73,6 +117,10 @@ class PyPIRepository(BaseRepository): + if pip_options.no_index: +@@ -67,7 +112,7 @@ class PyPIRepository(BaseRepository): + } + + # pip 19.0 has removed process_dependency_links from the PackageFinder constructor +- if pkg_resources.parse_version(pip.__version__) < pkg_resources.parse_version('19.0'): ++ if pkg_resources.parse_version(pip_version) < pkg_resources.parse_version('19.0'): + finder_kwargs["process_dependency_links"] = pip_options.process_dependency_links + + self.finder = PackageFinder(**finder_kwargs) +@@ -82,6 +127,10 @@ class PyPIRepository(BaseRepository): # of all secondary dependencies for the given requirement, so we # only have to go to disk once for each requirement self._dependencies_cache = {} @@ -232,7 +270,7 @@ index bf69803..31b85b9 100644 # Setup file paths self.freshen_build_caches() -@@ -113,10 +161,13 @@ class PyPIRepository(BaseRepository): +@@ -122,10 +171,13 @@ class PyPIRepository(BaseRepository): if ireq.editable: return ireq # return itself as the best match @@ -240,22 +278,36 @@ index bf69803..31b85b9 100644 + all_candidates = clean_requires_python(self.find_all_candidates(ireq.name)) candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version, unique=True) - matching_versions = ireq.specifier.filter((candidate.version for candidate in all_candidates), +- prereleases=prereleases) + try: + matching_versions = ireq.specifier.filter((candidate.version for candidate in all_candidates), - prereleases=prereleases) ++ prereleases=prereleases) + except TypeError: + matching_versions = [candidate.version for candidate in all_candidates] # Reuses pip's internal candidate sort key to sort matching_candidates = [candidates_by_version[ver] for ver in matching_versions] -@@ -126,25 +177,87 @@ class PyPIRepository(BaseRepository): +@@ -135,14 +187,75 @@ class PyPIRepository(BaseRepository): # Turn the candidate into a pinned InstallRequirement return make_install_requirement( - best_candidate.project, best_candidate.version, ireq.extras, constraint=ireq.constraint -- ) -+ best_candidate.project, best_candidate.version, ireq.extras, ireq.markers, constraint=ireq.constraint -+ ) ++ best_candidate.project, best_candidate.version, ireq.extras, ireq.markers, constraint=ireq.constraint + ) + ++ def get_dependencies(self, ireq): ++ json_results = set() ++ ++ if self.use_json: ++ try: ++ json_results = self.get_json_dependencies(ireq) ++ except TypeError: ++ json_results = set() ++ ++ legacy_results = self.get_legacy_dependencies(ireq) ++ json_results.update(legacy_results) ++ ++ return json_results + + def get_json_dependencies(self, ireq): + @@ -294,42 +346,26 @@ index bf69803..31b85b9 100644 + try: + if ireq not in self._json_dep_cache: + self._json_dep_cache[ireq] = [g for g in gen(ireq)] - -- def resolve_reqs(self, download_dir, ireq, wheel_cache): ++ + return set(self._json_dep_cache[ireq]) + except Exception: + return set() + -+ def get_dependencies(self, ireq): -+ json_results = set() -+ -+ if self.use_json: -+ try: -+ json_results = self.get_json_dependencies(ireq) -+ except TypeError: -+ json_results = set() -+ -+ legacy_results = self.get_legacy_dependencies(ireq) -+ json_results.update(legacy_results) -+ -+ return json_results -+ -+ def resolve_reqs(self, download_dir, ireq, wheel_cache, setup_requires={}, dist=None): + def resolve_reqs(self, download_dir, ireq, wheel_cache): results = None -+ setup_requires = {} -+ dist = None -+ ireq.isolated = False ++ ireq.isolated = self.build_isolation + ireq._wheel_cache = wheel_cache -+ ++ if ireq and not ireq.link: ++ ireq.populate_link(self.finder, False, False) ++ if ireq.link and not ireq.link.is_wheel: ++ ireq.ensure_has_source_dir(self.source_dir) try: from pip._internal.operations.prepare import RequirementPreparer - from pip._internal.resolve import Resolver as PipResolver except ImportError: -- # Pip 9 and below -+ # Pip 9 and below + # Pip 9 and below reqset = RequirementSet( - self.build_dir, - self.source_dir, +@@ -151,9 +264,11 @@ class PyPIRepository(BaseRepository): download_dir=download_dir, wheel_download_dir=self._wheel_download_dir, session=self.session, @@ -342,31 +378,31 @@ index bf69803..31b85b9 100644 else: # pip >= 10 preparer_kwargs = { -@@ -159,13 +272,14 @@ class PyPIRepository(BaseRepository): - 'finder': self.finder, - 'session': self.session, +@@ -170,11 +285,13 @@ class PyPIRepository(BaseRepository): 'upgrade_strategy': "to-satisfy-only", -- 'force_reinstall': False, -+ 'force_reinstall': True, + 'force_reinstall': False, 'ignore_dependencies': False, - 'ignore_requires_python': False, + 'ignore_requires_python': True, 'ignore_installed': True, - 'isolated': False, +- 'isolated': False, ++ 'ignore_compatibility': False, ++ 'isolated': True, 'wheel_cache': wheel_cache, - 'use_user_site': False + 'use_user_site': False, -+ 'ignore_compatibility': False ++ 'use_pep517': True } resolver = None preparer = None -@@ -177,15 +291,109 @@ class PyPIRepository(BaseRepository): +@@ -186,15 +303,21 @@ class PyPIRepository(BaseRepository): resolver_kwargs['preparer'] = preparer reqset = RequirementSet() ireq.is_direct = True - reqset.add_requirement(ireq) +- resolver = PipResolver(**resolver_kwargs) + # reqset.add_requirement(ireq) - resolver = PipResolver(**resolver_kwargs) ++ resolver = pip_shims.shims.Resolver(**resolver_kwargs) resolver.require_hashes = False results = resolver._resolve_one(reqset, ireq) - reqset.cleanup_files() @@ -378,111 +414,15 @@ index bf69803..31b85b9 100644 + cleanup_fn() + except OSError: + pass -+ -+ if ireq.editable and (not ireq.source_dir or not os.path.exists(ireq.source_dir)): -+ if ireq.editable: -+ self._source_dir = TemporaryDirectory(fs_str("source")) -+ ireq.ensure_has_source_dir(self.source_dir) -+ -+ if ireq.editable and (ireq.source_dir and os.path.exists(ireq.source_dir)): -+ # Collect setup_requires info from local eggs. -+ # Do this after we call the preparer on these reqs to make sure their -+ # egg info has been created -+ from pipenv.utils import chdir -+ with chdir(ireq.setup_py_dir): -+ try: -+ from setuptools.dist import distutils -+ dist = distutils.core.run_setup(ireq.setup_py) -+ except InstallationError: -+ ireq.run_egg_info() -+ except (TypeError, ValueError, AttributeError): -+ pass -+ if not dist: -+ try: -+ dist = ireq.get_dist() -+ except (ImportError, ValueError, TypeError, AttributeError): -+ pass -+ if ireq.editable and dist: -+ setup_requires = getattr(dist, "extras_require", None) -+ if not setup_requires: -+ setup_requires = {"setup_requires": getattr(dist, "setup_requires", None)} -+ if not getattr(ireq, 'req', None): -+ try: -+ ireq.req = dist.as_requirement() if dist else None -+ except (ValueError, TypeError) as e: -+ pass - -- def get_dependencies(self, ireq): -+ # Convert setup_requires dict into a somewhat usable form. -+ if setup_requires: -+ for section in setup_requires: -+ python_version = section -+ not_python = not (section.startswith('[') and ':' in section) -+ -+ # This is for cleaning up :extras: formatted markers -+ # by adding them to the results of the resolver -+ # since any such extra would have been returned as a result anyway -+ for value in setup_requires[section]: -+ # This is a marker. -+ if value.startswith('[') and ':' in value: -+ python_version = value[1:-1] -+ not_python = False -+ # Strip out other extras. -+ if value.startswith('[') and ':' not in value: -+ not_python = True -+ -+ if ':' not in value: -+ try: -+ if not not_python: -+ results.add(InstallRequirement.from_line("{0}{1}".format(value, python_version).replace(':', ';'))) -+ # Anything could go wrong here -- can't be too careful. -+ except Exception: -+ pass -+ -+ # this section properly creates 'python_version' markers for cross-python -+ # virtualenv creation and for multi-python compatibility. -+ requires_python = reqset.requires_python if hasattr(reqset, 'requires_python') else resolver.requires_python -+ if requires_python: -+ marker_str = '' -+ # This corrects a logic error from the previous code which said that if -+ # we Encountered any 'requires_python' attributes, basically only create a -+ # single result no matter how many we resolved. This should fix -+ # a majority of the remaining non-deterministic resolution issues. -+ if any(requires_python.startswith(op) for op in Specifier._operators.keys()): -+ # We are checking first if we have leading specifier operator -+ # if not, we can assume we should be doing a == comparison -+ specifierset = SpecifierSet(requires_python) -+ # for multiple specifiers, the correct way to represent that in -+ # a specifierset is `Requirement('fakepkg; python_version<"3.0,>=2.6"')` -+ from passa.internals.specifiers import cleanup_pyspecs -+ marker_str = str(Marker(" and ".join(dedup([ -+ "python_version {0[0]} '{0[1]}'".format(spec) -+ for spec in cleanup_pyspecs(specifierset) -+ ])))) -+ # The best way to add markers to a requirement is to make a separate requirement -+ # with only markers on it, and then to transfer the object istelf -+ marker_to_add = Requirement('fakepkg; {0}'.format(marker_str)).marker -+ if ireq in results: -+ results.remove(ireq) -+ print(marker_to_add) -+ ireq.req.marker = marker_to_add -+ + results = set(results) if results else set() + return results, ireq -+ + +- def get_dependencies(self, ireq): + def get_legacy_dependencies(self, ireq): """ Given a pinned or an editable InstallRequirement, returns a set of dependencies (also InstallRequirements, but not necessarily pinned). -@@ -200,6 +408,7 @@ class PyPIRepository(BaseRepository): - # If a download_dir is passed, pip will unnecessarely - # archive the entire source directory - download_dir = None -+ - elif ireq.link and not ireq.link.is_artifact: - # No download_dir for VCS sources. This also works around pip - # using git-checkout-index, which gets rid of the .git dir. -@@ -214,7 +423,8 @@ class PyPIRepository(BaseRepository): +@@ -223,7 +346,8 @@ class PyPIRepository(BaseRepository): wheel_cache = WheelCache(CACHE_DIR, self.pip_options.format_control) prev_tracker = os.environ.get('PIP_REQ_TRACKER') try: @@ -492,7 +432,7 @@ index bf69803..31b85b9 100644 finally: if 'PIP_REQ_TRACKER' in os.environ: if prev_tracker: -@@ -236,6 +446,10 @@ class PyPIRepository(BaseRepository): +@@ -245,6 +369,10 @@ class PyPIRepository(BaseRepository): if ireq.editable: return set() @@ -503,7 +443,7 @@ index bf69803..31b85b9 100644 if not is_pinned_requirement(ireq): raise TypeError( "Expected pinned requirement, got {}".format(ireq)) -@@ -243,24 +457,22 @@ class PyPIRepository(BaseRepository): +@@ -252,24 +380,16 @@ class PyPIRepository(BaseRepository): # We need to get all of the candidates that match our current version # pin, these will represent all of the files that could possibly # satisfy this constraint. @@ -512,22 +452,16 @@ index bf69803..31b85b9 100644 - matching_versions = list( - ireq.specifier.filter((candidate.version for candidate in all_candidates))) - matching_candidates = candidates_by_version[matching_versions[0]] -+ ### Modification -- this is much more efficient.... -+ ### modification again -- still more efficient + matching_candidates = ( + c for c in clean_requires_python(self.find_all_candidates(ireq.name)) + if c.version in ireq.specifier + ) -+ # candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version) -+ # matching_versions = list( -+ # ireq.specifier.filter((candidate.version for candidate in all_candidates))) -+ # matching_candidates = candidates_by_version[matching_versions[0]] return { - self._get_file_hash(candidate.location) - for candidate in matching_candidates + h for h in map(lambda c: self._hash_cache.get_hash(c.location), -+ matching_candidates) if h is not None ++ matching_candidates) if h is not None } - def _get_file_hash(self, location): @@ -541,26 +475,17 @@ index bf69803..31b85b9 100644 def allow_all_wheels(self): """ diff --git a/pipenv/patched/piptools/resolver.py b/pipenv/patched/piptools/resolver.py -index c2d323c..d5a471d 100644 +index 494d385..b642bc9 100644 --- a/pipenv/patched/piptools/resolver.py +++ b/pipenv/patched/piptools/resolver.py -@@ -13,7 +13,7 @@ from . import click - from .cache import DependencyCache - from .exceptions import UnsupportedConstraint - from .logging import log --from .utils import (format_requirement, format_specifier, full_groupby, -+from .utils import (format_requirement, format_specifier, full_groupby, dedup, simplify_markers, - is_pinned_requirement, key_from_ireq, key_from_req, UNSAFE_PACKAGES) - - green = partial(click.style, fg='green') -@@ -27,6 +27,7 @@ class RequirementSummary(object): - def __init__(self, ireq): +@@ -28,6 +28,7 @@ class RequirementSummary(object): self.req = ireq.req self.key = key_from_req(ireq.req) -+ self.markers = ireq.markers self.extras = str(sorted(ireq.extras)) ++ self.markers = ireq.markers self.specifier = str(ireq.specifier) + def __eq__(self, other): @@ -119,7 +120,7 @@ class Resolver(object): @staticmethod def check_constraints(constraints): @@ -570,7 +495,7 @@ index c2d323c..d5a471d 100644 msg = ('pip-compile does not support URLs as packages, unless they are editable. ' 'Perhaps add -e option?') raise UnsupportedConstraint(msg, constraint) -@@ -155,6 +156,13 @@ class Resolver(object): +@@ -155,6 +156,12 @@ class Resolver(object): # NOTE we may be losing some info on dropped reqs here combined_ireq.req.specifier &= ireq.req.specifier combined_ireq.constraint &= ireq.constraint @@ -580,11 +505,10 @@ index c2d323c..d5a471d 100644 + _markers = combined_ireq.markers._markers + if not isinstance(_markers[0], (tuple, list)): + combined_ireq.markers._markers = [_markers, 'and', ireq.markers._markers] -+ # Return a sorted, de-duped tuple of extras combined_ireq.extras = tuple(sorted(set(tuple(combined_ireq.extras) + tuple(ireq.extras)))) yield combined_ireq -@@ -272,6 +280,15 @@ class Resolver(object): +@@ -272,6 +279,15 @@ class Resolver(object): for dependency in self.repository.get_dependencies(ireq): yield dependency return @@ -600,7 +524,7 @@ index c2d323c..d5a471d 100644 elif not is_pinned_requirement(ireq): raise TypeError('Expected pinned or editable requirement, got {}'.format(ireq)) -@@ -282,7 +299,7 @@ class Resolver(object): +@@ -282,7 +298,7 @@ class Resolver(object): if ireq not in self.dependency_cache: log.debug(' {} not in cache, need to check index'.format(format_requirement(ireq)), fg='yellow') dependencies = self.repository.get_dependencies(ireq) @@ -610,29 +534,32 @@ index c2d323c..d5a471d 100644 # Example: ['Werkzeug>=0.9', 'Jinja2>=2.4'] dependency_strings = self.dependency_cache[ireq] diff --git a/pipenv/patched/piptools/utils.py b/pipenv/patched/piptools/utils.py -index 2360a04..6f62eb9 100644 +index 9b4b4c2..8875543 100644 --- a/pipenv/patched/piptools/utils.py +++ b/pipenv/patched/piptools/utils.py -@@ -4,6 +4,7 @@ from __future__ import (absolute_import, division, print_function, +@@ -2,10 +2,17 @@ + from __future__ import (absolute_import, division, print_function, + unicode_literals) - import os ++import os import sys -+import six from itertools import chain, groupby from collections import OrderedDict - from contextlib import contextmanager -@@ -11,11 +12,78 @@ from contextlib import contextmanager + ++import six ++ ++from pipenv.vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier ++from pipenv.vendor.packaging.version import Version, InvalidVersion, parse as parse_version ++from pipenv.vendor.packaging.markers import Marker, Op, Value, Variable ++ from ._compat import install_req_from_line from .click import style -+from pipenv.patched.notpip._vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier -+from pipenv.patched.notpip._vendor.packaging.version import Version, InvalidVersion, parse as parse_version -+from pipenv.patched.notpip._vendor.packaging.markers import Marker, Op, Value, Variable - - +@@ -14,6 +21,71 @@ from .click import style UNSAFE_PACKAGES = {'setuptools', 'distribute', 'pip'} ++ +def simplify_markers(ireq): + """simplify_markers "This code cleans up markers for a specific :class:`~InstallRequirement`" + @@ -681,7 +608,7 @@ index 2360a04..6f62eb9 100644 + all_candidates = [] + py_version = parse_version(os.environ.get('PIP_PYTHON_VERSION', '.'.join(map(str, sys.version_info[:3])))) + for c in candidates: -+ if c.requires_python: ++ if getattr(c, "requires_python", None): + # Old specifications had people setting this to single digits + # which is effectively the same as '>=digit, None +diff --git a/pipenv/vendor/dotenv/cli.py b/pipenv/vendor/dotenv/cli.py +index 45f4b76..829b14a 100644 +--- a/pipenv/vendor/dotenv/cli.py ++++ b/pipenv/vendor/dotenv/cli.py +@@ -1,6 +1,5 @@ + import os + import sys +-from typing import Any, List + + try: + import click +@@ -9,9 +8,13 @@ except ImportError: + 'Run pip install "python-dotenv[cli]" to fix this.') + sys.exit(1) + ++from .compat import IS_TYPE_CHECKING + from .main import dotenv_values, get_key, set_key, unset_key, run_command + from .version import __version__ + ++if IS_TYPE_CHECKING: ++ from typing import Any, List ++ + + @click.group() + @click.option('-f', '--file', default=os.path.join(os.getcwd(), '.env'), +diff --git a/pipenv/vendor/dotenv/compat.py b/pipenv/vendor/dotenv/compat.py +index 99ffb39..7a8694f 100644 +--- a/pipenv/vendor/dotenv/compat.py ++++ b/pipenv/vendor/dotenv/compat.py +@@ -1,3 +1,4 @@ ++import os + import sys + + if sys.version_info >= (3, 0): +@@ -6,3 +7,15 @@ else: + from StringIO import StringIO # noqa + + PY2 = sys.version_info[0] == 2 # type: bool ++ ++ ++def is_type_checking(): ++ # type: () -> bool ++ try: ++ from typing import TYPE_CHECKING ++ except ImportError: ++ return False ++ return TYPE_CHECKING ++ ++ ++IS_TYPE_CHECKING = os.environ.get("MYPY_RUNNING", is_type_checking()) +diff --git a/pipenv/vendor/dotenv/main.py b/pipenv/vendor/dotenv/main.py +index 0812282..64d4269 100644 +--- a/pipenv/vendor/dotenv/main.py ++++ b/pipenv/vendor/dotenv/main.py +@@ -9,15 +9,17 @@ import shutil + import sys + from subprocess import Popen + import tempfile +-from typing import (Any, Dict, Iterator, List, Match, NamedTuple, Optional, # noqa +- Pattern, Union, TYPE_CHECKING, Text, IO, Tuple) # noqa + import warnings + from collections import OrderedDict + from contextlib import contextmanager + +-from .compat import StringIO, PY2 ++from .compat import StringIO, PY2, IS_TYPE_CHECKING + +-if TYPE_CHECKING: # pragma: no cover ++if IS_TYPE_CHECKING: # pragma: no cover ++ from typing import ( ++ Dict, Iterator, List, Match, Optional, Pattern, Union, ++ Text, IO, Tuple ++ ) + if sys.version_info >= (3, 6): + _PathLike = os.PathLike + else: +@@ -59,10 +61,14 @@ _binding = re.compile( + + _escape_sequence = re.compile(r"\\[\\'\"abfnrtv]") # type: Pattern[Text] + +- +-Binding = NamedTuple("Binding", [("key", Optional[Text]), +- ("value", Optional[Text]), +- ("original", Text)]) ++try: ++ from typing import NamedTuple, Optional, Text ++ Binding = NamedTuple("Binding", [("key", Optional[Text]), ++ ("value", Optional[Text]), ++ ("original", Text)]) ++except ImportError: ++ from collections import namedtuple ++ Binding = namedtuple("Binding", ["key", "value", "original"]) + + + def decode_escapes(string): diff --git a/tasks/vendoring/patches/vendor/dotenv-windows-unicode.patch b/tasks/vendoring/patches/vendor/dotenv-windows-unicode.patch deleted file mode 100644 index c090e885..00000000 --- a/tasks/vendoring/patches/vendor/dotenv-windows-unicode.patch +++ /dev/null @@ -1,18 +0,0 @@ -diff --git a/pipenv/vendor/dotenv/main.py b/pipenv/vendor/dotenv/main.py -index 3d1bd72f..75f49c4a 100644 ---- a/pipenv/vendor/dotenv/main.py -+++ b/pipenv/vendor/dotenv/main.py -@@ -94,6 +94,13 @@ class DotEnv(): - for k, v in self.dict().items(): - if k in os.environ and not override: - continue -+ # With Python 2 on Windows, ensuree environment variables are -+ # system strings to avoid "TypeError: environment can only contain -+ # strings" in Python's subprocess module. -+ if sys.version_info.major < 3 and sys.platform == 'win32': -+ from pipenv.utils import fs_str -+ k = fs_str(k) -+ v = fs_str(v) - os.environ[k] = v - - return True diff --git a/tasks/vendoring/patches/vendor/pipdeptree-updated-pip18.patch b/tasks/vendoring/patches/vendor/pipdeptree-updated-pip18.patch index d479ebfa..5d5a381f 100644 --- a/tasks/vendoring/patches/vendor/pipdeptree-updated-pip18.patch +++ b/tasks/vendoring/patches/vendor/pipdeptree-updated-pip18.patch @@ -7,7 +7,7 @@ index 7820aa5..2082fc8 100644 from ordereddict import OrderedDict -try: -- from pipenv.patched.notpip._internal import get_installed_distributions +- from pipenv.patched.notpip._internal.utils.misc import get_installed_distributions - from pipenv.patched.notpip._internal.operations.freeze import FrozenRequirement -except ImportError: - from pipenv.patched.notpip import get_installed_distributions, FrozenRequirement @@ -17,3 +17,4 @@ index 7820aa5..2082fc8 100644 import pkg_resources # inline: + diff --git a/tasks/vendoring/patches/vendor/tomlkit-fix.patch b/tasks/vendoring/patches/vendor/tomlkit-fix.patch index 36e2f808..4aa6c16f 100644 --- a/tasks/vendoring/patches/vendor/tomlkit-fix.patch +++ b/tasks/vendoring/patches/vendor/tomlkit-fix.patch @@ -14,8 +14,10 @@ diff --git a/pipenv/vendor/tomlkit/container.py b/pipenv/vendor/tomlkit/containe index cb8af1d5..9b5db5cb 100644 --- a/pipenv/vendor/tomlkit/container.py +++ b/pipenv/vendor/tomlkit/container.py -@@ -1,13 +1,5 @@ +@@ -1,15 +1,7 @@ from __future__ import unicode_literals + + import copy -from typing import Any -from typing import Dict diff --git a/tasks/vendoring/patches/vendor/vistir-imports.patch b/tasks/vendoring/patches/vendor/vistir-imports.patch index 673efad8..725e8a56 100644 --- a/tasks/vendoring/patches/vendor/vistir-imports.patch +++ b/tasks/vendoring/patches/vendor/vistir-imports.patch @@ -1,8 +1,8 @@ diff --git a/pipenv/vendor/vistir/backports/tempfile.py b/pipenv/vendor/vistir/backports/tempfile.py -index 483a479a..43470a6e 100644 +index f5594a2d..a3d7f3df 100644 --- a/pipenv/vendor/vistir/backports/tempfile.py +++ b/pipenv/vendor/vistir/backports/tempfile.py -@@ -13,7 +13,7 @@ import six +@@ -12,7 +12,7 @@ import six try: from weakref import finalize except ImportError: @@ -10,31 +10,35 @@ index 483a479a..43470a6e 100644 + from pipenv.vendor.backports.weakref import finalize - __all__ = ["finalize", "NamedTemporaryFile"] + def fs_encode(path): diff --git a/pipenv/vendor/vistir/compat.py b/pipenv/vendor/vistir/compat.py -index 9ae33fdc..ec3b65cb 100644 +index b5904bc7..a44aafbe 100644 --- a/pipenv/vendor/vistir/compat.py +++ b/pipenv/vendor/vistir/compat.py -@@ -31,11 +31,11 @@ if sys.version_info >= (3, 5): - from functools import lru_cache - else: - from pathlib2 import Path +@@ -43,7 +43,7 @@ __all__ = [ + if sys.version_info >= (3, 5): # pragma: no cover + from pathlib import Path + else: # pragma: no cover +- from pathlib2 import Path ++ from pipenv.vendor.pathlib2 import Path + + if six.PY3: # pragma: no cover + # Only Python 3.4+ is supported +@@ -53,14 +53,14 @@ if six.PY3: # pragma: no cover + from weakref import finalize + else: # pragma: no cover + # Only Python 2.7 is supported - from backports.functools_lru_cache import lru_cache + from pipenv.vendor.backports.functools_lru_cache import lru_cache - - from .backports.tempfile import NamedTemporaryFile as _NamedTemporaryFile - if sys.version_info < (3, 3): + from .backports.functools import partialmethod # type: ignore - from backports.shutil_get_terminal_size import get_terminal_size + from pipenv.vendor.backports.shutil_get_terminal_size import get_terminal_size + from .backports.surrogateescape import register_surrogateescape + + register_surrogateescape() NamedTemporaryFile = _NamedTemporaryFile - else: - from tempfile import NamedTemporaryFile -@@ -44,7 +44,7 @@ else: - try: - from weakref import finalize - except ImportError: -- from backports.weakref import finalize -+ from pipenv.vendor.backports.weakref import finalize +- from backports.weakref import finalize # type: ignore ++ from pipenv.vendor.backports.weakref import finalize # type: ignore try: - from functools import partialmethod + # Introduced Python 3.5 diff --git a/tasks/vendoring/patches/vendor/yaspin-signal-handling.patch b/tasks/vendoring/patches/vendor/yaspin-signal-handling.patch index a1d27cd3..511e782a 100644 --- a/tasks/vendoring/patches/vendor/yaspin-signal-handling.patch +++ b/tasks/vendoring/patches/vendor/yaspin-signal-handling.patch @@ -7,7 +7,7 @@ index d01fb98e..06b8b621 100644 import time +import colorama -+import cursor ++from pipenv.vendor.vistir import cursor + from .base_spinner import default_spinner from .compat import PY2, basestring, builtin_str, bytes, iteritems, str @@ -48,13 +48,13 @@ index d01fb98e..06b8b621 100644 def _hide_cursor(): - sys.stdout.write("\033[?25l") - sys.stdout.flush() -+ cursor.hide() ++ cursor.hide_cursor() @staticmethod def _show_cursor(): - sys.stdout.write("\033[?25h") - sys.stdout.flush() -+ cursor.show() ++ cursor.show_cursor() @staticmethod def _clear_line(): diff --git a/tasks/vendoring/vendor_passa.py b/tasks/vendoring/vendor_passa.py index ad179a10..f2c58745 100644 --- a/tasks/vendoring/vendor_passa.py +++ b/tasks/vendoring/vendor_passa.py @@ -1,6 +1,6 @@ -from pipenv._compat import TemporaryDirectory import invoke +from pipenv._compat import TemporaryDirectory from . import _get_git_root, _get_vendor_dir, log diff --git a/tests/fixtures/fake-package/.coveragerc b/tests/fixtures/fake-package/.coveragerc new file mode 100644 index 00000000..1b3a0571 --- /dev/null +++ b/tests/fixtures/fake-package/.coveragerc @@ -0,0 +1,27 @@ +[run] +branch = True +parallel = True +source = src/fake_package/ + +[report] +# Regexes for lines to exclude from consideration +exclude_lines = + # Have to re-enable the standard pragma + pragma: no cover + + # Don't complain about missing debug-only code: + def __repr__ + if self\.debug + + # Don't complain if tests don't hit defensive assertion code: + raise AssertionError + raise NotImplementedError + # Don't complain if non-runnable code isn't run: + if 0: + if __name__ == .__main__.: + +[html] +directory = htmlcov + +[xml] +output = coverage.xml diff --git a/tests/fixtures/fake-package/.editorconfig b/tests/fixtures/fake-package/.editorconfig new file mode 100644 index 00000000..7470e9db --- /dev/null +++ b/tests/fixtures/fake-package/.editorconfig @@ -0,0 +1,27 @@ +root = true + +[*] +indent_style = space +indent_size = 4 +end_of_line = lf +charset = utf-8 +trim_trailing_whitespace = true +insert_final_newline = true + +[*.md] +trim_trailing_whitespace = false + +[*.toml] +indent_size = 2 + +[*.{yaml,yml}] +indent_size = 2 + +# Makefiles always use tabs for indentation. +[Makefile] +indent_style = tab + +# Batch files use tabs for indentation, and old Notepad hates LF. +[*.bat] +indent_style = tab +end_of_line = crlf diff --git a/tests/fixtures/fake-package/.gitignore b/tests/fixtures/fake-package/.gitignore new file mode 100644 index 00000000..ab621d86 --- /dev/null +++ b/tests/fixtures/fake-package/.gitignore @@ -0,0 +1,108 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.typeshed + +.vscode/ + +pip-wheel-metadata diff --git a/tests/fixtures/fake-package/.pre-commit-config.yaml b/tests/fixtures/fake-package/.pre-commit-config.yaml new file mode 100644 index 00000000..7ecca7dc --- /dev/null +++ b/tests/fixtures/fake-package/.pre-commit-config.yaml @@ -0,0 +1,20 @@ +repos: + - repo: https://github.com/ambv/black + rev: stable + hooks: + - id: black + + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.0.0 + hooks: + - id: flake8 + + - repo: https://github.com/asottile/seed-isort-config + rev: v1.7.0 + hooks: + - id: seed-isort-config + + - repo: https://github.com/pre-commit/mirrors-isort + rev: v4.3.9 + hooks: + - id: isort diff --git a/tests/fixtures/fake-package/.travis.yml b/tests/fixtures/fake-package/.travis.yml new file mode 100644 index 00000000..84bf6c03 --- /dev/null +++ b/tests/fixtures/fake-package/.travis.yml @@ -0,0 +1,38 @@ +language: python +sudo: false +cache: pip +dist: trusty + +matrix: + fast_finish: true + +install: + - "python -m pip install --upgrade pip pytest-timeout" + - "python -m pip install -e .[tests]" +script: + - "python -m pytest -v -n 8 tests/" + +jobs: + include: + - stage: test + - python: "3.7" + dist: xenial + sudo: required + - python: "3.6" + - python: "2.7" + - python: "3.5" + - python: "3.4" + - stage: packaging + python: "3.6" + install: + - "pip install --upgrade twine readme-renderer[md]" + script: + - "python setup.py sdist" + - "twine check dist/*" + - stage: coverage + python: "3.6" + install: + - "python -m pip install --upgrade pip pytest-timeout pytest-cov" + - "python -m pip install --upgrade -e .[tests]" + script: + - "python -m pytest -n auto --timeout 300 --cov=fake_package --cov-report=term-missing --cov-report=xml --cov-report=html tests" diff --git a/tests/fixtures/fake-package/LICENSE b/tests/fixtures/fake-package/LICENSE new file mode 100644 index 00000000..0beb71e0 --- /dev/null +++ b/tests/fixtures/fake-package/LICENSE @@ -0,0 +1,13 @@ +Copyright (c) 2019, Dan Ryan + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/tests/fixtures/fake-package/MANIFEST.in b/tests/fixtures/fake-package/MANIFEST.in new file mode 100644 index 00000000..7ab57a21 --- /dev/null +++ b/tests/fixtures/fake-package/MANIFEST.in @@ -0,0 +1,19 @@ +include LICENSE* README* +include CHANGELOG.rst +include pyproject.toml + +exclude .editorconfig +exclude .coveragerc +exclude .travis.yml +exclude tox.ini +exclude appveyor.yml +exclude Pipfile* + +recursive-include docs Makefile *.rst *.py *.bat +recursive-exclude docs requirements*.txt + +prune .github +prune docs/build +prune news +prune tasks +prune tests diff --git a/tests/fixtures/fake-package/Pipfile b/tests/fixtures/fake-package/Pipfile new file mode 100644 index 00000000..284b7798 --- /dev/null +++ b/tests/fixtures/fake-package/Pipfile @@ -0,0 +1,14 @@ +[packages] +fake_package = { path = '.', editable = true, extras = ["dev", "tests"] } + +[dev-packages] +towncrier = '*' +sphinx = '*' +sphinx-rtd-theme = '*' + +[scripts] +release = 'inv release' +tests = "pytest -v tests" +draft = "towncrier --draft" +changelog = "towncrier" +build = "setup.py sdist bdist_wheel" diff --git a/tests/fixtures/fake-package/README.rst b/tests/fixtures/fake-package/README.rst new file mode 100644 index 00000000..4256cd1f --- /dev/null +++ b/tests/fixtures/fake-package/README.rst @@ -0,0 +1,3 @@ +=============================================================================== +fake_package: A fake python package. +=============================================================================== diff --git a/tests/fixtures/fake-package/appveyor.yml b/tests/fixtures/fake-package/appveyor.yml new file mode 100644 index 00000000..758f4cfc --- /dev/null +++ b/tests/fixtures/fake-package/appveyor.yml @@ -0,0 +1,61 @@ +build: off +version: 1.0.{build} +skip_branch_with_pr: true + +init: +- ps: >- + + git config --global core.sharedRepository true + + git config --global core.longpaths true + + git config --global core.autocrlf input + + if ($env:APPVEYOR_PULL_REQUEST_NUMBER -and $env:APPVEYOR_BUILD_NUMBER -ne ((Invoke-RestMethod ` + https://ci.appveyor.com/api/projects/$env:APPVEYOR_ACCOUNT_NAME/$env:APPVEYOR_PROJECT_SLUG/history?recordsNumber=50).builds | ` + Where-Object pullRequestId -eq $env:APPVEYOR_PULL_REQUEST_NUMBER)[0].buildNumber) { ` + Write-Host "There are newer queued builds for this pull request, skipping build." + Exit-AppveyorBuild + } + + If (($env:SKIP_NOTAG -eq "true") -and ($env:APPVEYOR_REPO_TAG -ne "true")) { + Write-Host "Skipping build, not at a tag." + Exit-AppveyorBuild + } + +environment: + GIT_ASK_YESNO: 'false' + APPVEYOR_SAVE_CACHE_ON_ERROR: 'true' + APPVEYOR_SKIP_FINALIZE_ON_EXIT: 'true' + SHELL: 'windows' + PYTHON_ARCH: '64' + PYTHONIOENCODING: 'utf-8' + + matrix: + # Unit and integration tests. + - PYTHON: "C:\\Python27" + RUN_INTEGRATION_TESTS: "True" + - PYTHON: "C:\\Python37-x64" + RUN_INTEGRATION_TESTS: "True" + # Unit tests only. + - PYTHON: "C:\\Python36-x64" + - PYTHON: "C:\\Python34-x64" + - PYTHON: "C:\\Python35-x64" + +cache: +- '%LocalAppData%\pip\cache' +- '%LocalAppData%\pipenv\cache' + +install: + - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" + - "python --version" + - "python -m pip install --upgrade pip pytest-timeout" + - "python -m pip install -e .[tests]" + + +test_script: + # Shorten paths, workaround https://bugs.python.org/issue18199 + - "subst T: %TEMP%" + - "set TEMP=T:\\" + - "set TMP=T:\\" + - "python -m pytest -n auto -v tests" diff --git a/tests/fixtures/fake-package/docs/conf.py b/tests/fixtures/fake-package/docs/conf.py new file mode 100644 index 00000000..3aded982 --- /dev/null +++ b/tests/fixtures/fake-package/docs/conf.py @@ -0,0 +1,208 @@ +# -*- coding: utf-8 -*- +# +# Configuration file for the Sphinx documentation builder. +# +# This file does only contain a selection of the most common options. For a +# full list see the documentation: +# http://www.sphinx-doc.org/en/master/config + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +PACKAGE_DIR = os.path.join(ROOT, "src/fake_package") +sys.path.insert(0, PACKAGE_DIR) + + +# -- Project information ----------------------------------------------------- + +project = 'fake_package' +copyright = '2019, Dan Ryan ' +author = 'Dan Ryan ' + +# The short X.Y version +version = '0.0' +# The full version, including alpha/beta/rc tags +release = '0.0.0.dev0' + + +# -- General configuration --------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.viewcode', + 'sphinx.ext.todo', + 'sphinx.ext.intersphinx', + 'sphinx.ext.autosummary' +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = 'en' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path . +exclude_patterns = ['_build', '_man', 'Thumbs.db', '.DS_Store'] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' +autosummary_generate = True + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'sphinx_rtd_theme' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# The default sidebars (for documents that don't match any pattern) are +# defined by theme itself. Builtin themes are using these templates by +# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', +# 'searchbox.html']``. +# +# html_sidebars = {} + + +# -- Options for HTMLHelp output --------------------------------------------- + +# Output file base name for HTML help builder. +htmlhelp_basename = 'fake_packagedoc' +extlinks = { + 'issue': ('https://github.com/sarugaku/fake_package/issues/%s', '#'), + 'pull': ('https://github.com/sarugaku/fake_package/pull/%s', 'PR #'), +} +html_theme_options = { + 'display_version': True, + 'prev_next_buttons_location': 'bottom', + 'style_external_links': True, + 'vcs_pageview_mode': '', + # Toc options + 'collapse_navigation': True, + 'sticky_navigation': True, + 'navigation_depth': 4, + 'includehidden': True, + 'titles_only': False +} + +# -- Options for LaTeX output ------------------------------------------------ + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'fake_package.tex', 'fake_package Documentation', + 'Dan Ryan \\textless{}dan@danryan.co\\textgreater{}', 'manual'), +] + + +# -- Options for manual page output ------------------------------------------ + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'fake_package', 'fake_package Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ---------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'fake_package', 'fake_package Documentation', + author, 'fake_package', 'A fake python package.', + 'Miscellaneous'), +] + + +# -- Options for Epub output ------------------------------------------------- + +# Bibliographic Dublin Core info. +epub_title = project +epub_author = author +epub_publisher = author +epub_copyright = copyright + +# The unique identifier of the text. This can be a ISBN number +# or the project homepage. +# +# epub_identifier = '' + +# A unique identification for the text. +# +# epub_uid = '' + +# A list of files that should not be packed into the epub file. +epub_exclude_files = ['search.html'] + + +# -- Extension configuration ------------------------------------------------- + +# -- Options for todo extension ---------------------------------------------- + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True +intersphinx_mapping = {'python': ('https://docs.python.org/3', None)} diff --git a/tests/fixtures/fake-package/docs/requirements.txt b/tests/fixtures/fake-package/docs/requirements.txt new file mode 100644 index 00000000..82133027 --- /dev/null +++ b/tests/fixtures/fake-package/docs/requirements.txt @@ -0,0 +1,2 @@ +sphinx +sphinx_rtd_theme diff --git a/tests/fixtures/fake-package/news/.gitignore b/tests/fixtures/fake-package/news/.gitignore new file mode 100644 index 00000000..f935021a --- /dev/null +++ b/tests/fixtures/fake-package/news/.gitignore @@ -0,0 +1 @@ +!.gitignore diff --git a/tests/fixtures/fake-package/pyproject.toml b/tests/fixtures/fake-package/pyproject.toml new file mode 100644 index 00000000..e157956b --- /dev/null +++ b/tests/fixtures/fake-package/pyproject.toml @@ -0,0 +1,50 @@ +[build-system] +requires = ['setuptools>=40.8.0', 'wheel>=0.33.0'] + +[tool.black] +line-length = 90 +include = '\.pyi?$' +exclude = ''' +/( + \.eggs + | \.git + | \.hg + | \.mypy_cache + | \.tox + | \.pyre_configuration + | \.venv + | _build + | buck-out + | build + | dist +) +''' + +[tool.towncrier] +package = 'fake-package' +package_dir = 'src' +filename = 'CHANGELOG.rst' +directory = 'news/' +title_format = '{version} ({project_date})' +issue_format = '`#{issue} `_' +template = 'tasks/CHANGELOG.rst.jinja2' + + [[tool.towncrier.type]] + directory = 'feature' + name = 'Features' + showcontent = true + + [[tool.towncrier.type]] + directory = 'bugfix' + name = 'Bug Fixes' + showcontent = true + + [[tool.towncrier.type]] + directory = 'trivial' + name = 'Trivial Changes' + showcontent = false + + [[tool.towncrier.type]] + directory = 'removal' + name = 'Removals and Deprecations' + showcontent = true diff --git a/tests/fixtures/fake-package/setup.cfg b/tests/fixtures/fake-package/setup.cfg new file mode 100644 index 00000000..c357cea9 --- /dev/null +++ b/tests/fixtures/fake-package/setup.cfg @@ -0,0 +1,120 @@ +[metadata] +name = fake_package +description = A fake python package. +url = https://github.com/sarugaku/fake_package +author = Dan Ryan +author_email = dan@danryan.co +long_description = file: README.rst +license = ISC License +keywords = fake package test +classifier = + Development Status :: 1 - Planning + License :: OSI Approved :: ISC License (ISCL) + Operating System :: OS Independent + Programming Language :: Python :: 2 + Programming Language :: Python :: 2.6 + Programming Language :: Python :: 2.7 + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.4 + Programming Language :: Python :: 3.5 + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 + Topic :: Software Development :: Libraries :: Python Modules + +[options.extras_require] +tests = + pytest + pytest-xdist + pytest-cov + pytest-timeout + readme-renderer[md] + twine +dev = + black;python_version>="3.6" + flake8 + flake8-bugbear;python_version>="3.5" + invoke + isort + mypy;python_version>="3.5" + parver + pre-commit + rope + wheel + +[options] +zip_safe = true +python_requires = >=2.6,!=3.0,!=3.1,!=3.2,!=3.3 +setup_requires = setuptools>=40.8.0 +install_requires = + invoke + attrs + +[bdist_wheel] +universal = 1 + +[tool:pytest] +strict = true +plugins = cov flake8 +addopts = -ra +testpaths = tests/ +norecursedirs = .* build dist news tasks docs +flake8-ignore = + docs/source/* ALL + tests/*.py ALL + setup.py ALL +filterwarnings = + ignore::DeprecationWarning + ignore::PendingDeprecationWarning + +[isort] +atomic = true +not_skip = __init__.py +line_length = 90 +indent = ' ' +multi_line_output = 3 +known_third_party = invoke,parver,pytest,setuptools,towncrier +known_first_party = + fake_package + tests +combine_as_imports=True +include_trailing_comma = True +force_grid_wrap=0 + +[flake8] +max-line-length = 90 +select = C,E,F,W,B,B950 +ignore = + # The default ignore list: + D203,F401,E123,E203,W503,E501,E402 + #E121,E123,E126,E226,E24,E704, + # Our additions: + # E123: closing bracket does not match indentation of opening bracket’s line + # E203: whitespace before ‘:’ + # E129: visually indented line with same indent as next logical line + # E222: multiple spaces after operator + # E231: missing whitespace after ',' + # D203: 1 blank line required before class docstring + # E402: module level import not at top of file + # E501: line too long (using B950 from flake8-bugbear) + # F401: Module imported but unused + # W503: line break before binary operator (not a pep8 issue, should be ignored) +exclude = + .tox, + .git, + __pycache__, + docs/source/*, + build, + dist, + tests/*, + *.pyc, + *.egg-info, + .cache, + .eggs, + setup.py, +max-complexity=13 + +[mypy] +ignore_missing_imports=true +follow_imports=skip +html_report=mypyhtml +python_version=2.7 diff --git a/tests/fixtures/fake-package/setup.py b/tests/fixtures/fake-package/setup.py new file mode 100644 index 00000000..1d2c88f8 --- /dev/null +++ b/tests/fixtures/fake-package/setup.py @@ -0,0 +1,35 @@ +import ast +import os + +from setuptools import find_packages, setup + + +ROOT = os.path.dirname(__file__) + +PACKAGE_NAME = 'fake_package' + +VERSION = None + +with open(os.path.join(ROOT, 'src', PACKAGE_NAME.replace("-", "_"), '__init__.py')) as f: + for line in f: + if line.startswith('__version__ = '): + VERSION = ast.literal_eval(line[len('__version__ = '):].strip()) + break +if VERSION is None: + raise EnvironmentError('failed to read version') + + +# Put everything in setup.cfg, except those that don't actually work? +setup( + # These really don't work. + package_dir={'': 'src'}, + packages=find_packages('src'), + + # I don't know how to specify an empty key in setup.cfg. + package_data={ + '': ['LICENSE*', 'README*'], + }, + + # I need this to be dynamic. + version=VERSION, +) diff --git a/tests/fixtures/fake-package/src/fake_package/__init__.py b/tests/fixtures/fake-package/src/fake_package/__init__.py new file mode 100644 index 00000000..b8023d8b --- /dev/null +++ b/tests/fixtures/fake-package/src/fake_package/__init__.py @@ -0,0 +1 @@ +__version__ = '0.0.1' diff --git a/tests/fixtures/fake-package/tasks/CHANGELOG.rst.jinja2 b/tests/fixtures/fake-package/tasks/CHANGELOG.rst.jinja2 new file mode 100644 index 00000000..8aff2057 --- /dev/null +++ b/tests/fixtures/fake-package/tasks/CHANGELOG.rst.jinja2 @@ -0,0 +1,40 @@ +{% for section in sections %} +{% set underline = "-" %} +{% if section %} +{{section}} +{{ underline * section|length }}{% set underline = "~" %} + +{% endif %} +{% if sections[section] %} +{% for category, val in definitions.items() if category in sections[section] and category != 'trivial' %} + +{{ definitions[category]['name'] }} +{{ underline * definitions[category]['name']|length }} + +{% if definitions[category]['showcontent'] %} +{% for text, values in sections[section][category]|dictsort(by='value') %} +- {{ text }}{% if category != 'process' %} + {{ values|sort|join(',\n ') }} + {% endif %} + +{% endfor %} +{% else %} +- {{ sections[section][category]['']|sort|join(', ') }} + + +{% endif %} +{% if sections[section][category]|length == 0 %} + +No significant changes. + + +{% else %} +{% endif %} +{% endfor %} +{% else %} + +No significant changes. + + +{% endif %} +{% endfor %} diff --git a/tests/fixtures/fake-package/tasks/__init__.py b/tests/fixtures/fake-package/tasks/__init__.py new file mode 100644 index 00000000..a8cedab4 --- /dev/null +++ b/tests/fixtures/fake-package/tasks/__init__.py @@ -0,0 +1,175 @@ +import pathlib +import shutil +import subprocess + +import invoke +import parver + +from towncrier._builder import ( + find_fragments, render_fragments, split_fragments, +) +from towncrier._settings import load_config + + +ROOT = pathlib.Path(__file__).resolve().parent.parent + +PACKAGE_NAME = 'fake_package' + +INIT_PY = ROOT.joinpath('src', PACKAGE_NAME, '__init__.py') + + +@invoke.task() +def typecheck(ctx): + src_dir = ROOT / "src" / PACKAGE_NAME + src_dir = src_dir.as_posix() + config_file = ROOT / "setup.cfg" + env = {"MYPYPATH": src_dir} + ctx.run(f"mypy {src_dir} --config-file={config_file}", env=env) + + +@invoke.task() +def clean(ctx): + """Clean previously built package artifacts. + """ + ctx.run(f'python setup.py clean') + dist = ROOT.joinpath('dist') + print(f'[clean] Removing {dist}') + if dist.exists(): + shutil.rmtree(str(dist)) + + +def _read_version(): + out = subprocess.check_output(['git', 'tag'], encoding='ascii') + try: + version = max(parver.Version.parse(v).normalize() for v in ( + line.strip() for line in out.split('\n') + ) if v) + except ValueError: + version = parver.Version.parse('0.0.0') + return version + + +def _write_version(v): + lines = [] + with INIT_PY.open() as f: + for line in f: + if line.startswith("__version__ = "): + line = f"__version__ = {repr(str(v))}\n".replace("'", '"') + lines.append(line) + with INIT_PY.open("w", newline="\n") as f: + f.write("".join(lines)) + + +def _render_log(): + """Totally tap into Towncrier internals to get an in-memory result. + """ + config = load_config(ROOT) + definitions = config["types"] + fragments, fragment_filenames = find_fragments( + pathlib.Path(config["directory"]).absolute(), + config["sections"], + None, + definitions, + ) + rendered = render_fragments( + pathlib.Path(config["template"]).read_text(encoding="utf-8"), + config["issue_format"], + split_fragments(fragments, definitions), + definitions, + config["underlines"][1:], + False, # Don't add newlines to wrapped text. + ) + return rendered + + +REL_TYPES = ("major", "minor", "patch", "post") + + +def _bump_release(version, type_): + if type_ not in REL_TYPES: + raise ValueError(f"{type_} not in {REL_TYPES}") + index = REL_TYPES.index(type_) + next_version = version.base_version().bump_release(index=index) + print(f"[bump] {version} -> {next_version}") + return next_version + + +def _prebump(version, prebump): + next_version = version.bump_release(index=prebump).bump_dev() + print(f"[bump] {version} -> {next_version}") + return next_version + + +PREBUMP = 'patch' + + +@invoke.task(pre=[clean]) +def release(ctx, type_, repo, prebump=PREBUMP): + """Make a new release. + """ + if prebump not in REL_TYPES: + raise ValueError(f'{type_} not in {REL_TYPES}') + prebump = REL_TYPES.index(prebump) + + version = _read_version() + version = _bump_release(version, type_) + _write_version(version) + + # Needs to happen before Towncrier deletes fragment files. + tag_content = _render_log() + + ctx.run('towncrier') + + ctx.run(f'git commit -am "Release {version}"') + + tag_content = tag_content.replace('"', '\\"') + ctx.run(f'git tag -a {version} -m "Version {version}\n\n{tag_content}"') + + ctx.run(f'python setup.py sdist bdist_wheel') + + dist_pattern = f'{PACKAGE_NAME.replace("-", "[-_]")}-*' + artifacts = list(ROOT.joinpath('dist').glob(dist_pattern)) + filename_display = '\n'.join(f' {a}' for a in artifacts) + print(f'[release] Will upload:\n{filename_display}') + try: + input('[release] Release ready. ENTER to upload, CTRL-C to abort: ') + except KeyboardInterrupt: + print('\nAborted!') + return + + arg_display = ' '.join(f'"{n}"' for n in artifacts) + ctx.run(f'twine upload --repository="{repo}" {arg_display}') + + version = _prebump(version, prebump) + _write_version(version) + + ctx.run(f'git commit -am "Prebump to {version}"') + + +@invoke.task +def build_docs(ctx): + _current_version = _read_version() + minor = [str(i) for i in _current_version.release[:2]] + docs_folder = (ROOT / 'docs').as_posix() + if not docs_folder.endswith('/'): + docs_folder = '{0}/'.format(docs_folder) + args = ["--ext-autodoc", "--ext-viewcode", "-o", docs_folder] + args.extend(["-A", "'Dan Ryan '"]) + args.extend(["-R", str(_current_version)]) + args.extend(["-V", ".".join(minor)]) + args.extend(["-e", "-M", "-F", f"src/{PACKAGE_NAME}"]) + print("Building docs...") + ctx.run("sphinx-apidoc {0}".format(" ".join(args))) + + +@invoke.task +def clean_mdchangelog(ctx): + changelog = ROOT / "CHANGELOG.md" + content = changelog.read_text() + content = re.sub( + r"([^\n]+)\n?\s+\[[\\]+(#\d+)\]\(https://github\.com/sarugaku/[\w\-]+/issues/\d+\)", + r"\1 \2", + content, + flags=re.MULTILINE, + ) + changelog.write_text(content) diff --git a/tests/fixtures/fake-package/tox.ini b/tests/fixtures/fake-package/tox.ini new file mode 100644 index 00000000..2bc8e1d2 --- /dev/null +++ b/tests/fixtures/fake-package/tox.ini @@ -0,0 +1,37 @@ +[tox] +envlist = + docs, packaging, py27, py35, py36, py37, coverage-report + +[testenv] +passenv = CI GIT_SSL_CAINFO +setenv = + LC_ALL = en_US.UTF-8 +deps = + coverage + -e .[tests] +commands = coverage run --parallel -m pytest --timeout 300 [] +install_command = python -m pip install {opts} {packages} +usedevelop = True + +[testenv:coverage-report] +deps = coverage +skip_install = true +commands = + coverage combine + coverage report + +[testenv:docs] +deps = + -r{toxinidir}/docs/requirements.txt + -e .[tests] +commands = + sphinx-build -d {envtmpdir}/doctrees -b html docs docs/build/html + sphinx-build -d {envtmpdir}/doctrees -b man docs docs/build/man + +[testenv:packaging] +deps = + check-manifest + readme_renderer +commands = + check-manifest + python setup.py check -m -r -s diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index d2ad24ea..b05eacfa 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,38 +1,60 @@ +# -*- coding=utf-8 -*- +from __future__ import absolute_import, print_function +import errno import json +import logging import os +import shutil +import signal +import socket import sys +import time import warnings +from shutil import copyfileobj, rmtree as _rmtree + import pytest +import requests -from pipenv._compat import TemporaryDirectory, Path +from pipenv.vendor.vistir.compat import ResourceWarning, fs_str, fs_encode, FileNotFoundError, PermissionError, TemporaryDirectory +from pipenv.vendor.vistir.misc import run +from pipenv.vendor.vistir.contextmanagers import temp_environ +from pipenv.vendor.vistir.path import mkdir_p, create_tracked_tempdir, handle_remove_readonly + +from pipenv._compat import Path from pipenv.exceptions import VirtualenvActivationException -from pipenv.utils import temp_environ -from pipenv.vendor import delegator -from pipenv.vendor import requests -from pipenv.vendor import toml -from pipenv.vendor import tomlkit -from pytest_pypi.app import prepare_packages as prepare_pypi_packages -from vistir.compat import ResourceWarning, fs_str -from vistir.path import mkdir_p - +from pipenv.vendor import delegator, toml, tomlkit +from pytest_pypi.app import prepare_fixtures, prepare_packages as prepare_pypi_packages +log = logging.getLogger(__name__) warnings.simplefilter("default", category=ResourceWarning) HAS_WARNED_GITHUB = False +def try_internet(url="http://httpbin.org/ip", timeout=1.5): + resp = requests.get(url, timeout=timeout) + resp.raise_for_status() + + def check_internet(): - try: - # Kenneth represents the Internet LGTM. - resp = requests.get('http://httpbin.org/ip', timeout=1.0) - resp.raise_for_status() - except Exception: - warnings.warn('Cannot connect to HTTPBin...', RuntimeWarning) - warnings.warn('Will skip tests requiring Internet', RuntimeWarning) - return False - return True + has_internet = False + for url in ("http://httpbin.org/ip", "http://clients3.google.com/generate_204"): + try: + try_internet(url) + except KeyboardInterrupt: + warnings.warn( + "Skipped connecting to internet: {0}".format(url), RuntimeWarning + ) + except Exception: + warnings.warn( + "Failed connecting to internet: {0}".format(url), RuntimeWarning + ) + else: + has_internet = True + break + return has_internet def check_github_ssh(): @@ -45,6 +67,10 @@ def check_github_ssh(): # return_code=255 and say 'Permission denied (publickey).' c = delegator.run('ssh -T git@github.com') res = True if c.return_code == 1 else False + except KeyboardInterrupt: + warnings.warn( + "KeyboardInterrupt while checking GitHub ssh access", RuntimeWarning + ) except Exception: pass global HAS_WARNED_GITHUB @@ -70,16 +96,31 @@ def check_for_mercurial(): TESTS_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) PYPI_VENDOR_DIR = os.path.join(TESTS_ROOT, 'pypi') WE_HAVE_HG = check_for_mercurial() +prepare_fixtures(os.path.join(PYPI_VENDOR_DIR, "fixtures")) prepare_pypi_packages(PYPI_VENDOR_DIR) def pytest_runtest_setup(item): - if item.get_marker('needs_internet') is not None and not WE_HAVE_INTERNET: + if item.get_closest_marker('needs_internet') is not None and not WE_HAVE_INTERNET: pytest.skip('requires internet') - if item.get_marker('needs_github_ssh') is not None and not WE_HAVE_GITHUB_SSH_KEYS: + if item.get_closest_marker('needs_github_ssh') is not None and not WE_HAVE_GITHUB_SSH_KEYS: pytest.skip('requires github ssh') - if item.get_marker('needs_hg') is not None and not WE_HAVE_HG: + if item.get_closest_marker('needs_hg') is not None and not WE_HAVE_HG: pytest.skip('requires mercurial') + if item.get_closest_marker('skip_py27_win') is not None and ( + sys.version_info[:2] <= (2, 7) and os.name == "nt" + ): + pytest.skip('must use python > 2.7 on windows') + if item.get_closest_marker('py3_only') is not None and ( + sys.version_info < (3, 0) + ): + pytest.skip('test only runs on python 3') + if item.get_closest_marker('skip_osx') is not None and sys.platform == 'darwin': + pytest.skip('test does not apply on OSX') + if item.get_closest_marker('lte_py36') is not None and ( + sys.version_info >= (3, 7) + ): + pytest.skip('test only runs on python < 3.7') @pytest.fixture @@ -91,9 +132,33 @@ def pathlib_tmpdir(request, tmpdir): pass +def _create_tracked_dir(): + tmp_location = os.environ.get("TEMP", os.environ.get("TMP")) + temp_args = {"prefix": "pipenv-", "suffix": "-test"} + if tmp_location is not None: + temp_args["dir"] = tmp_location + temp_path = create_tracked_tempdir(**temp_args) + return temp_path + + +@pytest.fixture +def vistir_tmpdir(): + temp_path = _create_tracked_dir() + yield Path(temp_path) + + +@pytest.fixture(name='create_tmpdir') +def vistir_tmpdir_factory(): + + def create_tmpdir(): + return Path(_create_tracked_dir()) + + yield create_tmpdir + + # Borrowed from pip's test runner filesystem isolation @pytest.fixture(autouse=True) -def isolate(pathlib_tmpdir): +def isolate(create_tmpdir): """ Isolate our tests so that things like global configuration files and the like do not affect our test results. @@ -102,18 +167,22 @@ def isolate(pathlib_tmpdir): """ # Create a directory to use as our home location. - home_dir = os.path.join(str(pathlib_tmpdir), "home") + home_dir = os.path.join(str(create_tmpdir()), "home") os.makedirs(home_dir) mkdir_p(os.path.join(home_dir, ".config", "git")) - with open(os.path.join(home_dir, ".config", "git", "config"), "wb") as fp: + git_config_file = os.path.join(home_dir, ".config", "git", "config") + with open(git_config_file, "wb") as fp: fp.write( b"[user]\n\tname = pipenv\n\temail = pipenv@pipenv.org\n" ) + # os.environ["GIT_CONFIG"] = fs_str(git_config_file) os.environ["GIT_CONFIG_NOSYSTEM"] = fs_str("1") os.environ["GIT_AUTHOR_NAME"] = fs_str("pipenv") os.environ["GIT_AUTHOR_EMAIL"] = fs_str("pipenv@pipenv.org") - mkdir_p(os.path.join(home_dir, ".virtualenvs")) - os.environ["WORKON_HOME"] = fs_str(os.path.join(home_dir, ".virtualenvs")) + workon_home = create_tmpdir() + os.environ["WORKON_HOME"] = fs_str(str(workon_home)) + os.environ["HOME"] = home_dir + mkdir_p(os.path.join(home_dir, "projects")) # Ignore PIPENV_ACTIVE so that it works as under a bare environment. os.environ.pop("PIPENV_ACTIVE", None) os.environ.pop("VIRTUAL_ENV", None) @@ -128,11 +197,15 @@ WE_HAVE_GITHUB_SSH_KEYS = check_github_ssh() class _Pipfile(object): def __init__(self, path): self.path = path - self.document = tomlkit.document() - self.document["sources"] = tomlkit.aot() - self.document["requires"] = tomlkit.table() - self.document["packages"] = tomlkit.table() - self.document["dev_packages"] = tomlkit.table() + if self.path.exists(): + self.loads() + else: + self.document = tomlkit.document() + self.document["source"] = self.document.get("source", tomlkit.aot()) + self.document["requires"] = self.document.get("requires", tomlkit.table()) + self.document["packages"] = self.document.get("packages", tomlkit.table()) + self.document["dev_packages"] = self.document.get("dev_packages", tomlkit.table()) + super(_Pipfile, self).__init__() def install(self, package, value, dev=False): section = "packages" if not dev else "dev_packages" @@ -144,15 +217,30 @@ class _Pipfile(object): self.document[section][package] = value self.write() + def remove(self, package, dev=False): + section = "packages" if not dev else "dev_packages" + if not dev and package not in self.document[section]: + if package in self.document["dev_packages"]: + section = "dev_packages" + del self.document[section][package] + self.write() + + def add(self, package, value, dev=False): + self.install(package, value, dev=dev) + + def update(self, package, value, dev=False): + self.install(package, value, dev=dev) + def loads(self): self.document = tomlkit.loads(self.path.read_text()) def dumps(self): source_table = tomlkit.table() - source_table["url"] = os.environ.get("PIPENV_TEST_INDEX") + pypi_url = os.environ.get("PIPENV_PYPI_URL", "https://pypi.org/simple") + source_table["url"] = os.environ.get("PIPENV_TEST_INDEX", pypi_url) source_table["verify_ssl"] = False source_table["name"] = "pipenv_test_index" - self.document["sources"].append(source_table) + self.document["source"].append(source_table) return tomlkit.dumps(self.document) def write(self): @@ -162,25 +250,70 @@ class _Pipfile(object): def get_fixture_path(cls, path): return Path(__file__).absolute().parent.parent / "test_artifacts" / path + @classmethod + def get_url(cls, pkg=None, filename=None): + pypi = os.environ.get("PIPENV_PYPI_URL") + if not pkg and not filename: + return pypi if pypi else "https://pypi.org/" + file_path = filename + if pkg and filename: + file_path = os.path.join(pkg, filename) + if filename and not pkg: + pkg = os.path.basename(filename) + fixture_pypi = os.getenv("ARTIFACT_PYPI_URL") + if fixture_pypi: + if pkg and not filename: + url = "{0}/artifacts/{1}".format(fixture_pypi, pkg) + else: + url = "{0}/artifacts/{1}/{2}".format(fixture_pypi, pkg, filename) + return url + if pkg and not filename: + return cls.get_fixture_path(file_path).as_uri() + class _PipenvInstance(object): """An instance of a Pipenv Project...""" - def __init__(self, pypi=None, pipfile=True, chdir=False, path=None, home_dir=None): - self.pypi = pypi - self.original_umask = os.umask(0o007) + def __init__( + self, pypi=None, pipfile=True, chdir=False, path=None, home_dir=None, + venv_root=None, ignore_virtualenvs=True, venv_in_project=True, name=None + ): + self.index_url = os.getenv("PIPENV_TEST_INDEX") + self.pypi = None + if pypi: + self.pypi = pypi.url + elif self.index_url is not None: + self.pypi, _, _ = self.index_url.rpartition("/") if self.index_url else "" + self.index = os.getenv("PIPENV_PYPI_INDEX") + os.environ["PYTHONWARNINGS"] = "ignore:DEPRECATION" + if ignore_virtualenvs: + os.environ["PIPENV_IGNORE_VIRTUALENVS"] = fs_str("1") + if venv_root: + os.environ["VIRTUAL_ENV"] = venv_root + if venv_in_project: + os.environ["PIPENV_VENV_IN_PROJECT"] = fs_str("1") + else: + os.environ.pop("PIPENV_VENV_IN_PROJECT", None) + self.original_dir = os.path.abspath(os.curdir) - os.environ["PIPENV_NOSPIN"] = fs_str("1") - os.environ["CI"] = fs_str("1") - warnings.simplefilter("ignore", category=ResourceWarning) - warnings.filterwarnings("ignore", category=ResourceWarning, message="unclosed.*") path = path if path else os.environ.get("PIPENV_PROJECT_DIR", None) + if name is not None: + path = Path(os.environ["HOME"]) / "projects" / name + path.mkdir(exist_ok=True) if not path: - self._path = TemporaryDirectory(suffix='-project', prefix='pipenv-') + path = TemporaryDirectory(suffix='-project', prefix='pipenv-') + if isinstance(path, TemporaryDirectory): + self._path = path path = Path(self._path.name) try: self.path = str(path.resolve()) except OSError: self.path = str(path.absolute()) + elif isinstance(path, Path): + self._path = path + try: + self.path = str(path.resolve()) + except OSError: + self.path = str(path.absolute()) else: self._path = path self.path = path @@ -188,8 +321,10 @@ class _PipenvInstance(object): self.pipfile_path = None self.chdir = chdir - if self.pypi: - os.environ['PIPENV_TEST_INDEX'] = fs_str('{0}/simple'.format(self.pypi.url)) + if self.pypi and "PIPENV_PYPI_URL" not in os.environ: + os.environ['PIPENV_PYPI_URL'] = fs_str('{0}'.format(self.pypi)) + # os.environ['PIPENV_PYPI_URL'] = fs_str('{0}'.format(self.pypi.url)) + # os.environ['PIPENV_TEST_INDEX'] = fs_str('{0}/simple'.format(self.pypi.url)) if pipfile: p_path = os.sep.join([self.path, 'Pipfile']) @@ -201,10 +336,6 @@ class _PipenvInstance(object): self._pipfile = _Pipfile(Path(p_path)) def __enter__(self): - os.environ['PIPENV_DONT_USE_PYENV'] = fs_str('1') - os.environ['PIPENV_IGNORE_VIRTUALENVS'] = fs_str('1') - os.environ['PIPENV_VENV_IN_PROJECT'] = fs_str('1') - os.environ['PIPENV_NOSPIN'] = fs_str('1') if self.chdir: os.chdir(self.path) return self @@ -214,13 +345,12 @@ class _PipenvInstance(object): if self.chdir: os.chdir(self.original_dir) self.path = None - if self._path: + if self._path and getattr(self._path, "cleanup", None): try: self._path.cleanup() except OSError as e: _warn_msg = warn_msg.format(e) warnings.warn(_warn_msg, ResourceWarning) - os.umask(self.original_umask) def pipenv(self, cmd, block=True): if self.pipfile_path and os.path.isfile(self.pipfile_path): @@ -229,8 +359,10 @@ class _PipenvInstance(object): with TemporaryDirectory(prefix='pipenv-', suffix='-cache') as tempdir: os.environ['PIPENV_CACHE_DIR'] = fs_str(tempdir.name) - c = delegator.run('pipenv {0}'.format(cmd), block=block, - cwd=os.path.abspath(self.path)) + c = delegator.run( + 'pipenv {0}'.format(cmd), block=block, + cwd=os.path.abspath(self.path), env=os.environ.copy() + ) if 'PIPENV_CACHE_DIR' in os.environ: del os.environ['PIPENV_CACHE_DIR'] @@ -241,7 +373,7 @@ class _PipenvInstance(object): if block: print('$ pipenv {0}'.format(cmd)) print(c.out) - print(c.err) + print(c.err, file=sys.stderr) if c.return_code != 0: print("Command failed...") @@ -265,15 +397,24 @@ class _PipenvInstance(object): return os.sep.join([self.path, 'Pipfile.lock']) +def _rmtree_func(path, ignore_errors=True, onerror=None): + directory = fs_encode(path) + global _rmtree + shutil_rmtree = _rmtree + if onerror is None: + onerror = handle_remove_readonly + try: + shutil_rmtree(directory, ignore_errors=ignore_errors, onerror=onerror) + except (IOError, OSError, FileNotFoundError, PermissionError) as exc: + # Ignore removal failures where the file doesn't exist + if exc.errno != errno.ENOENT: + raise + + @pytest.fixture() -def PipenvInstance(): - yield _PipenvInstance - - -@pytest.fixture(autouse=True) -def pip_src_dir(request, pathlib_tmpdir): +def pip_src_dir(request, vistir_tmpdir): old_src_dir = os.environ.get('PIP_SRC', '') - os.environ['PIP_SRC'] = pathlib_tmpdir.as_posix() + os.environ['PIP_SRC'] = vistir_tmpdir.as_posix() def finalize(): os.environ['PIP_SRC'] = fs_str(old_src_dir) @@ -282,25 +423,100 @@ def pip_src_dir(request, pathlib_tmpdir): return request +@pytest.fixture() +def PipenvInstance(pip_src_dir, monkeypatch, pypi): + with temp_environ(), monkeypatch.context() as m: + m.setattr(shutil, "rmtree", _rmtree_func) + original_umask = os.umask(0o007) + m.setenv("PIPENV_NOSPIN", fs_str("1")) + m.setenv("CI", fs_str("1")) + m.setenv('PIPENV_DONT_USE_PYENV', fs_str('1')) + m.setenv("PIPENV_TEST_INDEX", "{0}/simple".format(pypi.url)) + m.setenv("PIPENV_PYPI_INDEX", "simple") + m.setenv("ARTIFACT_PYPI_URL", pypi.url) + m.setenv("PIPENV_PYPI_URL", pypi.url) + warnings.simplefilter("ignore", category=ResourceWarning) + warnings.filterwarnings("ignore", category=ResourceWarning, message="unclosed.*") + try: + yield _PipenvInstance + finally: + os.umask(original_umask) + + +@pytest.fixture() +def PipenvInstance_NoPyPI(monkeypatch, pip_src_dir, pypi): + with temp_environ(), monkeypatch.context() as m: + m.setattr(shutil, "rmtree", _rmtree_func) + original_umask = os.umask(0o007) + m.setenv("PIPENV_NOSPIN", fs_str("1")) + m.setenv("CI", fs_str("1")) + m.setenv('PIPENV_DONT_USE_PYENV', fs_str('1')) + m.setenv("PIPENV_TEST_INDEX", "{0}/simple".format(pypi.url)) + m.setenv("ARTIFACT_PYPI_URL", pypi.url) + warnings.simplefilter("ignore", category=ResourceWarning) + warnings.filterwarnings("ignore", category=ResourceWarning, message="unclosed.*") + try: + yield _PipenvInstance + finally: + os.umask(original_umask) + + @pytest.fixture() def testsroot(): return TESTS_ROOT -@pytest.fixture() -def virtualenv(pathlib_tmpdir): - virtualenv_path = pathlib_tmpdir / "venv" - with temp_environ(): - c = delegator.run("virtualenv {}".format(virtualenv_path), block=True) - assert c.return_code == 0 - for name in ("bin", "Scripts"): - activate_this = virtualenv_path / name / "activate_this.py" - if activate_this.exists(): - with open(str(activate_this)) as f: - code = compile(f.read(), str(activate_this), "exec") - exec(code, dict(__file__=str(activate_this))) - break +class VirtualEnv(object): + def __init__(self, name="venv", base_dir=None): + if base_dir is None: + base_dir = Path(_create_tracked_dir()) + self.base_dir = base_dir + self.name = name + self.path = base_dir / name + + def __enter__(self): + self._old_environ = os.environ.copy() + self.create() + return self.activate() + + def __exit__(self, *args, **kwargs): + os.environ = self._old_environ + + def create(self): + python = Path(sys.executable).absolute().as_posix() + cmd = [ + python, "-m", "virtualenv", self.path.absolute().as_posix() + ] + c = run( + cmd, verbose=False, return_object=True, write_to_stdout=False, + combine_stderr=False, block=True, nospin=True, + ) + # cmd = "{0} -m virtualenv {1}".format(python, self.path.as_posix()) + # c = delegator.run(cmd, block=True) + assert c.returncode == 0 + + def activate(self): + script_path = "Scripts" if os.name == "nt" else "bin" + activate_this = self.path / script_path / "activate_this.py" + if activate_this.exists(): + with open(str(activate_this)) as f: + code = compile(f.read(), str(activate_this), "exec") + exec(code, dict(__file__=str(activate_this))) + os.environ["VIRTUAL_ENV"] = str(self.path) + try: + return self.path.absolute().resolve() + except OSError: + return self.path.absolute() else: raise VirtualenvActivationException("Can't find the activate_this.py script.") - os.environ["VIRTUAL_ENV"] = str(virtualenv_path) - yield virtualenv_path + + +@pytest.fixture() +def virtualenv(vistir_tmpdir): + with temp_environ(), VirtualEnv(base_dir=vistir_tmpdir) as venv: + yield venv + + +@pytest.fixture() +def raw_venv(): + yield VirtualEnv diff --git a/tests/integration/test_cli.py b/tests/integration/test_cli.py index 7ebcee1d..1ee9f64a 100644 --- a/tests/integration/test_cli.py +++ b/tests/integration/test_cli.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function """Tests to ensure `pipenv --option` works. """ @@ -5,13 +7,15 @@ import os import re import pytest + from flaky import flaky + from pipenv.utils import normalize_drive @pytest.mark.cli -def test_pipenv_where(PipenvInstance, pypi_secure): - with PipenvInstance(pypi=pypi_secure) as p: +def test_pipenv_where(PipenvInstance): + with PipenvInstance() as p: c = p.pipenv("--where") assert c.ok assert normalize_drive(p.path) in c.out @@ -39,6 +43,20 @@ def test_pipenv_py(PipenvInstance): assert os.path.basename(python).startswith('python') +@pytest.mark.cli +def test_pipenv_site_packages(PipenvInstance): + with PipenvInstance() as p: + c = p.pipenv('--python python --site-packages') + assert c.return_code == 0 + assert 'Making site-packages available' in c.err + + # no-global-site-packages.txt under stdlib dir should not exist. + c = p.pipenv('run python -c "import sysconfig; print(sysconfig.get_path(\'stdlib\'))"') + assert c.return_code == 0 + stdlib_path = c.out.strip() + assert not os.path.isfile(os.path.join(stdlib_path, 'no-global-site-packages.txt')) + + @pytest.mark.cli def test_pipenv_support(PipenvInstance): with PipenvInstance() as p: @@ -64,54 +82,68 @@ def test_pipenv_rm(PipenvInstance): @pytest.mark.cli -def test_pipenv_graph(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi) as p: - c = p.pipenv('install requests') +def test_pipenv_graph(PipenvInstance): + with PipenvInstance() as p: + c = p.pipenv('install tablib') assert c.ok graph = p.pipenv("graph") assert graph.ok - assert "requests" in graph.out + assert "tablib" in graph.out graph_json = p.pipenv("graph --json") assert graph_json.ok - assert "requests" in graph_json.out + assert "tablib" in graph_json.out graph_json_tree = p.pipenv("graph --json-tree") assert graph_json_tree.ok - assert "requests" in graph_json_tree.out + assert "tablib" in graph_json_tree.out @pytest.mark.cli -def test_pipenv_graph_reverse(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi) as p: - c = p.pipenv('install requests==2.18.4') +def test_pipenv_graph_reverse(PipenvInstance): + with PipenvInstance() as p: + c = p.pipenv('install tablib==0.13.0') assert c.ok c = p.pipenv('graph --reverse') assert c.ok output = c.out - requests_dependency = [ - ('certifi', 'certifi>=2017.4.17'), - ('chardet', 'chardet(>=3.0.2,<3.1.0|<3.1.0,>=3.0.2)'), - ('idna', 'idna(>=2.5,<2.7|<2.7,>=2.5)'), - ('urllib3', 'urllib3(>=1.21.1,<1.23|<1.23,>=1.21.1)') - ] - - for dep_name, dep_constraint in requests_dependency: - dep_match = re.search(r'^{}==[\d.]+$'.format(dep_name), output, flags=re.MULTILINE) - dep_requests_match = re.search(r'^ - requests==2.18.4 \[requires: {}\]$'.format(dep_constraint), output, flags=re.MULTILINE) - assert dep_match is not None - assert dep_requests_match is not None - assert dep_requests_match.start() > dep_match.start() - c = p.pipenv('graph --reverse --json') assert c.return_code == 1 assert 'Warning: Using both --reverse and --json together is not supported.' in c.err + requests_dependency = [ + ('backports.csv', 'backports.csv'), + ('odfpy', 'odfpy'), + ('openpyxl', 'openpyxl>=2.4.0'), + ('pyyaml', 'pyyaml'), + ('xlrd', 'xlrd'), + ('xlwt', 'xlwt'), + ] + + for dep_name, dep_constraint in requests_dependency: + pat = r'^[ -]*{}==[\d.]+'.format(dep_name) + dep_match = re.search(pat, output, flags=re.MULTILINE) + assert dep_match is not None, '{} not found in {}'.format(pat, output) + + # openpyxl should be indented + if dep_name == 'openpyxl': + openpyxl_dep = re.search(r'^openpyxl', output, flags=re.MULTILINE) + assert openpyxl_dep is None, 'openpyxl should not appear at begining of lines in {}'.format(output) + + assert ' - openpyxl==2.5.4 [requires: et-xmlfile]' in output + else: + dep_match = re.search(r'^[ -]*{}==[\d.]+$'.format(dep_name), output, flags=re.MULTILINE) + assert dep_match is not None, '{} not found at beginning of line in {}'.format(dep_name, output) + + dep_requests_match = re.search(r'^ +- tablib==0.13.0 \[requires: {}\]$'.format(dep_constraint), output, flags=re.MULTILINE) + assert dep_requests_match is not None, 'constraint {} not found in {}'.format(dep_constraint, output) + assert dep_requests_match.start() > dep_match.start() + @pytest.mark.cli @pytest.mark.needs_internet(reason='required by check') @flaky -def test_pipenv_check(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi) as p: +def test_pipenv_check(PipenvInstance): + with PipenvInstance() as p: p.pipenv('install requests==1.0.0') c = p.pipenv('check') assert c.return_code != 0 @@ -179,8 +211,8 @@ def test_man(PipenvInstance): @pytest.mark.cli -def test_install_parse_error(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi) as p: +def test_install_parse_error(PipenvInstance): + with PipenvInstance() as p: # Make sure unparseable packages don't wind up in the pipfile # Escape $ for shell input @@ -199,20 +231,37 @@ def test_install_parse_error(PipenvInstance, pypi): @pytest.mark.code @pytest.mark.check @pytest.mark.unused -@pytest.mark.skip(reason="non-deterministic") -def test_check_unused(PipenvInstance, pypi): - with PipenvInstance(chdir=True, pypi=pypi) as p: +@pytest.mark.skip_osx +@pytest.mark.needs_internet(reason='required by check') +def test_check_unused(PipenvInstance): + with PipenvInstance(chdir=True) as p: with open('__init__.py', 'w') as f: contents = """ -import tablib +import click import records +import flask """.strip() f.write(contents) - p.pipenv('install requests') - p.pipenv('install tablib') - p.pipenv('install records') + p.pipenv('install requests click flask') - assert all(pkg in p.pipfile['packages'] for pkg in ['requests', 'tablib', 'records']) + assert all(pkg in p.pipfile['packages'] for pkg in ['requests', 'click', 'flask']), p.pipfile["packages"] c = p.pipenv('check --unused .') - assert 'tablib' not in c.out + assert 'click' not in c.out + assert 'flask' not in c.out + + +@pytest.mark.cli +def test_pipenv_clear(PipenvInstance): + with PipenvInstance() as p: + c = p.pipenv('--clear') + assert c.return_code == 0 + assert 'Clearing caches' in c.out + + +@pytest.mark.cli +def test_pipenv_three(PipenvInstance): + with PipenvInstance() as p: + c = p.pipenv('--three') + assert c.return_code == 0 + assert 'Successfully created virtual environment' in c.err diff --git a/tests/integration/test_dot_venv.py b/tests/integration/test_dot_venv.py index 53f5fb1d..aa52dd5e 100644 --- a/tests/integration/test_dot_venv.py +++ b/tests/integration/test_dot_venv.py @@ -1,18 +1,20 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function import os -from pipenv._compat import TemporaryDirectory, Path -from pipenv.project import Project -from pipenv.utils import temp_environ, normalize_drive, get_windows_path -from pipenv.vendor import delegator - import pytest +from pipenv._compat import Path, TemporaryDirectory +from pipenv.project import Project +from pipenv.utils import get_windows_path, normalize_drive, temp_environ +from pipenv.vendor import delegator + @pytest.mark.dotvenv -def test_venv_in_project(PipenvInstance, pypi): +def test_venv_in_project(PipenvInstance): with temp_environ(): os.environ['PIPENV_VENV_IN_PROJECT'] = '1' - with PipenvInstance(pypi=pypi) as p: + with PipenvInstance() as p: c = p.pipenv('install requests') assert c.return_code == 0 assert normalize_drive(p.path) in p.pipenv('--venv').out @@ -34,8 +36,8 @@ def test_venv_at_project_root(PipenvInstance): @pytest.mark.dotvenv -def test_reuse_previous_venv(PipenvInstance, pypi): - with PipenvInstance(chdir=True, pypi=pypi) as p: +def test_reuse_previous_venv(PipenvInstance): + with PipenvInstance(chdir=True) as p: os.mkdir('.venv') c = p.pipenv('install requests') assert c.return_code == 0 @@ -44,11 +46,11 @@ def test_reuse_previous_venv(PipenvInstance, pypi): @pytest.mark.dotvenv @pytest.mark.parametrize('venv_name', ('test-venv', os.path.join('foo', 'test-venv'))) -def test_venv_file(venv_name, PipenvInstance, pypi): +def test_venv_file(venv_name, PipenvInstance): """Tests virtualenv creation when a .venv file exists at the project root and contains a venv name. """ - with PipenvInstance(pypi=pypi, chdir=True) as p: + with PipenvInstance(chdir=True) as p: file_path = os.path.join(p.path, '.venv') with open(file_path, 'w') as f: f.write(venv_name) @@ -68,20 +70,20 @@ def test_venv_file(venv_name, PipenvInstance, pypi): venv_loc = Path(c.out.strip()).absolute() assert venv_loc.exists() assert venv_loc.joinpath('.project').exists() - venv_path = venv_loc.as_posix() + venv_path = normalize_drive(venv_loc.as_posix()) if os.path.sep in venv_name: venv_expected_path = Path(p.path).joinpath(venv_name).absolute().as_posix() else: venv_expected_path = Path(workon_home.name).joinpath(venv_name).absolute().as_posix() - assert venv_path == venv_expected_path + assert venv_path == normalize_drive(venv_expected_path) @pytest.mark.dotvenv -def test_venv_file_with_path(PipenvInstance, pypi): +def test_venv_file_with_path(PipenvInstance): """Tests virtualenv creation when a .venv file exists at the project root and contains an absolute path. """ - with temp_environ(), PipenvInstance(chdir=True, pypi=pypi) as p: + with temp_environ(), PipenvInstance(chdir=True) as p: with TemporaryDirectory( prefix='pipenv-', suffix='-test_venv' ) as venv_path: diff --git a/tests/integration/test_install_basic.py b/tests/integration/test_install_basic.py index 25cb55ea..80ccdf0e 100644 --- a/tests/integration/test_install_basic.py +++ b/tests/integration/test_install_basic.py @@ -1,20 +1,21 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function import os -from pipenv.utils import temp_environ -from pipenv._compat import TemporaryDirectory, Path -from pipenv.vendor import delegator -from pipenv.project import Project - import pytest from flaky import flaky +from pipenv._compat import Path, TemporaryDirectory +from pipenv.project import Project +from pipenv.utils import temp_environ +from pipenv.vendor import delegator + @pytest.mark.install @pytest.mark.setup -@pytest.mark.skip(reason="this doesn't work on travis") -def test_basic_setup(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi) as p: +def test_basic_setup(PipenvInstance): + with PipenvInstance() as p: with PipenvInstance(pipfile=False) as p: c = p.pipenv("install requests") assert c.return_code == 0 @@ -27,10 +28,11 @@ def test_basic_setup(PipenvInstance, pypi): assert "certifi" in p.lockfile["default"] -@pytest.mark.install @flaky -def test_basic_install(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi) as p: +@pytest.mark.install +@pytest.mark.skip_osx +def test_basic_install(PipenvInstance): + with PipenvInstance() as p: c = p.pipenv("install requests") assert c.return_code == 0 assert "requests" in p.pipfile["packages"] @@ -41,10 +43,10 @@ def test_basic_install(PipenvInstance, pypi): assert "certifi" in p.lockfile["default"] -@pytest.mark.install @flaky -def test_mirror_install(PipenvInstance, pypi): - with temp_environ(), PipenvInstance(chdir=True, pypi=pypi) as p: +@pytest.mark.install +def test_mirror_install(PipenvInstance): + with temp_environ(), PipenvInstance(chdir=True) as p: mirror_url = os.environ.pop( "PIPENV_TEST_INDEX", "https://pypi.python.org/simple" ) @@ -67,10 +69,10 @@ def test_mirror_install(PipenvInstance, pypi): assert "certifi" in p.lockfile["default"] +@flaky @pytest.mark.install @pytest.mark.needs_internet -@flaky -def test_bad_mirror_install(PipenvInstance, pypi): +def test_bad_mirror_install(PipenvInstance): with temp_environ(), PipenvInstance(chdir=True) as p: # This demonstrates that the mirror parameter is being used os.environ.pop("PIPENV_TEST_INDEX", None) @@ -78,11 +80,11 @@ def test_bad_mirror_install(PipenvInstance, pypi): assert c.return_code != 0 -@pytest.mark.complex @pytest.mark.lock +@pytest.mark.complex @pytest.mark.skip(reason="Does not work unless you can explicitly install into py2") -def test_complex_lock(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi) as p: +def test_complex_lock(PipenvInstance): + with PipenvInstance() as p: c = p.pipenv("install apscheduler") assert c.return_code == 0 assert "apscheduler" in p.pipfile["packages"] @@ -90,11 +92,11 @@ def test_complex_lock(PipenvInstance, pypi): assert "futures" in p.lockfile[u"default"] +@flaky @pytest.mark.dev @pytest.mark.run -@flaky -def test_basic_dev_install(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi) as p: +def test_basic_dev_install(PipenvInstance): + with PipenvInstance() as p: c = p.pipenv("install requests --dev") assert c.return_code == 0 assert "requests" in p.pipfile["dev-packages"] @@ -108,37 +110,37 @@ def test_basic_dev_install(PipenvInstance, pypi): assert c.return_code == 0 +@flaky @pytest.mark.dev @pytest.mark.install -@flaky -def test_install_without_dev(PipenvInstance, pypi): +def test_install_without_dev(PipenvInstance): """Ensure that running `pipenv install` doesn't install dev packages""" - with PipenvInstance(pypi=pypi, chdir=True) as p: + with PipenvInstance(chdir=True) as p: with open(p.pipfile_path, "w") as f: contents = """ [packages] six = "*" [dev-packages] -pytz = "*" +tablib = "*" """.strip() f.write(contents) c = p.pipenv("install") assert c.return_code == 0 assert "six" in p.pipfile["packages"] - assert "pytz" in p.pipfile["dev-packages"] + assert "tablib" in p.pipfile["dev-packages"] assert "six" in p.lockfile["default"] - assert "pytz" in p.lockfile["develop"] - c = p.pipenv('run python -c "import pytz"') + assert "tablib" in p.lockfile["develop"] + c = p.pipenv('run python -c "import tablib"') assert c.return_code != 0 c = p.pipenv('run python -c "import six"') assert c.return_code == 0 -@pytest.mark.install @flaky -def test_install_without_dev_section(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi) as p: +@pytest.mark.install +def test_install_without_dev_section(PipenvInstance): + with PipenvInstance() as p: with open(p.pipfile_path, "w") as f: contents = """ [packages] @@ -155,11 +157,11 @@ six = "*" assert c.return_code == 0 +@flaky @pytest.mark.extras @pytest.mark.install -@flaky -def test_extras_install(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi, chdir=True) as p: +def test_extras_install(PipenvInstance): + with PipenvInstance(chdir=True) as p: c = p.pipenv("install requests[socks]") assert c.return_code == 0 assert "requests" in p.pipfile["packages"] @@ -172,11 +174,11 @@ def test_extras_install(PipenvInstance, pypi): assert "pysocks" in p.lockfile["default"] -@pytest.mark.install -@pytest.mark.pin @flaky -def test_windows_pinned_pipfile(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi) as p: +@pytest.mark.pin +@pytest.mark.install +def test_windows_pinned_pipfile(PipenvInstance): + with PipenvInstance() as p: with open(p.pipfile_path, "w") as f: contents = """ [packages] @@ -189,12 +191,12 @@ requests = "==2.19.1" assert "requests" in p.lockfile["default"] +@flaky @pytest.mark.install @pytest.mark.resolver @pytest.mark.backup_resolver -@flaky -def test_backup_resolver(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi) as p: +def test_backup_resolver(PipenvInstance): + with PipenvInstance() as p: with open(p.pipfile_path, "w") as f: contents = """ [packages] @@ -207,11 +209,11 @@ def test_backup_resolver(PipenvInstance, pypi): assert "ibm-db-sa-py3" in p.lockfile["default"] +@flaky @pytest.mark.run @pytest.mark.alt -@flaky -def test_alternative_version_specifier(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi) as p: +def test_alternative_version_specifier(PipenvInstance): + with PipenvInstance() as p: with open(p.pipfile_path, "w") as f: contents = """ [packages] @@ -232,11 +234,11 @@ requests = {version = "*"} assert c.return_code == 0 +@flaky @pytest.mark.run @pytest.mark.alt -@flaky -def test_outline_table_specifier(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi) as p: +def test_outline_table_specifier(PipenvInstance): + with PipenvInstance() as p: with open(p.pipfile_path, "w") as f: contents = """ [packages.requests] @@ -259,8 +261,8 @@ version = "*" @pytest.mark.bad @pytest.mark.install -def test_bad_packages(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi) as p: +def test_bad_packages(PipenvInstance): + with PipenvInstance() as p: c = p.pipenv("install NotAPackage") assert c.return_code > 0 @@ -269,9 +271,9 @@ def test_bad_packages(PipenvInstance, pypi): @pytest.mark.install @pytest.mark.requirements @pytest.mark.skip(reason="Not mocking this.") -def test_requirements_to_pipfile(PipenvInstance, pypi): +def test_requirements_to_pipfile(PipenvInstance): - with PipenvInstance(pipfile=False, chdir=True, pypi=pypi) as p: + with PipenvInstance(pipfile=False, chdir=True) as p: # Write a requirements file with open("requirements.txt", "w") as f: @@ -296,14 +298,15 @@ def test_requirements_to_pipfile(PipenvInstance, pypi): @pytest.mark.install +@pytest.mark.skip_osx @pytest.mark.requirements -def test_skip_requirements_when_pipfile(PipenvInstance, pypi): +def test_skip_requirements_when_pipfile(PipenvInstance): """Ensure requirements.txt is NOT imported when 1. We do `pipenv install [package]` 2. A Pipfile already exists when we run `pipenv install`. """ - with PipenvInstance(chdir=True, pypi=pypi) as p: + with PipenvInstance(chdir=True) as p: with open("requirements.txt", "w") as f: f.write("requests==2.18.1\n") c = p.pipenv("install six") @@ -312,12 +315,13 @@ def test_skip_requirements_when_pipfile(PipenvInstance, pypi): contents = """ [packages] six = "*" -tablib = "<0.12" +fake_package = "<0.12" """.strip() f.write(contents) c = p.pipenv("install") - assert "tablib" in p.pipfile["packages"] - assert "tablib" in p.lockfile["default"] + assert c.ok + assert "fake_package" in p.pipfile["packages"] + assert "fake-package" in p.lockfile["default"] assert "six" in p.pipfile["packages"] assert "six" in p.lockfile["default"] assert "requests" not in p.pipfile["packages"] @@ -326,18 +330,19 @@ tablib = "<0.12" @pytest.mark.cli @pytest.mark.clean -def test_clean_on_empty_venv(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi) as p: +def test_clean_on_empty_venv(PipenvInstance): + with PipenvInstance() as p: c = p.pipenv("clean") assert c.return_code == 0 @pytest.mark.install -def test_install_does_not_extrapolate_environ(PipenvInstance, pypi): +def test_install_does_not_extrapolate_environ(PipenvInstance): """Ensure environment variables are not expanded in lock file. """ - with temp_environ(), PipenvInstance(pypi=pypi, chdir=True) as p: - os.environ["PYPI_URL"] = pypi.url + with temp_environ(), PipenvInstance(chdir=True) as p: + # os.environ["PYPI_URL"] = pypi.url + os.environ["PYPI_URL"] = p.pypi with open(p.pipfile_path, "w") as f: f.write( @@ -374,10 +379,10 @@ def test_editable_no_args(PipenvInstance): @pytest.mark.install @pytest.mark.virtualenv -def test_install_venv_project_directory(PipenvInstance, pypi): +def test_install_venv_project_directory(PipenvInstance): """Test the project functionality during virtualenv creation. """ - with PipenvInstance(pypi=pypi, chdir=True) as p: + with PipenvInstance(chdir=True) as p: with temp_environ(), TemporaryDirectory( prefix="pipenv-", suffix="temp_workon_home" ) as workon_home: @@ -398,8 +403,8 @@ def test_install_venv_project_directory(PipenvInstance, pypi): @pytest.mark.deploy @pytest.mark.system -def test_system_and_deploy_work(PipenvInstance, pypi): - with PipenvInstance(chdir=True, pypi=pypi) as p: +def test_system_and_deploy_work(PipenvInstance): + with PipenvInstance(chdir=True) as p: c = p.pipenv("install six requests") assert c.return_code == 0 c = p.pipenv("--rm") @@ -431,3 +436,42 @@ def test_install_creates_pipfile(PipenvInstance): c = p.pipenv("install") assert c.return_code == 0 assert os.path.isfile(p.pipfile_path) + + +@pytest.mark.install +def test_install_non_exist_dep(PipenvInstance): + with PipenvInstance(chdir=True) as p: + c = p.pipenv("install dateutil") + assert not c.ok + assert "dateutil" not in p.pipfile["packages"] + + +@pytest.mark.install +def test_install_package_with_dots(PipenvInstance): + with PipenvInstance(chdir=True) as p: + c = p.pipenv("install backports.html") + assert c.ok + assert "backports.html" in p.pipfile["packages"] + + +@pytest.mark.install +def test_rewrite_outline_table(PipenvInstance): + with PipenvInstance(chdir=True) as p: + with open(p.pipfile_path, 'w') as f: + contents = """ +[packages] +six = {version = "*"} + +[packages.requests] +version = "*" +extras = ["socks"] + """.strip() + f.write(contents) + c = p.pipenv("install plette") + assert c.return_code == 0 + with open(p.pipfile_path) as f: + contents = f.read() + assert "[packages.requests]" not in contents + assert 'six = {version = "*"}' in contents + assert 'requests = {version = "*"' in contents + assert 'plette = "*"' in contents diff --git a/tests/integration/test_install_markers.py b/tests/integration/test_install_markers.py index 0967026a..00f9c789 100644 --- a/tests/integration/test_install_markers.py +++ b/tests/integration/test_install_markers.py @@ -1,46 +1,45 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function import os import sys -from pipenv.patched import pipfile -from pipenv.project import Project -from pipenv.utils import temp_environ - import pytest from flaky import flaky - -py3_only = pytest.mark.skipif(sys.version_info < (3, 0), reason="requires Python3") -skip_py37 = pytest.mark.skipif(sys.version_info >= (3, 7), reason="Skip for python 3.7") +from pipenv.patched import pipfile +from pipenv.project import Project +from pipenv.utils import temp_environ -@pytest.mark.markers @flaky -def test_package_environment_markers(PipenvInstance, pypi): +@pytest.mark.markers +def test_package_environment_markers(PipenvInstance): - with PipenvInstance(pypi=pypi) as p: + with PipenvInstance() as p: with open(p.pipfile_path, 'w') as f: contents = """ [packages] -tablib = {version = "*", markers="os_name=='splashwear'"} +fake_package = {version = "*", markers="os_name=='splashwear'"} """.strip() f.write(contents) c = p.pipenv('install') assert c.return_code == 0 assert 'Ignoring' in c.out - assert 'markers' in p.lockfile['default']['tablib'] + assert 'markers' in p.lockfile['default']['fake-package'], p.lockfile["default"] - c = p.pipenv('run python -c "import tablib;"') + c = p.pipenv('run python -c "import fake_package;"') assert c.return_code == 1 -@pytest.mark.markers + @flaky -def test_platform_python_implementation_marker(PipenvInstance, pypi): +@pytest.mark.markers +def test_platform_python_implementation_marker(PipenvInstance): """Markers should be converted during locking to help users who input this incorrectly. """ - with PipenvInstance(pypi=pypi) as p: + with PipenvInstance() as p: with open(p.pipfile_path, 'w') as f: contents = """ [packages] @@ -59,17 +58,17 @@ depends-on-marked-package = "*" "platform_python_implementation == 'CPython'" +@flaky @pytest.mark.run @pytest.mark.alt @pytest.mark.install -@flaky -def test_specific_package_environment_markers(PipenvInstance, pypi): +def test_specific_package_environment_markers(PipenvInstance): - with PipenvInstance(pypi=pypi) as p: + with PipenvInstance() as p: with open(p.pipfile_path, 'w') as f: contents = """ [packages] -requests = {version = "*", os_name = "== 'splashwear'"} +fake-package = {version = "*", os_name = "== 'splashwear'"} """.strip() f.write(contents) @@ -77,18 +76,18 @@ requests = {version = "*", os_name = "== 'splashwear'"} assert c.return_code == 0 assert 'Ignoring' in c.out - assert 'markers' in p.lockfile['default']['requests'] + assert 'markers' in p.lockfile['default']['fake-package'] - c = p.pipenv('run python -c "import requests;"') + c = p.pipenv('run python -c "import fake_package;"') assert c.return_code == 1 -@pytest.mark.markers @flaky -def test_top_level_overrides_environment_markers(PipenvInstance, pypi): +@pytest.mark.markers +def test_top_level_overrides_environment_markers(PipenvInstance): """Top-level environment markers should take precedence. """ - with PipenvInstance(pypi=pypi) as p: + with PipenvInstance() as p: with open(p.pipfile_path, 'w') as f: contents = """ [packages] @@ -99,21 +98,21 @@ funcsigs = {version = "*", os_name = "== 'splashwear'"} c = p.pipenv('install') assert c.return_code == 0 - - assert p.lockfile['default']['funcsigs']['markers'] == "os_name == 'splashwear'" + assert "markers" in p.lockfile['default']['funcsigs'], p.lockfile['default']['funcsigs'] + assert p.lockfile['default']['funcsigs']['markers'] == "os_name == 'splashwear'", p.lockfile['default']['funcsigs'] +@flaky @pytest.mark.markers @pytest.mark.install -@flaky -def test_global_overrides_environment_markers(PipenvInstance, pypi): +def test_global_overrides_environment_markers(PipenvInstance): """Empty (unconditional) dependency should take precedence. If a dependency is specified without environment markers, it should override dependencies with environment markers. In this example, APScheduler requires funcsigs only on Python 2, but since funcsigs is also specified as an unconditional dep, its markers should be empty. """ - with PipenvInstance(pypi=pypi) as p: + with PipenvInstance() as p: with open(p.pipfile_path, 'w') as f: contents = """ [packages] @@ -128,12 +127,12 @@ funcsigs = "*" assert p.lockfile['default']['funcsigs'].get('markers', '') == '' +@flaky @pytest.mark.lock @pytest.mark.complex -@flaky -@py3_only -@skip_py37 -def test_resolver_unique_markers(PipenvInstance, pypi): +@pytest.mark.py3_only +@pytest.mark.lte_py36 +def test_resolver_unique_markers(PipenvInstance): """vcrpy has a dependency on `yarl` which comes with a marker of 'python version in "3.4, 3.5, 3.6" - this marker duplicates itself: @@ -141,8 +140,8 @@ def test_resolver_unique_markers(PipenvInstance, pypi): This verifies that we clean that successfully. """ - with PipenvInstance(chdir=True, pypi=pypi) as p: - c = p.pipenv('install vcrpy==1.11.0') + with PipenvInstance(chdir=True) as p: + c = p.pipenv('install vcrpy==2.0.1') assert c.return_code == 0 c = p.pipenv('lock') assert c.return_code == 0 @@ -150,13 +149,13 @@ def test_resolver_unique_markers(PipenvInstance, pypi): yarl = p.lockfile['default']['yarl'] assert 'markers' in yarl # Two possible marker sets are ok here - assert yarl['markers'] in ["python_version in '3.4, 3.5, 3.6'", "python_version >= '3.4.1'"] + assert yarl['markers'] in ["python_version in '3.4, 3.5, 3.6'", "python_version >= '3.4'"] -@pytest.mark.project @flaky -def test_environment_variable_value_does_not_change_hash(PipenvInstance, pypi): - with PipenvInstance(chdir=True, pypi=pypi) as p: +@pytest.mark.project +def test_environment_variable_value_does_not_change_hash(PipenvInstance): + with PipenvInstance(chdir=True) as p: with temp_environ(): with open(p.pipfile_path, 'w') as f: f.write(""" diff --git a/tests/integration/test_install_twists.py b/tests/integration/test_install_twists.py index 2ad12691..44973df5 100644 --- a/tests/integration/test_install_twists.py +++ b/tests/integration/test_install_twists.py @@ -1,43 +1,46 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function import os import shutil -from pipenv.project import Project -from pipenv._compat import Path - -from pipenv.utils import mkdir_p, temp_environ +import sys import pytest from flaky import flaky +from pipenv._compat import Path +from pipenv.project import Project +from pipenv.utils import mkdir_p, temp_environ +from pipenv.vendor import delegator + @pytest.mark.extras @pytest.mark.install @pytest.mark.local -def test_local_extras_install(PipenvInstance, pypi): +def test_local_extras_install(PipenvInstance): """Ensure -e .[extras] installs. """ - with PipenvInstance(pypi=pypi, chdir=True) as p: + with PipenvInstance(chdir=True) as p: setup_py = os.path.join(p.path, "setup.py") with open(setup_py, "w") as fh: contents = """ from setuptools import setup, find_packages setup( -name='testpipenv', -version='0.1', -description='Pipenv Test Package', -author='Pipenv Test', -author_email='test@pipenv.package', -license='MIT', -packages=find_packages(), -install_requires=[], -extras_require={'dev': ['six']}, -zip_safe=False + name='testpipenv', + version='0.1', + description='Pipenv Test Package', + author='Pipenv Test', + author_email='test@pipenv.package', + license='MIT', + packages=find_packages(), + install_requires=[], + extras_require={'dev': ['six']}, + zip_safe=False ) """.strip() fh.write(contents) line = "-e .[dev]" - # pipfile = {"testpipenv": {"path": ".", "editable": True, "extras": ["dev"]}} - project = Project() + pipfile = {"testpipenv": {"path": ".", "editable": True, "extras": ["dev"]}} with open(os.path.join(p.path, 'Pipfile'), 'w') as fh: fh.write(""" [packages] @@ -51,9 +54,9 @@ testpipenv = {path = ".", editable = true, extras = ["dev"]} assert "testpipenv" in p.lockfile["default"] assert p.lockfile["default"]["testpipenv"]["extras"] == ["dev"] assert "six" in p.lockfile["default"] - c = p.pipenv("--rm") + c = p.pipenv("uninstall --all") assert c.return_code == 0 - project.write_toml({"packages": {}, "dev-packages": {}}) + print("Current directory: {0}".format(os.getcwd()), file=sys.stderr) c = p.pipenv("install {0}".format(line)) assert c.return_code == 0 assert "testpipenv" in p.pipfile["packages"] @@ -66,7 +69,7 @@ testpipenv = {path = ".", editable = true, extras = ["dev"]} @pytest.mark.local @pytest.mark.needs_internet @flaky -class TestDependencyLinks(object): +class TestDirectDependencies(object): """Ensure dependency_links are parsed and installed. This is needed for private repo dependencies. @@ -84,41 +87,39 @@ setup( version='0.1', packages=[], install_requires=[ - 'test-private-dependency' - ], - dependency_links=[ '{0}' - ] + ], ) """.strip().format(deplink) fh.write(contents) @staticmethod def helper_dependency_links_install_test(pipenv_instance, deplink): - TestDependencyLinks.helper_dependency_links_install_make_setup(pipenv_instance, deplink) + TestDirectDependencies.helper_dependency_links_install_make_setup(pipenv_instance, deplink) c = pipenv_instance.pipenv("install -v -e .") assert c.return_code == 0 assert "test-private-dependency" in pipenv_instance.lockfile["default"] assert "version" in pipenv_instance.lockfile["default"]["test-private-dependency"] assert "0.1" in pipenv_instance.lockfile["default"]["test-private-dependency"]["version"] - def test_https_dependency_links_install(self, PipenvInstance, pypi): + def test_https_dependency_links_install(self, PipenvInstance): """Ensure dependency_links are parsed and installed (needed for private repo dependencies). """ - with temp_environ(), PipenvInstance(pypi=pypi, chdir=True) as p: - os.environ['PIP_PROCESS_DEPENDENCY_LINKS'] = '1' - TestDependencyLinks.helper_dependency_links_install_test( + with temp_environ(), PipenvInstance(chdir=True) as p: + os.environ["PIP_NO_BUILD_ISOLATION"] = '1' + TestDirectDependencies.helper_dependency_links_install_test( p, - 'git+https://github.com/atzannes/test-private-dependency@v0.1#egg=test-private-dependency-v0.1' + 'test-private-dependency@ git+https://github.com/atzannes/test-private-dependency@v0.1' ) @pytest.mark.needs_github_ssh - def test_ssh_dependency_links_install(self, PipenvInstance, pypi): - with temp_environ(), PipenvInstance(pypi=pypi, chdir=True) as p: + def test_ssh_dependency_links_install(self, PipenvInstance): + with temp_environ(), PipenvInstance(chdir=True) as p: os.environ['PIP_PROCESS_DEPENDENCY_LINKS'] = '1' - TestDependencyLinks.helper_dependency_links_install_test( + os.environ["PIP_NO_BUILD_ISOLATION"] = '1' + TestDirectDependencies.helper_dependency_links_install_test( p, - 'git+ssh://git@github.com/atzannes/test-private-dependency@v0.1#egg=test-private-dependency-v0.1' + 'test-private-dependency@ git+ssh://git@github.com/atzannes/test-private-dependency@v0.1' ) @@ -139,11 +140,11 @@ def test_e_dot(PipenvInstance, pip_src_dir): @pytest.mark.install @flaky -def test_multiprocess_bug_and_install(PipenvInstance, pypi): +def test_multiprocess_bug_and_install(PipenvInstance): with temp_environ(): os.environ["PIPENV_MAX_SUBPROCESS"] = "2" - with PipenvInstance(pypi=pypi, chdir=True) as p: + with PipenvInstance(chdir=True) as p: with open(p.pipfile_path, "w") as f: contents = """ [packages] @@ -167,9 +168,9 @@ urllib3 = "*" @pytest.mark.sequential @pytest.mark.install @flaky -def test_sequential_mode(PipenvInstance, pypi): +def test_sequential_mode(PipenvInstance): - with PipenvInstance(pypi=pypi, chdir=True) as p: + with PipenvInstance(chdir=True) as p: with open(p.pipfile_path, "w") as f: contents = """ [packages] @@ -192,8 +193,8 @@ pytz = "*" @pytest.mark.install @pytest.mark.run -def test_normalize_name_install(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi) as p: +def test_normalize_name_install(PipenvInstance): + with PipenvInstance() as p: with open(p.pipfile_path, "w") as f: contents = """ # Pre comment @@ -221,18 +222,18 @@ Requests = "==2.14.0" # Inline comment assert "# Inline comment" in contents +@flaky @pytest.mark.files @pytest.mark.resolver @pytest.mark.eggs -@flaky -def test_local_package(PipenvInstance, pip_src_dir, pypi, testsroot): +def test_local_package(PipenvInstance, pip_src_dir, testsroot): """This test ensures that local packages (directories with a setup.py) installed in editable mode have their dependencies resolved as well""" file_name = "requests-2.19.1.tar.gz" package = "requests-2.19.1" # Not sure where travis/appveyor run tests from source_path = os.path.abspath(os.path.join(testsroot, "test_artifacts", file_name)) - with PipenvInstance(chdir=True, pypi=pypi) as p: + with PipenvInstance(chdir=True) as p: # This tests for a bug when installing a zipfile in the current dir copy_to = os.path.join(p.path, file_name) shutil.copy(source_path, copy_to) @@ -250,12 +251,12 @@ def test_local_package(PipenvInstance, pip_src_dir, pypi, testsroot): @pytest.mark.files @flaky -def test_local_zipfiles(PipenvInstance, pypi, testsroot): +def test_local_zipfiles(PipenvInstance, testsroot): file_name = "requests-2.19.1.tar.gz" # Not sure where travis/appveyor run tests from source_path = os.path.abspath(os.path.join(testsroot, "test_artifacts", file_name)) - with PipenvInstance(chdir=True, pypi=pypi) as p: + with PipenvInstance(chdir=True) as p: # This tests for a bug when installing a zipfile in the current dir shutil.copy(source_path, os.path.join(p.path, file_name)) @@ -267,19 +268,19 @@ def test_local_zipfiles(PipenvInstance, pypi, testsroot): assert "file" in dep or "path" in dep assert c.return_code == 0 - key = [k for k in p.lockfile["default"].keys()][0] - dep = p.lockfile["default"][key] + # This now gets resolved to its name correctly + dep = p.lockfile["default"]["requests"] assert "file" in dep or "path" in dep @pytest.mark.files @flaky -def test_relative_paths(PipenvInstance, pypi, testsroot): +def test_relative_paths(PipenvInstance, testsroot): file_name = "requests-2.19.1.tar.gz" source_path = os.path.abspath(os.path.join(testsroot, "test_artifacts", file_name)) - with PipenvInstance(pypi=pypi) as p: + with PipenvInstance() as p: artifact_dir = "artifacts" artifact_path = os.path.join(p.path, artifact_dir) mkdir_p(artifact_path) @@ -298,8 +299,8 @@ def test_relative_paths(PipenvInstance, pypi, testsroot): @pytest.mark.install @pytest.mark.local_file @flaky -def test_install_local_file_collision(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi) as p: +def test_install_local_file_collision(PipenvInstance): + with PipenvInstance() as p: target_package = "alembic" fake_file = os.path.join(p.path, target_package) with open(fake_file, "w") as f: @@ -338,7 +339,7 @@ six = {{path = "./artifacts/{}"}} @pytest.mark.files @pytest.mark.install @pytest.mark.run -def test_multiple_editable_packages_should_not_race(PipenvInstance, pypi, tmpdir, testsroot): +def test_multiple_editable_packages_should_not_race(PipenvInstance, testsroot): """Test for a race condition that can occur when installing multiple 'editable' packages at once, and which causes some of them to not be importable. @@ -347,40 +348,42 @@ def test_multiple_editable_packages_should_not_race(PipenvInstance, pypi, tmpdir So this test locally installs packages from tarballs that have already been committed in the local `pypi` dir to avoid using VCS packages. """ - pkgs = { - "requests-2.19.1": "requests/requests-2.19.1.tar.gz", - "Flask-0.12.2": "flask/Flask-0.12.2.tar.gz", - "six-1.11.0": "six/six-1.11.0.tar.gz", - "Jinja2-2.10": "jinja2/Jinja2-2.10.tar.gz", - } + pkgs = ["requests", "flask", "six", "jinja2"] + + pipfile_string = """ +[dev-packages] - pipfile_string=""" [packages] """ - # Unzip tarballs to known location, and update Pipfile template. - for pkg_name, file_name in pkgs.items(): - source_path = str(Path(testsroot, "pypi", file_name)) - unzip_path = str(Path(tmpdir.strpath, pkg_name)) - import tarfile + with PipenvInstance(chdir=True) as p: + for pkg_name in pkgs: + source_path = p._pipfile.get_fixture_path("git/{0}/".format(pkg_name)).as_posix() + c = delegator.run("git clone {0} ./{1}".format(source_path, pkg_name)) + assert c.return_code == 0 - with tarfile.open(source_path, "r:gz") as tgz: - tgz.extractall(path=tmpdir.strpath) + pipfile_string += '"{0}" = {{path = "./{0}", editable = true}}\n'.format(pkg_name) - pipfile_string += "'{0}' = {{path = '{1}', editable = true}}\n".format(pkg_name, unzip_path) - - with PipenvInstance(pypi=pypi, chdir=True) as p: with open(p.pipfile_path, 'w') as f: f.write(pipfile_string.strip()) c = p.pipenv('install') assert c.return_code == 0 - c = p.pipenv('run python -c "import requests"') + c = p.pipenv('run python -c "import requests, flask, six, jinja2"') + assert c.return_code == 0, c.err + + +@pytest.mark.outdated +@pytest.mark.py3_only +def test_outdated_should_compare_postreleases_without_failing(PipenvInstance): + with PipenvInstance(chdir=True) as p: + c = p.pipenv("install ibm-db-sa-py3==0.3.0") assert c.return_code == 0 - c = p.pipenv('run python -c "import flask"') - assert c.return_code == 0 - c = p.pipenv('run python -c "import six"') - assert c.return_code == 0 - c = p.pipenv('run python -c "import jinja2"') + c = p.pipenv("update --outdated") assert c.return_code == 0 + assert "Skipped Update" in c.err + p._pipfile.update("ibm-db-sa-py3", "*") + c = p.pipenv("update --outdated") + assert c.return_code != 0 + assert "out-of-date" in c.out diff --git a/tests/integration/test_install_uri.py b/tests/integration/test_install_uri.py index 337181fa..b71df965 100644 --- a/tests/integration/test_install_uri.py +++ b/tests/integration/test_install_uri.py @@ -1,16 +1,20 @@ +# -*- coding=utf-8 -*- +from __future__ import absolute_import, print_function import pytest -import os + from flaky import flaky + import delegator + from pipenv._compat import Path +@flaky @pytest.mark.vcs @pytest.mark.install @pytest.mark.needs_internet -@flaky -def test_basic_vcs_install(PipenvInstance, pip_src_dir, pypi): - with PipenvInstance(pypi=pypi, chdir=True) as p: +def test_basic_vcs_install(PipenvInstance): # ! This is failing + with PipenvInstance(chdir=True) as p: c = p.pipenv("install git+https://github.com/benjaminp/six.git@1.11.0#egg=six") assert c.return_code == 0 # edge case where normal package starts with VCS name shouldn't be flagged as vcs @@ -21,16 +25,17 @@ def test_basic_vcs_install(PipenvInstance, pip_src_dir, pypi): assert p.lockfile["default"]["six"] == { "git": "https://github.com/benjaminp/six.git", "ref": "15e31431af97e5e64b80af0a3f598d382bcdd49a", + "version": "==1.11.0" } assert "gitdb2" in p.lockfile["default"] +@flaky @pytest.mark.vcs @pytest.mark.install @pytest.mark.needs_internet -@flaky -def test_git_vcs_install(PipenvInstance, pip_src_dir, pypi): - with PipenvInstance(pypi=pypi, chdir=True) as p: +def test_git_vcs_install(PipenvInstance): + with PipenvInstance(chdir=True) as p: c = p.pipenv("install git+git://github.com/benjaminp/six.git@1.11.0#egg=six") assert c.return_code == 0 assert "six" in p.pipfile["packages"] @@ -38,16 +43,17 @@ def test_git_vcs_install(PipenvInstance, pip_src_dir, pypi): assert p.lockfile["default"]["six"] == { "git": "git://github.com/benjaminp/six.git", "ref": "15e31431af97e5e64b80af0a3f598d382bcdd49a", + "version": "==1.11.0" } +@flaky @pytest.mark.vcs @pytest.mark.install -@pytest.mark.needs_github_ssh @pytest.mark.needs_internet -@flaky -def test_ssh_vcs_install(PipenvInstance, pip_src_dir, pypi): - with PipenvInstance(pypi=pypi, chdir=True) as p: +@pytest.mark.needs_github_ssh +def test_ssh_vcs_install(PipenvInstance): + with PipenvInstance(chdir=True) as p: c = p.pipenv("install git+ssh://git@github.com/benjaminp/six.git@1.11.0#egg=six") assert c.return_code == 0 assert "six" in p.pipfile["packages"] @@ -55,29 +61,33 @@ def test_ssh_vcs_install(PipenvInstance, pip_src_dir, pypi): assert p.lockfile["default"]["six"] == { "git": "ssh://git@github.com/benjaminp/six.git", "ref": "15e31431af97e5e64b80af0a3f598d382bcdd49a", + "version": "==1.11.0" } -@pytest.mark.files -@pytest.mark.urls -@pytest.mark.needs_internet @flaky -def test_urls_work(PipenvInstance, pypi, pip_src_dir): - with PipenvInstance(pypi=pypi) as p: +@pytest.mark.urls +@pytest.mark.files +@pytest.mark.needs_internet +def test_urls_work(PipenvInstance): + with PipenvInstance(chdir=True) as p: + # the library this installs is "django-cms" + path = p._pipfile.get_url("django", "3.4.x.zip") c = p.pipenv( - "install https://github.com/divio/django-cms/archive/release/3.4.x.zip" + "install {0}".format(path) ) assert c.return_code == 0 dep = list(p.pipfile["packages"].values())[0] assert "file" in dep, p.pipfile - dep = list(p.lockfile["default"].values())[0] + # now that we handle resolution with requirementslib, this will resolve to a name + dep = p.lockfile["default"]["django-cms"] assert "file" in dep, p.lockfile -@pytest.mark.files @pytest.mark.urls +@pytest.mark.files def test_file_urls_work(PipenvInstance, pip_src_dir): with PipenvInstance(chdir=True) as p: whl = Path(__file__).parent.parent.joinpath( @@ -94,13 +104,13 @@ def test_file_urls_work(PipenvInstance, pip_src_dir): assert "file" in p.pipfile["packages"]["six"] -@pytest.mark.files @pytest.mark.urls +@pytest.mark.files @pytest.mark.needs_internet -def test_local_vcs_urls_work(PipenvInstance, pypi, tmpdir): +def test_local_vcs_urls_work(PipenvInstance, tmpdir): six_dir = tmpdir.join("six") six_path = Path(six_dir.strpath) - with PipenvInstance(pypi=pypi, chdir=True) as p: + with PipenvInstance(chdir=True) as p: c = delegator.run( "git clone https://github.com/benjaminp/six.git {0}".format(six_dir.strpath) ) @@ -115,11 +125,10 @@ def test_local_vcs_urls_work(PipenvInstance, pypi, tmpdir): @pytest.mark.vcs @pytest.mark.install @pytest.mark.needs_internet -@flaky -def test_editable_vcs_install(PipenvInstance, pip_src_dir, pypi): - with PipenvInstance(pypi=pypi) as p: +def test_editable_vcs_install(PipenvInstance_NoPyPI): + with PipenvInstance_NoPyPI(chdir=True) as p: c = p.pipenv( - "install -e git+https://github.com/requests/requests.git#egg=requests" + "install -e git+https://github.com/kennethreitz/requests.git#egg=requests" ) assert c.return_code == 0 assert "requests" in p.pipfile["packages"] @@ -132,15 +141,14 @@ def test_editable_vcs_install(PipenvInstance, pip_src_dir, pypi): assert "certifi" in p.lockfile["default"] -@pytest.mark.install @pytest.mark.vcs @pytest.mark.tablib +@pytest.mark.install @pytest.mark.needs_internet -@flaky -def test_install_editable_git_tag(PipenvInstance, pip_src_dir, pypi): +def test_install_editable_git_tag(PipenvInstance_NoPyPI): # This uses the real PyPI since we need Internet to access the Git # dependency anyway. - with PipenvInstance(pypi=pypi) as p: + with PipenvInstance_NoPyPI(chdir=True) as p: c = p.pipenv( "install -e git+https://github.com/benjaminp/six.git@1.11.0#egg=six" ) @@ -155,11 +163,11 @@ def test_install_editable_git_tag(PipenvInstance, pip_src_dir, pypi): assert "ref" in p.lockfile["default"]["six"] -@pytest.mark.install @pytest.mark.index +@pytest.mark.install @pytest.mark.needs_internet -def test_install_named_index_alias(PipenvInstance): - with PipenvInstance() as p: +def test_install_named_index_alias(PipenvInstance_NoPyPI): + with PipenvInstance_NoPyPI() as p: with open(p.pipfile_path, "w") as f: contents = """ [[source]] @@ -185,36 +193,24 @@ six = "*" @pytest.mark.vcs @pytest.mark.install @pytest.mark.needs_internet -def test_install_local_vcs_not_in_lockfile(PipenvInstance, pip_src_dir): +def test_install_local_vcs_not_in_lockfile(PipenvInstance): with PipenvInstance(chdir=True) as p: - six_path = os.path.join(p.path, "six") - c = delegator.run( - "git clone https://github.com/benjaminp/six.git {0}".format(six_path) - ) + # six_path = os.path.join(p.path, "six") + six_path = p._pipfile.get_fixture_path("git/six/").as_posix() + c = delegator.run("git clone {0} ./six".format(six_path)) assert c.return_code == 0 - c = p.pipenv("install -e ./six") + c = p.pipenv("install -e ./six".format(six_path)) assert c.return_code == 0 six_key = list(p.pipfile["packages"].keys())[0] - c = p.pipenv( - "install -e git+https://github.com/requests/requests.git#egg=requests" - ) - assert c.return_code == 0 - c = p.pipenv("lock") - assert c.return_code == 0 - assert "requests" in p.pipfile["packages"] - assert "requests" in p.lockfile["default"] - # This is the hash of ./six - assert six_key in p.pipfile["packages"] - assert six_key in p.lockfile["default"] - # The hash isn't a hash anymore, its actually the name of the package (we now resolve this) - assert "six" in p.pipfile["packages"] + # we don't need the rest of the test anymore, this just works on its own + assert six_key == "six" @pytest.mark.vcs @pytest.mark.install @pytest.mark.needs_internet -def test_get_vcs_refs(PipenvInstance, pip_src_dir): - with PipenvInstance(chdir=True) as p: +def test_get_vcs_refs(PipenvInstance_NoPyPI): + with PipenvInstance_NoPyPI(chdir=True) as p: c = p.pipenv( "install -e git+https://github.com/benjaminp/six.git@1.9.0#egg=six" ) @@ -226,8 +222,8 @@ def test_get_vcs_refs(PipenvInstance, pip_src_dir): == "5efb522b0647f7467248273ec1b893d06b984a59" ) pipfile = Path(p.pipfile_path) - new_content = pipfile.read_bytes().replace(b"1.9.0", b"1.11.0") - pipfile.write_bytes(new_content) + new_content = pipfile.read_text().replace(u"1.9.0", u"1.11.0") + pipfile.write_text(new_content) c = p.pipenv("lock") assert c.return_code == 0 assert ( @@ -241,12 +237,15 @@ def test_get_vcs_refs(PipenvInstance, pip_src_dir): @pytest.mark.vcs @pytest.mark.install @pytest.mark.needs_internet -def test_vcs_entry_supersedes_non_vcs(PipenvInstance, pip_src_dir): +@pytest.mark.skip_py27_win +def test_vcs_entry_supersedes_non_vcs(PipenvInstance): """See issue #2181 -- non-editable VCS dep was specified, but not showing up in the lockfile -- due to not running pip install before locking and not locking the resolution graph of non-editable vcs dependencies. """ with PipenvInstance(chdir=True) as p: + # pyinstaller_path = p._pipfile.get_fixture_path("git/pyinstaller") + pyinstaller_uri = "https://github.com/pyinstaller/pyinstaller.git" with open(p.pipfile_path, "w") as f: f.write( """ @@ -257,26 +256,27 @@ name = "pypi" [packages] PyUpdater = "*" -PyInstaller = {ref = "develop", git = "https://github.com/pyinstaller/pyinstaller.git"} - """.strip() +PyInstaller = {{ref = "develop", git = "{0}"}} + """.format(pyinstaller_uri).strip() ) - p.pipenv("install") + c = p.pipenv("install") + assert c.return_code == 0 installed_packages = ["PyUpdater", "PyInstaller"] assert all([k in p.pipfile["packages"] for k in installed_packages]) assert all([k.lower() in p.lockfile["default"] for k in installed_packages]) - assert all([k in p.lockfile["default"]["pyinstaller"] for k in ["ref", "git"]]) + assert all([k in p.lockfile["default"]["pyinstaller"] for k in ["ref", "git"]]), str(p.lockfile["default"]) assert p.lockfile["default"]["pyinstaller"].get("ref") is not None assert ( p.lockfile["default"]["pyinstaller"]["git"] - == "https://github.com/pyinstaller/pyinstaller.git" + == pyinstaller_uri ) @pytest.mark.vcs @pytest.mark.install @pytest.mark.needs_internet -def test_vcs_can_use_markers(PipenvInstance, pip_src_dir, pypi): - with PipenvInstance(chdir=True, pypi=pypi) as p: +def test_vcs_can_use_markers(PipenvInstance): + with PipenvInstance(chdir=True) as p: path = p._pipfile.get_fixture_path("git/six/.git") p._pipfile.install("six", {"git": "{0}".format(path.as_uri()), "markers": "sys_platform == 'linux'"}) assert "six" in p.pipfile["packages"] diff --git a/tests/integration/test_lock.py b/tests/integration/test_lock.py index f2f0ac76..4c227395 100644 --- a/tests/integration/test_lock.py +++ b/tests/integration/test_lock.py @@ -1,14 +1,20 @@ -import pytest -import os +# -*- coding: utf-8 -*- -from pipenv.utils import temp_environ +import json +import os +import sys + +import pytest from flaky import flaky +from vistir.compat import Path +from vistir.misc import to_text +from pipenv.utils import temp_environ @pytest.mark.lock @pytest.mark.requirements -def test_lock_handle_eggs(PipenvInstance, pypi): +def test_lock_handle_eggs(PipenvInstance): """Ensure locking works with packages provoding egg formats. """ with PipenvInstance() as p: @@ -25,9 +31,9 @@ RandomWords = "*" @pytest.mark.lock @pytest.mark.requirements -def test_lock_requirements_file(PipenvInstance, pypi): +def test_lock_requirements_file(PipenvInstance): - with PipenvInstance(pypi=pypi) as p: + with PipenvInstance() as p: with open(p.pipfile_path, 'w') as f: contents = """ [packages] @@ -54,9 +60,10 @@ flask = "==0.12.2" @pytest.mark.lock -def test_lock_keep_outdated(PipenvInstance, pypi): +@pytest.mark.keep_outdated +def test_lock_keep_outdated(PipenvInstance): - with PipenvInstance(pypi=pypi) as p: + with PipenvInstance() as p: with open(p.pipfile_path, 'w') as f: contents = """ [packages] @@ -90,6 +97,70 @@ PyTest = "*" assert lock['default']['pytest']['version'] == "==3.1.0" +@pytest.mark.lock +@pytest.mark.keep_outdated +def test_keep_outdated_doesnt_remove_lockfile_entries(PipenvInstance): + with PipenvInstance(chdir=True) as p: + p._pipfile.add("requests", "==2.18.4") + p._pipfile.add("colorama", {"version": "*", "markers": "os_name=='FakeOS'"}) + p.pipenv("install") + p._pipfile.add("six", "*") + p.pipenv("lock --keep-outdated") + assert "colorama" in p.lockfile["default"] + assert p.lockfile["default"]["colorama"]["markers"] == "os_name == 'FakeOS'" + + +@pytest.mark.lock +@pytest.mark.keep_outdated +def test_keep_outdated_doesnt_upgrade_pipfile_pins(PipenvInstance): + with PipenvInstance(chdir=True) as p: + p._pipfile.add("urllib3", "==1.21.1") + c = p.pipenv("install") + assert c.ok + p._pipfile.add("requests", "==2.18.4") + c = p.pipenv("lock --keep-outdated") + assert c.ok + assert "requests" in p.lockfile["default"] + assert "urllib3" in p.lockfile["default"] + assert p.lockfile["default"]["requests"]["version"] == "==2.18.4" + assert p.lockfile["default"]["urllib3"]["version"] == "==1.21.1" + + +def test_keep_outdated_keeps_markers_not_removed(PipenvInstance): + with PipenvInstance(chdir=True) as p: + c = p.pipenv("install six click") + assert c.ok + lockfile = Path(p.lockfile_path) + lockfile_content = lockfile.read_text() + lockfile_json = json.loads(lockfile_content) + assert "six" in lockfile_json["default"] + lockfile_json["default"]["six"]["markers"] = "python_version >= '2.7'" + lockfile.write_text(to_text(json.dumps(lockfile_json))) + c = p.pipenv("lock --keep-outdated") + assert c.ok + assert p.lockfile["default"]["six"].get("markers", "") == "python_version >= '2.7'" + + +@pytest.mark.lock +@pytest.mark.keep_outdated +def test_keep_outdated_doesnt_update_satisfied_constraints(PipenvInstance): + with PipenvInstance(chdir=True) as p: + p._pipfile.add("requests", "==2.18.4") + c = p.pipenv("install") + assert c.ok + p._pipfile.add("requests", "*") + assert p.pipfile["packages"]["requests"] == "*" + c = p.pipenv("lock --keep-outdated") + assert c.ok + assert "requests" in p.lockfile["default"] + assert "urllib3" in p.lockfile["default"] + # ensure this didn't update requests + assert p.lockfile["default"]["requests"]["version"] == "==2.18.4" + c = p.pipenv("lock") + assert c.ok + assert p.lockfile["default"]["requests"]["version"] != "==2.18.4" + + @pytest.mark.lock @pytest.mark.complex @pytest.mark.needs_internet @@ -103,7 +174,7 @@ def test_complex_lock_with_vcs_deps(PipenvInstance, pip_src_dir): click = "==6.7" [dev-packages] -requests = {git = "https://github.com/requests/requests.git"} +requests = {git = "https://github.com/kennethreitz/requests.git"} """.strip() f.write(contents) @@ -127,9 +198,9 @@ requests = {git = "https://github.com/requests/requests.git"} @pytest.mark.lock @pytest.mark.requirements -def test_lock_with_prereleases(PipenvInstance, pypi): +def test_lock_with_prereleases(PipenvInstance): - with PipenvInstance(pypi=pypi) as p: + with PipenvInstance() as p: with open(p.pipfile_path, 'w') as f: contents = """ [packages] @@ -146,13 +217,13 @@ allow_prereleases = true @pytest.mark.lock -@pytest.mark.complex @pytest.mark.maya +@pytest.mark.complex @pytest.mark.needs_internet @flaky -def test_complex_deps_lock_and_install_properly(PipenvInstance, pip_src_dir, pypi): +def test_complex_deps_lock_and_install_properly(PipenvInstance, pip_src_dir): # This uses the real PyPI because Maya has too many dependencies... - with PipenvInstance(chdir=True, pypi=pypi) as p: + with PipenvInstance(chdir=True) as p: with open(p.pipfile_path, 'w') as f: contents = """ [packages] @@ -167,10 +238,10 @@ maya = "*" assert c.return_code == 0 -@pytest.mark.extras @pytest.mark.lock -def test_lock_extras_without_install(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi) as p: +@pytest.mark.extras +def test_lock_extras_without_install(PipenvInstance): + with PipenvInstance() as p: with open(p.pipfile_path, 'w') as f: contents = """ [packages] @@ -189,16 +260,16 @@ requests = {version = "*", extras = ["socks"]} assert "extra == 'socks'" not in c.out.strip() -@pytest.mark.extras @pytest.mark.lock +@pytest.mark.extras @pytest.mark.complex -@pytest.mark.skip(reason='Needs numpy to be mocked') @pytest.mark.needs_internet -def test_complex_lock_deep_extras(PipenvInstance, pypi): +@pytest.mark.skip(reason='Needs numpy to be mocked') +def test_complex_lock_deep_extras(PipenvInstance): # records[pandas] requires tablib[pandas] which requires pandas. # This uses the real PyPI; Pandas has too many requirements to mock. - with PipenvInstance(pypi=pypi) as p: + with PipenvInstance() as p: with open(p.pipfile_path, 'w') as f: contents = """ [packages] @@ -214,12 +285,12 @@ records = {extras = ["pandas"], version = "==0.5.2"} assert 'pandas' in p.lockfile['default'] -@pytest.mark.skip_lock @pytest.mark.index -@pytest.mark.needs_internet @pytest.mark.install # private indexes need to be uncached for resolution -def test_private_index_skip_lock(PipenvInstance): - with PipenvInstance() as p: +@pytest.mark.skip_lock +@pytest.mark.needs_internet +def test_private_index_skip_lock(PipenvInstance_NoPyPI): + with PipenvInstance_NoPyPI() as p: with open(p.pipfile_path, 'w') as f: contents = """ [[source]] @@ -241,14 +312,14 @@ requests = "*" assert c.return_code == 0 -@pytest.mark.requirements @pytest.mark.lock @pytest.mark.index @pytest.mark.install # private indexes need to be uncached for resolution +@pytest.mark.requirements @pytest.mark.needs_internet -def test_private_index_lock_requirements(PipenvInstance): +def test_private_index_lock_requirements(PipenvInstance_NoPyPI): # Don't use the local fake pypi - with PipenvInstance() as p: + with PipenvInstance_NoPyPI() as p: with open(p.pipfile_path, 'w') as f: contents = """ [[source]] @@ -274,14 +345,14 @@ requests = "*" assert '--extra-index-url https://test.pypi.org/simple' in c.out.strip() -@pytest.mark.requirements @pytest.mark.lock @pytest.mark.index @pytest.mark.install # private indexes need to be uncached for resolution +@pytest.mark.requirements @pytest.mark.needs_internet -def test_private_index_mirror_lock_requirements(PipenvInstance): +def test_private_index_mirror_lock_requirements(PipenvInstance_NoPyPI): # Don't use the local fake pypi - with temp_environ(), PipenvInstance(chdir=True) as p: + with temp_environ(), PipenvInstance_NoPyPI(chdir=True) as p: # Using pypi.python.org as pipenv-test-public-package is not # included in the local pypi mirror mirror_url = os.environ.pop('PIPENV_TEST_INDEX', "https://pypi.kennethreitz.org/simple") @@ -300,7 +371,7 @@ name = "testpypi" [packages] six = {version = "*", index = "testpypi"} -requests = "*" +fake-package = "*" """.strip() f.write(contents) c = p.pipenv('install --pypi-mirror {0}'.format(mirror_url)) @@ -314,11 +385,11 @@ requests = "*" assert '--extra-index-url {}'.format(mirror_url) not in c.out.strip() -@pytest.mark.install @pytest.mark.index -def test_lock_updated_source(PipenvInstance, pypi): +@pytest.mark.install +def test_lock_updated_source(PipenvInstance): - with PipenvInstance(pypi=pypi) as p: + with PipenvInstance() as p: with open(p.pipfile_path, 'w') as f: contents = """ [[source]] @@ -326,7 +397,7 @@ url = "{url}/${{MY_ENV_VAR}}" [packages] requests = "==2.14.0" - """.strip().format(url=pypi.url) + """.strip().format(url=p.pypi) f.write(contents) with temp_environ(): @@ -342,7 +413,7 @@ url = "{url}/simple" [packages] requests = "==2.14.0" - """.strip().format(url=pypi.url) + """.strip().format(url=p.pypi) f.write(contents) c = p.pipenv('lock') @@ -350,15 +421,15 @@ requests = "==2.14.0" assert 'requests' in p.lockfile['default'] -@pytest.mark.lock @pytest.mark.vcs +@pytest.mark.lock @pytest.mark.needs_internet -def test_lock_editable_vcs_without_install(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi, chdir=True) as p: +def test_lock_editable_vcs_without_install(PipenvInstance): + with PipenvInstance(chdir=True) as p: with open(p.pipfile_path, 'w') as f: f.write(""" [packages] -requests = {git = "https://github.com/requests/requests.git", ref = "master", editable = true} +requests = {git = "https://github.com/kennethreitz/requests.git", ref = "master", editable = true} """.strip()) c = p.pipenv('lock') assert c.return_code == 0 @@ -369,52 +440,52 @@ requests = {git = "https://github.com/requests/requests.git", ref = "master", ed assert c.return_code == 0 -@pytest.mark.lock @pytest.mark.vcs +@pytest.mark.lock @pytest.mark.needs_internet -def test_lock_editable_vcs_with_ref_in_git(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi, chdir=True) as p: +def test_lock_editable_vcs_with_ref_in_git(PipenvInstance): + with PipenvInstance(chdir=True) as p: with open(p.pipfile_path, 'w') as f: f.write(""" [packages] -requests = {git = "https://github.com/requests/requests.git@883caaf", editable = true} +requests = {git = "https://github.com/kennethreitz/requests.git@883caaf", editable = true} """.strip()) c = p.pipenv('lock') assert c.return_code == 0 - assert p.lockfile['default']['requests']['git'] == 'https://github.com/requests/requests.git' + assert p.lockfile['default']['requests']['git'] == 'https://github.com/kennethreitz/requests.git' assert p.lockfile['default']['requests']['ref'] == '883caaf145fbe93bd0d208a6b864de9146087312' c = p.pipenv('install') assert c.return_code == 0 -@pytest.mark.lock @pytest.mark.vcs +@pytest.mark.lock @pytest.mark.needs_internet -def test_lock_editable_vcs_with_ref(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi, chdir=True) as p: +def test_lock_editable_vcs_with_ref(PipenvInstance): + with PipenvInstance(chdir=True) as p: with open(p.pipfile_path, 'w') as f: f.write(""" [packages] -requests = {git = "https://github.com/requests/requests.git", ref = "883caaf", editable = true} +requests = {git = "https://github.com/kennethreitz/requests.git", ref = "883caaf", editable = true} """.strip()) c = p.pipenv('lock') assert c.return_code == 0 - assert p.lockfile['default']['requests']['git'] == 'https://github.com/requests/requests.git' + assert p.lockfile['default']['requests']['git'] == 'https://github.com/kennethreitz/requests.git' assert p.lockfile['default']['requests']['ref'] == '883caaf145fbe93bd0d208a6b864de9146087312' c = p.pipenv('install') assert c.return_code == 0 +@pytest.mark.vcs +@pytest.mark.lock @pytest.mark.extras -@pytest.mark.lock -@pytest.mark.vcs @pytest.mark.needs_internet -def test_lock_editable_vcs_with_extras_without_install(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi, chdir=True) as p: +def test_lock_editable_vcs_with_extras_without_install(PipenvInstance): + with PipenvInstance(chdir=True) as p: with open(p.pipfile_path, 'w') as f: f.write(""" [packages] -requests = {git = "https://github.com/requests/requests.git", editable = true, extras = ["socks"]} +requests = {git = "https://github.com/kennethreitz/requests.git", editable = true, extras = ["socks"]} """.strip()) c = p.pipenv('lock') assert c.return_code == 0 @@ -426,15 +497,15 @@ requests = {git = "https://github.com/requests/requests.git", editable = true, e assert c.return_code == 0 -@pytest.mark.lock @pytest.mark.vcs +@pytest.mark.lock @pytest.mark.needs_internet -def test_lock_editable_vcs_with_markers_without_install(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi, chdir=True) as p: +def test_lock_editable_vcs_with_markers_without_install(PipenvInstance): + with PipenvInstance(chdir=True) as p: with open(p.pipfile_path, 'w') as f: f.write(""" [packages] -requests = {git = "https://github.com/requests/requests.git", ref = "master", editable = true, markers = "python_version >= '2.6'"} +requests = {git = "https://github.com/kennethreitz/requests.git", ref = "master", editable = true, markers = "python_version >= '2.6'"} """.strip()) c = p.pipenv('lock') assert c.return_code == 0 @@ -447,8 +518,8 @@ requests = {git = "https://github.com/requests/requests.git", ref = "master", ed @pytest.mark.lock @pytest.mark.skip(reason="This doesn't work for some reason.") -def test_lock_respecting_python_version(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi, chdir=True) as p: +def test_lock_respecting_python_version(PipenvInstance): + with PipenvInstance(chdir=True) as p: with open(p.pipfile_path, 'w') as f: f.write(""" [packages] @@ -490,8 +561,9 @@ def test_lockfile_with_empty_dict(PipenvInstance): @pytest.mark.lock @pytest.mark.install -def test_lock_with_incomplete_source(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi, chdir=True) as p: +@pytest.mark.skip_lock +def test_lock_with_incomplete_source(PipenvInstance): + with PipenvInstance(chdir=True) as p: with open(p.pipfile_path, 'w') as f: f.write(""" [[source]] @@ -500,6 +572,91 @@ url = "https://test.pypi.org/simple" [packages] requests = "*" """) + c = p.pipenv('install --skip-lock') + assert c.return_code == 0 c = p.pipenv('install') assert c.return_code == 0 assert p.lockfile['_meta']['sources'] + + +@pytest.mark.lock +@pytest.mark.install +def test_lock_no_warnings(PipenvInstance): + with PipenvInstance(chdir=True) as p: + os.environ["PYTHONWARNINGS"] = str("once") + c = p.pipenv("install six") + assert c.return_code == 0 + c = p.pipenv('run python -c "import warnings; warnings.warn(\\"This is a warning\\", DeprecationWarning); print(\\"hello\\")"') + assert c.return_code == 0 + assert "Warning" in c.err + assert "Warning" not in c.out + assert "hello" in c.out + + +@pytest.mark.lock +@pytest.mark.install +@pytest.mark.skipif(sys.version_info >= (3, 5), reason="scandir doesn't get installed on python 3.5+") +def test_lock_missing_cache_entries_gets_all_hashes(PipenvInstance, tmpdir): + """ + Test locking pathlib2 on python2.7 which needs `scandir`, but fails to resolve when + using a fresh dependency cache. + """ + + with temp_environ(): + os.environ["PIPENV_CACHE_DIR"] = str(tmpdir.strpath) + with PipenvInstance(chdir=True) as p: + p._pipfile.add("pathlib2", "*") + assert "pathlib2" in p.pipfile["packages"] + c = p.pipenv("install") + assert c.return_code == 0, (c.err, ("\n".join(["{0}: {1}\n".format(k, v) for k, v in os.environ.items()]))) + c = p.pipenv("lock --clear") + assert c.return_code == 0, c.err + assert "pathlib2" in p.lockfile["default"] + assert "scandir" in p.lockfile["default"] + assert isinstance(p.lockfile["default"]["scandir"]["hashes"], list) + assert len(p.lockfile["default"]["scandir"]["hashes"]) > 1 + + +@pytest.mark.vcs +@pytest.mark.lock +def test_vcs_lock_respects_top_level_pins(PipenvInstance): + """Test that locking VCS dependencies respects top level packages pinned in Pipfiles""" + + with PipenvInstance(chdir=True) as p: + requests_uri = p._pipfile.get_fixture_path("git/requests").as_uri() + p._pipfile.add("requests", { + "editable": True, "git": "{0}".format(requests_uri), + "ref": "v2.18.4" + }) + p._pipfile.add("urllib3", "==1.21.1") + c = p.pipenv("install") + assert c.return_code == 0 + assert "requests" in p.lockfile["default"] + assert "git" in p.lockfile["default"]["requests"] + assert "urllib3" in p.lockfile["default"] + assert p.lockfile["default"]["urllib3"]["version"] == "==1.21.1" + + +@pytest.mark.lock +def test_lock_after_update_source_name(PipenvInstance): + with PipenvInstance(chdir=True) as p: + contents = """ +[[source]] +url = "https://test.pypi.org/simple" +verify_ssl = true +name = "test" + +[packages] +six = "*" + """.strip() + with open(p.pipfile_path, 'w') as f: + f.write(contents) + c = p.pipenv("lock") + assert c.return_code == 0 + assert p.lockfile["default"]["six"]["index"] == "test" + with open(p.pipfile_path, 'w') as f: + f.write(contents.replace('name = "test"', 'name = "custom"')) + c = p.pipenv("lock --clear") + assert c.return_code == 0 + assert "index" in p.lockfile["default"]["six"] + assert p.lockfile["default"]["six"]["index"] == "custom", Path(p.lockfile_path).read_text() # p.lockfile["default"]["six"] diff --git a/tests/integration/test_pipenv.py b/tests/integration/test_pipenv.py index 7824980f..12ae2348 100644 --- a/tests/integration/test_pipenv.py +++ b/tests/integration/test_pipenv.py @@ -1,18 +1,17 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function """Misc. tests that don't fit anywhere. XXX: Try our best to reduce tests in this file. """ import os -from tempfile import mkdtemp -import mock import pytest -from pipenv.utils import temp_environ from pipenv.project import Project +from pipenv.utils import temp_environ from pipenv.vendor import delegator -from pipenv._compat import Path @pytest.mark.code @@ -29,9 +28,9 @@ def test_code_import_manual(PipenvInstance): @pytest.mark.lock @pytest.mark.deploy @pytest.mark.cli -def test_deploy_works(PipenvInstance, pypi): +def test_deploy_works(PipenvInstance): - with PipenvInstance(pypi=pypi, chdir=True) as p: + with PipenvInstance(chdir=True) as p: with open(p.pipfile_path, 'w') as f: contents = """ [packages] @@ -62,29 +61,30 @@ requests = "==2.14.0" @pytest.mark.update @pytest.mark.lock -def test_update_locks(PipenvInstance, pypi): - - with PipenvInstance(pypi=pypi) as p: - c = p.pipenv('install requests==2.14.0') +def test_update_locks(PipenvInstance): + with PipenvInstance() as p: + c = p.pipenv('install jdcal==1.3') assert c.return_code == 0 + assert p.lockfile['default']['jdcal']['version'] == '==1.3' with open(p.pipfile_path, 'r') as fh: pipfile_contents = fh.read() - pipfile_contents = pipfile_contents.replace('==2.14.0', '*') + assert '==1.3' in pipfile_contents + pipfile_contents = pipfile_contents.replace('==1.3', '*') with open(p.pipfile_path, 'w') as fh: fh.write(pipfile_contents) - c = p.pipenv('update requests') + c = p.pipenv('update jdcal') assert c.return_code == 0 - assert p.lockfile['default']['requests']['version'] == '==2.19.1' + assert p.lockfile['default']['jdcal']['version'] == '==1.4' c = p.pipenv('run pip freeze') assert c.return_code == 0 lines = c.out.splitlines() - assert 'requests==2.19.1' in [l.strip() for l in lines] + assert 'jdcal==1.4' in [l.strip() for l in lines] @pytest.mark.project @pytest.mark.proper_names -def test_proper_names_unamanged_virtualenv(PipenvInstance, pypi): - with PipenvInstance(chdir=True, pypi=pypi): +def test_proper_names_unamanged_virtualenv(PipenvInstance): + with PipenvInstance(chdir=True): c = delegator.run('python -m virtualenv .venv') assert c.return_code == 0 project = Project() @@ -92,17 +92,16 @@ def test_proper_names_unamanged_virtualenv(PipenvInstance, pypi): @pytest.mark.cli -def test_directory_with_leading_dash(PipenvInstance): - def mocked_mkdtemp(suffix, prefix, dir): - if suffix == '-project': - prefix = '-dir-with-leading-dash' - return mkdtemp(suffix, prefix, dir) - - with mock.patch('pipenv._compat.mkdtemp', side_effect=mocked_mkdtemp): - with temp_environ(), PipenvInstance(chdir=True) as p: - del os.environ['PIPENV_VENV_IN_PROJECT'] - p.pipenv('--python python') - venv_path = p.pipenv('--venv').out.strip() +def test_directory_with_leading_dash(raw_venv, PipenvInstance): + with temp_environ(): + with PipenvInstance(chdir=True, venv_in_project=False, name="-project-with-dash") as p: + if "PIPENV_VENV_IN_PROJECT" in os.environ: + del os.environ['PIPENV_VENV_IN_PROJECT'] + c = p.pipenv('run pip freeze') + assert c.return_code == 0 + c = p.pipenv('--venv') + assert c.return_code == 0 + venv_path = c.out.strip() assert os.path.isdir(venv_path) # Manually clean up environment, since PipenvInstance assumes that # the virutalenv is in the project directory. diff --git a/tests/integration/test_project.py b/tests/integration/test_project.py index 8bb76da1..f7b0e460 100644 --- a/tests/integration/test_project.py +++ b/tests/integration/test_project.py @@ -1,11 +1,17 @@ # -*- coding=utf-8 -*- +from __future__ import absolute_import, print_function import io -import pytest import os import tarfile + +import pytest + +from pipenv.patched import pipfile from pipenv.project import Project from pipenv.utils import temp_environ -from pipenv.patched import pipfile +from pipenv.vendor.vistir.path import is_in_path +from pipenv.vendor.delegator import run as delegator_run +import pipenv.environments @pytest.mark.project @@ -33,8 +39,8 @@ pytz = "*" @pytest.mark.project @pytest.mark.sources @pytest.mark.parametrize('lock_first', [True, False]) -def test_get_source(PipenvInstance, pypi, lock_first): - with PipenvInstance(pypi=pypi, chdir=True) as p: +def test_get_source(PipenvInstance, lock_first): + with PipenvInstance(chdir=True) as p: with open(p.pipfile_path, 'w') as f: contents = """ [[source]] @@ -80,8 +86,8 @@ six = {{version = "*", index = "pypi"}} @pytest.mark.install @pytest.mark.project @pytest.mark.parametrize('newlines', [u'\n', u'\r\n']) -def test_maintain_file_line_endings(PipenvInstance, pypi, newlines): - with PipenvInstance(pypi=pypi, chdir=True) as p: +def test_maintain_file_line_endings(PipenvInstance, newlines): + with PipenvInstance(chdir=True) as p: # Initial pipfile + lockfile generation c = p.pipenv('install pytz') assert c.return_code == 0 @@ -116,8 +122,8 @@ def test_maintain_file_line_endings(PipenvInstance, pypi, newlines): @pytest.mark.project @pytest.mark.sources -def test_many_indexes(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi, chdir=True) as p: +def test_many_indexes(PipenvInstance): + with PipenvInstance(chdir=True) as p: with open(p.pipfile_path, 'w') as f: contents = """ [[source]] @@ -148,52 +154,75 @@ six = {{version = "*", index = "pypi"}} @pytest.mark.install @pytest.mark.project -def test_rewrite_outline_table(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi, chdir=True) as p: - with open(p.pipfile_path, 'w') as f: - contents = """ -[packages.requests] -version = "*" - """.strip() - f.write(contents) - c = p.pipenv('install click') - assert c.return_code == 0 - with open(p.pipfile_path) as f: - contents = f.read() - assert "[packages.requests]" not in contents - assert 'requests = {version = "*"}' in contents - - -@pytest.mark.install -@pytest.mark.project -def test_include_editable_packages(PipenvInstance, pypi, testsroot, pathlib_tmpdir): - file_name = "requests-2.19.1.tar.gz" - package = pathlib_tmpdir.joinpath("requests-2.19.1") - source_path = os.path.abspath(os.path.join(testsroot, "test_artifacts", file_name)) - with PipenvInstance(chdir=True, pypi=pypi) as p: +def test_include_editable_packages(PipenvInstance, testsroot, pathlib_tmpdir): + file_name = "tablib-0.12.1.tar.gz" + package = pathlib_tmpdir.joinpath("tablib-0.12.1") + source_path = os.path.abspath(os.path.join(testsroot, "pypi", "tablib", file_name)) + with PipenvInstance(chdir=True) as p: with tarfile.open(source_path, "r:gz") as tarinfo: tarinfo.extractall(path=str(pathlib_tmpdir)) - c = p.pipenv('install -e {}'.format(package)) + c = p.pipenv('install -e {0}'.format(package.as_posix())) assert c.return_code == 0 project = Project() - assert "requests" in [ + assert "tablib" in [ package.project_name for package in project.environment.get_installed_packages() ] @pytest.mark.project -def test_run_in_virtualenv(PipenvInstance, pypi, virtualenv): - with PipenvInstance(chdir=True, pypi=pypi) as p: - os.environ.pop("PIPENV_IGNORE_VIRTUALENVS", None) +@pytest.mark.virtualenv +def test_run_in_virtualenv_with_global_context(PipenvInstance, virtualenv): + with PipenvInstance(chdir=True, venv_root=virtualenv.as_posix(), ignore_virtualenvs=False, venv_in_project=False) as p: + c = delegator_run( + "pipenv run pip freeze", cwd=os.path.abspath(p.path), + env=os.environ.copy() + ) + assert c.return_code == 0, (c.out, c.err) + assert 'Creating a virtualenv' not in c.err, c.err + project = Project() + assert project.virtualenv_location == virtualenv.as_posix(), ( + project.virtualenv_location, virtualenv.as_posix() + ) + c = delegator_run( + "pipenv run pip install click", cwd=os.path.abspath(p.path), + env=os.environ.copy() + ) + assert c.return_code == 0, (c.out, c.err) + assert "Courtesy Notice" in c.err, (c.out, c.err) + c = delegator_run( + "pipenv install six", cwd=os.path.abspath(p.path), env=os.environ.copy() + ) + assert c.return_code == 0, (c.out, c.err) + c = delegator_run( + 'pipenv run python -c "import click;print(click.__file__)"', + cwd=os.path.abspath(p.path), env=os.environ.copy() + ) + assert c.return_code == 0, (c.out, c.err) + assert is_in_path(c.out.strip(), str(virtualenv)), (c.out.strip(), str(virtualenv)) + c = delegator_run( + "pipenv clean --dry-run", cwd=os.path.abspath(p.path), + env=os.environ.copy() + ) + assert c.return_code == 0, (c.out, c.err) + assert "click" in c.out, c.out + + +@pytest.mark.project +@pytest.mark.virtualenv +def test_run_in_virtualenv(PipenvInstance): + with PipenvInstance(chdir=True) as p: + c = p.pipenv('run pip freeze') + assert c.return_code == 0 + assert 'Creating a virtualenv' in c.err project = Project() - assert project.virtualenv_location == str(virtualenv) c = p.pipenv("run pip install click") assert c.return_code == 0 - assert "Courtesy Notice" in c.err + c = p.pipenv("install six") + assert c.return_code == 0 c = p.pipenv('run python -c "import click;print(click.__file__)"') assert c.return_code == 0 - assert c.out.strip().startswith(str(virtualenv)) + assert c.out.strip().startswith(str(project.virtualenv_location)) c = p.pipenv("clean --dry-run") assert c.return_code == 0 assert "click" in c.out diff --git a/tests/integration/test_run.py b/tests/integration/test_run.py index 8e82a6e2..28f97e48 100644 --- a/tests/integration/test_run.py +++ b/tests/integration/test_run.py @@ -1,10 +1,12 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function import os +import pytest + from pipenv.project import Project from pipenv.utils import temp_environ -import pytest - @pytest.mark.run @pytest.mark.dotenv diff --git a/tests/integration/test_sync.py b/tests/integration/test_sync.py index 2ef06ddc..d085aaf4 100644 --- a/tests/integration/test_sync.py +++ b/tests/integration/test_sync.py @@ -1,13 +1,16 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function +import json import os -from pipenv.utils import temp_environ - import pytest +from pipenv.utils import temp_environ + @pytest.mark.sync -def test_sync_error_without_lockfile(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi, chdir=True) as p: +def test_sync_error_without_lockfile(PipenvInstance): + with PipenvInstance(chdir=True) as p: with open(p.pipfile_path, 'w') as f: f.write(""" [packages] @@ -20,12 +23,17 @@ def test_sync_error_without_lockfile(PipenvInstance, pypi): @pytest.mark.sync @pytest.mark.lock -def test_mirror_lock_sync(PipenvInstance, pypi): +def test_mirror_lock_sync(PipenvInstance): with temp_environ(), PipenvInstance(chdir=True) as p: mirror_url = os.environ.pop('PIPENV_TEST_INDEX', "https://pypi.kennethreitz.org/simple") assert 'pypi.org' not in mirror_url with open(p.pipfile_path, 'w') as f: f.write(""" +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + [packages] six = "*" """.strip()) @@ -37,10 +45,10 @@ six = "*" @pytest.mark.sync @pytest.mark.lock -def test_sync_should_not_lock(PipenvInstance, pypi): +def test_sync_should_not_lock(PipenvInstance): """Sync should not touch the lock file, even if Pipfile is changed. """ - with PipenvInstance(pypi=pypi, chdir=True) as p: + with PipenvInstance(chdir=True) as p: with open(p.pipfile_path, 'w') as f: f.write(""" [packages] @@ -61,3 +69,46 @@ six = "*" c = p.pipenv('sync') assert c.return_code == 0 assert lockfile_content == p.lockfile + + +@pytest.mark.sync +@pytest.mark.lock +def test_sync_sequential_detect_errors(PipenvInstance): + with PipenvInstance() as p: + with open(p.pipfile_path, 'w') as f: + contents = """ +[packages] +requests = "*" + """.strip() + f.write(contents) + + c = p.pipenv('lock') + assert c.return_code == 0 + + # Force hash mismatch when installing `requests` + lock = p.lockfile + lock['default']['requests']['hashes'] = ['sha256:' + '0' * 64] + with open(p.lockfile_path, 'w') as f: + json.dump(lock, f) + + c = p.pipenv('sync --sequential') + assert c.return_code != 0 + + +@pytest.mark.sync +@pytest.mark.lock +def test_sync_sequential_verbose(PipenvInstance): + with PipenvInstance() as p: + with open(p.pipfile_path, 'w') as f: + contents = """ +[packages] +requests = "*" + """.strip() + f.write(contents) + + c = p.pipenv('lock') + assert c.return_code == 0 + + c = p.pipenv('sync --sequential --verbose') + for package in p.lockfile['default']: + assert 'Successfully installed {}'.format(package) in c.out diff --git a/tests/integration/test_uninstall.py b/tests/integration/test_uninstall.py index 5f493cac..bc82df8d 100644 --- a/tests/integration/test_uninstall.py +++ b/tests/integration/test_uninstall.py @@ -1,42 +1,62 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function import os import shutil -from pipenv.utils import temp_environ - import pytest +from pipenv.utils import temp_environ + @pytest.mark.run @pytest.mark.uninstall @pytest.mark.install -def test_uninstall(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi) as p: +def test_uninstall_requests(PipenvInstance): + # Uninstalling requests can fail even when uninstall Django below + # succeeds, if requests was de-vendored. + # See https://github.com/pypa/pipenv/issues/3644 for problems + # caused by devendoring + with PipenvInstance() as p: c = p.pipenv("install requests") assert c.return_code == 0 assert "requests" in p.pipfile["packages"] - assert "requests" in p.lockfile["default"] - assert "chardet" in p.lockfile["default"] - assert "idna" in p.lockfile["default"] - assert "urllib3" in p.lockfile["default"] - assert "certifi" in p.lockfile["default"] + + c = p.pipenv("run python -m requests.help") + assert c.return_code == 0 c = p.pipenv("uninstall requests") assert c.return_code == 0 assert "requests" not in p.pipfile["dev-packages"] - assert "requests" not in p.lockfile["develop"] - assert "chardet" not in p.lockfile["develop"] - assert "idna" not in p.lockfile["develop"] - assert "urllib3" not in p.lockfile["develop"] - assert "certifi" not in p.lockfile["develop"] c = p.pipenv("run python -m requests.help") assert c.return_code > 0 +def test_uninstall_django(PipenvInstance): + with PipenvInstance() as p: + c = p.pipenv("install Django==1.11.13") + assert c.return_code == 0 + assert "django" in p.pipfile["packages"] + assert "django" in p.lockfile["default"] + assert "pytz" in p.lockfile["default"] + + c = p.pipenv("run python -m django --version") + assert c.return_code == 0 + + c = p.pipenv("uninstall Django") + assert c.return_code == 0 + assert "django" not in p.pipfile["dev-packages"] + assert "django" not in p.lockfile["develop"] + assert p.lockfile["develop"] == {} + + c = p.pipenv("run python -m django --version") + assert c.return_code > 0 + + @pytest.mark.run @pytest.mark.uninstall @pytest.mark.install -def test_mirror_uninstall(PipenvInstance, pypi): +def test_mirror_uninstall(PipenvInstance): with temp_environ(), PipenvInstance(chdir=True) as p: mirror_url = os.environ.pop( @@ -44,35 +64,32 @@ def test_mirror_uninstall(PipenvInstance, pypi): ) assert "pypi.org" not in mirror_url - c = p.pipenv("install requests --pypi-mirror {0}".format(mirror_url)) + c = p.pipenv("install Django==1.11.13 --pypi-mirror {0}".format(mirror_url)) assert c.return_code == 0 - assert "requests" in p.pipfile["packages"] - assert "requests" in p.lockfile["default"] - assert "chardet" in p.lockfile["default"] - assert "idna" in p.lockfile["default"] - assert "urllib3" in p.lockfile["default"] - assert "certifi" in p.lockfile["default"] + assert "django" in p.pipfile["packages"] + assert "django" in p.lockfile["default"] + assert "pytz" in p.lockfile["default"] # Ensure the --pypi-mirror parameter hasn't altered the Pipfile or Pipfile.lock sources assert len(p.pipfile["source"]) == 1 assert len(p.lockfile["_meta"]["sources"]) == 1 assert "https://pypi.org/simple" == p.pipfile["source"][0]["url"] assert "https://pypi.org/simple" == p.lockfile["_meta"]["sources"][0]["url"] - c = p.pipenv("uninstall requests --pypi-mirror {0}".format(mirror_url)) + c = p.pipenv("run python -m django --version") assert c.return_code == 0 - assert "requests" not in p.pipfile["dev-packages"] - assert "requests" not in p.lockfile["develop"] - assert "chardet" not in p.lockfile["develop"] - assert "idna" not in p.lockfile["develop"] - assert "urllib3" not in p.lockfile["develop"] - assert "certifi" not in p.lockfile["develop"] + + c = p.pipenv("uninstall Django --pypi-mirror {0}".format(mirror_url)) + assert c.return_code == 0 + assert "django" not in p.pipfile["dev-packages"] + assert "django" not in p.lockfile["develop"] + assert p.lockfile["develop"] == {} # Ensure the --pypi-mirror parameter hasn't altered the Pipfile or Pipfile.lock sources assert len(p.pipfile["source"]) == 1 assert len(p.lockfile["_meta"]["sources"]) == 1 assert "https://pypi.org/simple" == p.pipfile["source"][0]["url"] assert "https://pypi.org/simple" == p.lockfile["_meta"]["sources"][0]["url"] - c = p.pipenv("run python -m requests.help") + c = p.pipenv("run python -m django --version") assert c.return_code > 0 @@ -80,61 +97,63 @@ def test_mirror_uninstall(PipenvInstance, pypi): @pytest.mark.uninstall @pytest.mark.install def test_uninstall_all_local_files(PipenvInstance, testsroot): - file_name = "requests-2.19.1.tar.gz" + file_name = "tablib-0.12.1.tar.gz" # Not sure where travis/appveyor run tests from - source_path = os.path.abspath(os.path.join(testsroot, "test_artifacts", file_name)) + source_path = os.path.abspath(os.path.join(testsroot, "pypi", "tablib", file_name)) with PipenvInstance(chdir=True) as p: shutil.copy(source_path, os.path.join(p.path, file_name)) - os.mkdir(os.path.join(p.path, "requests")) + os.mkdir(os.path.join(p.path, "tablib")) c = p.pipenv("install {}".format(file_name)) assert c.return_code == 0 c = p.pipenv("uninstall --all") assert c.return_code == 0 - assert "requests" in c.out + assert "tablib" in c.out # Uninstall --all is not supposed to remove things from the pipfile # Note that it didn't before, but that instead local filenames showed as hashes - assert "requests" in p.pipfile["packages"] + assert "tablib" in p.pipfile["packages"] @pytest.mark.run @pytest.mark.uninstall @pytest.mark.install -def test_uninstall_all_dev(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi) as p: - c = p.pipenv("install --dev requests six") +def test_uninstall_all_dev(PipenvInstance): + with PipenvInstance() as p: + c = p.pipenv("install --dev Django==1.11.13 six") assert c.return_code == 0 - c = p.pipenv("install pytz") + c = p.pipenv("install tablib") assert c.return_code == 0 - assert "pytz" in p.pipfile["packages"] - assert "requests" in p.pipfile["dev-packages"] + assert "tablib" in p.pipfile["packages"] + assert "django" in p.pipfile["dev-packages"] assert "six" in p.pipfile["dev-packages"] - assert "pytz" in p.lockfile["default"] - assert "requests" in p.lockfile["develop"] + assert "tablib" in p.lockfile["default"] + assert "django" in p.lockfile["develop"] assert "six" in p.lockfile["develop"] + c = p.pipenv('run python -c "import django"') + assert c.return_code == 0 + c = p.pipenv("uninstall --all-dev") assert c.return_code == 0 - assert "requests" not in p.pipfile["dev-packages"] - assert "six" not in p.pipfile["dev-packages"] - assert "requests" not in p.lockfile["develop"] + assert p.pipfile["dev-packages"] == {} + assert "django" not in p.lockfile["develop"] assert "six" not in p.lockfile["develop"] - assert "pytz" in p.pipfile["packages"] - assert "pytz" in p.lockfile["default"] + assert "tablib" in p.pipfile["packages"] + assert "tablib" in p.lockfile["default"] - c = p.pipenv("run python -m requests.help") + c = p.pipenv('run python -c "import django"') assert c.return_code > 0 - c = p.pipenv('run python -c "import pytz"') + c = p.pipenv('run python -c "import tablib"') assert c.return_code == 0 @pytest.mark.uninstall @pytest.mark.run -def test_normalize_name_uninstall(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi) as p: +def test_normalize_name_uninstall(PipenvInstance): + with PipenvInstance() as p: with open(p.pipfile_path, "w") as f: contents = """ # Pre comment diff --git a/tests/integration/test_windows.py b/tests/integration/test_windows.py index da138f7a..b303d0ab 100644 --- a/tests/integration/test_windows.py +++ b/tests/integration/test_windows.py @@ -1,20 +1,22 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function import os -from pipenv.project import Project -from pipenv._compat import Path - import pytest +from pipenv._compat import Path +from pipenv.project import Project + # This module is run only on Windows. pytestmark = pytest.mark.skipif(os.name != 'nt', reason="only relevant on windows") @pytest.mark.project -def test_case_changes_windows(PipenvInstance, pypi): +def test_case_changes_windows(PipenvInstance): """Test project matching for case changes on Windows. """ - with PipenvInstance(pypi=pypi, chdir=True) as p: + with PipenvInstance(chdir=True) as p: c = p.pipenv('install pytz') assert c.return_code == 0 @@ -38,7 +40,7 @@ def test_case_changes_windows(PipenvInstance, pypi): @pytest.mark.files -def test_local_path_windows(PipenvInstance, pypi): +def test_local_path_windows(PipenvInstance): whl = ( Path(__file__).parent.parent .joinpath('pypi', 'six', 'six-1.11.0-py2.py3-none-any.whl') @@ -47,13 +49,13 @@ def test_local_path_windows(PipenvInstance, pypi): whl = whl.resolve() except OSError: whl = whl.absolute() - with PipenvInstance(pypi=pypi, chdir=True) as p: + with PipenvInstance(chdir=True) as p: c = p.pipenv('install "{0}"'.format(whl)) assert c.return_code == 0 @pytest.mark.files -def test_local_path_windows_forward_slash(PipenvInstance, pypi): +def test_local_path_windows_forward_slash(PipenvInstance): whl = ( Path(__file__).parent.parent .joinpath('pypi', 'six', 'six-1.11.0-py2.py3-none-any.whl') @@ -62,14 +64,14 @@ def test_local_path_windows_forward_slash(PipenvInstance, pypi): whl = whl.resolve() except OSError: whl = whl.absolute() - with PipenvInstance(pypi=pypi, chdir=True) as p: + with PipenvInstance(chdir=True) as p: c = p.pipenv('install "{0}"'.format(whl.as_posix())) assert c.return_code == 0 @pytest.mark.cli -def test_pipenv_clean_windows(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi, chdir=True) as p: +def test_pipenv_clean_windows(PipenvInstance): + with PipenvInstance(chdir=True) as p: c = p.pipenv('install requests') assert c.return_code == 0 c = p.pipenv('run pip install click') diff --git a/tests/pypi/alembic/alembic-0.9.8.tar.gz b/tests/pypi/alembic/alembic-0.9.8.tar.gz deleted file mode 100644 index a23b9200..00000000 Binary files a/tests/pypi/alembic/alembic-0.9.8.tar.gz and /dev/null differ diff --git a/tests/pypi/alembic/alembic-0.9.9.tar.gz b/tests/pypi/alembic/alembic-0.9.9.tar.gz deleted file mode 100644 index 0c77f5df..00000000 Binary files a/tests/pypi/alembic/alembic-0.9.9.tar.gz and /dev/null differ diff --git a/tests/pypi/apscheduler/APScheduler-3.5.1-py2.py3-none-any.whl b/tests/pypi/apscheduler/APScheduler-3.5.1-py2.py3-none-any.whl deleted file mode 100644 index faf7dbf5..00000000 Binary files a/tests/pypi/apscheduler/APScheduler-3.5.1-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/apscheduler/APScheduler-3.5.1.tar.gz b/tests/pypi/apscheduler/APScheduler-3.5.1.tar.gz deleted file mode 100644 index 672b4d13..00000000 Binary files a/tests/pypi/apscheduler/APScheduler-3.5.1.tar.gz and /dev/null differ diff --git a/tests/pypi/certifi/certifi-2018.1.18-py2.py3-none-any.whl b/tests/pypi/certifi/certifi-2018.1.18-py2.py3-none-any.whl deleted file mode 100644 index 3348f725..00000000 Binary files a/tests/pypi/certifi/certifi-2018.1.18-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/certifi/certifi-2018.1.18.tar.gz b/tests/pypi/certifi/certifi-2018.1.18.tar.gz deleted file mode 100644 index a7f32456..00000000 Binary files a/tests/pypi/certifi/certifi-2018.1.18.tar.gz and /dev/null differ diff --git a/tests/pypi/certifi/certifi-2018.4.16-py2.py3-none-any.whl b/tests/pypi/certifi/certifi-2018.4.16-py2.py3-none-any.whl deleted file mode 100644 index 37d13a39..00000000 Binary files a/tests/pypi/certifi/certifi-2018.4.16-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/certifi/certifi-2018.4.16.tar.gz b/tests/pypi/certifi/certifi-2018.4.16.tar.gz deleted file mode 100644 index 47b37025..00000000 Binary files a/tests/pypi/certifi/certifi-2018.4.16.tar.gz and /dev/null differ diff --git a/tests/pypi/chardet/chardet-3.0.4-py2.py3-none-any.whl b/tests/pypi/chardet/chardet-3.0.4-py2.py3-none-any.whl deleted file mode 100644 index d276977d..00000000 Binary files a/tests/pypi/chardet/chardet-3.0.4-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/chardet/chardet-3.0.4.tar.gz b/tests/pypi/chardet/chardet-3.0.4.tar.gz deleted file mode 100644 index 13028bfc..00000000 Binary files a/tests/pypi/chardet/chardet-3.0.4.tar.gz and /dev/null differ diff --git a/tests/pypi/click/click-6.7-py2.py3-none-any.whl b/tests/pypi/click/click-6.7-py2.py3-none-any.whl deleted file mode 100644 index 56c7ff34..00000000 Binary files a/tests/pypi/click/click-6.7-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/click/click-6.7.tar.gz b/tests/pypi/click/click-6.7.tar.gz deleted file mode 100644 index b0790fc0..00000000 Binary files a/tests/pypi/click/click-6.7.tar.gz and /dev/null differ diff --git a/tests/pypi/colorama/colorama-0.3.9-py2.py3-none-any.whl b/tests/pypi/colorama/colorama-0.3.9-py2.py3-none-any.whl deleted file mode 100644 index 29b83f06..00000000 Binary files a/tests/pypi/colorama/colorama-0.3.9-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/dateparser/dateparser-0.7.0-py2.py3-none-any.whl b/tests/pypi/dateparser/dateparser-0.7.0-py2.py3-none-any.whl deleted file mode 100644 index 14f7d7e1..00000000 Binary files a/tests/pypi/dateparser/dateparser-0.7.0-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/dateparser/dateparser-0.7.0.tar.gz b/tests/pypi/dateparser/dateparser-0.7.0.tar.gz deleted file mode 100644 index a37b5536..00000000 Binary files a/tests/pypi/dateparser/dateparser-0.7.0.tar.gz and /dev/null differ diff --git a/tests/pypi/depends-on-marked-package/depends_on_marked_package-0.0.1-py2.py3-none-any.whl b/tests/pypi/depends-on-marked-package/depends_on_marked_package-0.0.1-py2.py3-none-any.whl deleted file mode 100644 index d320deb3..00000000 Binary files a/tests/pypi/depends-on-marked-package/depends_on_marked_package-0.0.1-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/django-classy-tags/django-classy-tags-0.8.0.tar.gz b/tests/pypi/django-classy-tags/django-classy-tags-0.8.0.tar.gz deleted file mode 100644 index b88f48ae..00000000 Binary files a/tests/pypi/django-classy-tags/django-classy-tags-0.8.0.tar.gz and /dev/null differ diff --git a/tests/pypi/django-classy-tags/django_classy_tags-0.8.0-py2.py3-none-any.whl b/tests/pypi/django-classy-tags/django_classy_tags-0.8.0-py2.py3-none-any.whl deleted file mode 100644 index 47ac7f85..00000000 Binary files a/tests/pypi/django-classy-tags/django_classy_tags-0.8.0-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/django-formtools/django-formtools-2.1.tar.gz b/tests/pypi/django-formtools/django-formtools-2.1.tar.gz deleted file mode 100644 index 2b8ca348..00000000 Binary files a/tests/pypi/django-formtools/django-formtools-2.1.tar.gz and /dev/null differ diff --git a/tests/pypi/django-formtools/django_formtools-2.1-py2.py3-none-any.whl b/tests/pypi/django-formtools/django_formtools-2.1-py2.py3-none-any.whl deleted file mode 100644 index 1086751a..00000000 Binary files a/tests/pypi/django-formtools/django_formtools-2.1-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/django-sekizai/django-sekizai-0.10.0.tar.gz b/tests/pypi/django-sekizai/django-sekizai-0.10.0.tar.gz deleted file mode 100644 index b11067dc..00000000 Binary files a/tests/pypi/django-sekizai/django-sekizai-0.10.0.tar.gz and /dev/null differ diff --git a/tests/pypi/django-sekizai/django_sekizai-0.10.0-py2.py3-none-any.whl b/tests/pypi/django-sekizai/django_sekizai-0.10.0-py2.py3-none-any.whl deleted file mode 100644 index 20c107cc..00000000 Binary files a/tests/pypi/django-sekizai/django_sekizai-0.10.0-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/django-treebeard/django-treebeard-4.2.0.tar.gz b/tests/pypi/django-treebeard/django-treebeard-4.2.0.tar.gz deleted file mode 100644 index 3d92cb7e..00000000 Binary files a/tests/pypi/django-treebeard/django-treebeard-4.2.0.tar.gz and /dev/null differ diff --git a/tests/pypi/django/Django-1.11.10-py2.py3-none-any.whl b/tests/pypi/django/Django-1.11.10-py2.py3-none-any.whl deleted file mode 100644 index 8d515c4a..00000000 Binary files a/tests/pypi/django/Django-1.11.10-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/django/Django-1.11.10.tar.gz b/tests/pypi/django/Django-1.11.10.tar.gz deleted file mode 100644 index 2d69623e..00000000 Binary files a/tests/pypi/django/Django-1.11.10.tar.gz and /dev/null differ diff --git a/tests/pypi/django/Django-1.11.13-py2.py3-none-any.whl b/tests/pypi/django/Django-1.11.13-py2.py3-none-any.whl deleted file mode 100644 index 38dfe813..00000000 Binary files a/tests/pypi/django/Django-1.11.13-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/django/Django-1.11.13.tar.gz b/tests/pypi/django/Django-1.11.13.tar.gz deleted file mode 100644 index ba99816e..00000000 Binary files a/tests/pypi/django/Django-1.11.13.tar.gz and /dev/null differ diff --git a/tests/pypi/django/Django-2.0.6-py3-none-any.whl b/tests/pypi/django/Django-2.0.6-py3-none-any.whl deleted file mode 100644 index b966b5d4..00000000 Binary files a/tests/pypi/django/Django-2.0.6-py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/django/Django-2.0.6.tar.gz b/tests/pypi/django/Django-2.0.6.tar.gz deleted file mode 100644 index bc69557b..00000000 Binary files a/tests/pypi/django/Django-2.0.6.tar.gz and /dev/null differ diff --git a/tests/pypi/djangocms-admin-style/djangocms-admin-style-1.2.7.tar.gz b/tests/pypi/djangocms-admin-style/djangocms-admin-style-1.2.7.tar.gz deleted file mode 100644 index f3afc45b..00000000 Binary files a/tests/pypi/djangocms-admin-style/djangocms-admin-style-1.2.7.tar.gz and /dev/null differ diff --git a/tests/pypi/docopt/docopt-0.6.2.tar.gz b/tests/pypi/docopt/docopt-0.6.2.tar.gz deleted file mode 100644 index 153ce415..00000000 Binary files a/tests/pypi/docopt/docopt-0.6.2.tar.gz and /dev/null differ diff --git a/tests/pypi/et-xmlfile/et_xmlfile-1.0.1.tar.gz b/tests/pypi/et-xmlfile/et_xmlfile-1.0.1.tar.gz deleted file mode 100644 index 73a5894e..00000000 Binary files a/tests/pypi/et-xmlfile/et_xmlfile-1.0.1.tar.gz and /dev/null differ diff --git a/tests/pypi/flask/Flask-0.12.2-py2.py3-none-any.whl b/tests/pypi/flask/Flask-0.12.2-py2.py3-none-any.whl deleted file mode 100644 index 98e55415..00000000 Binary files a/tests/pypi/flask/Flask-0.12.2-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/flask/Flask-0.12.2.tar.gz b/tests/pypi/flask/Flask-0.12.2.tar.gz deleted file mode 100644 index 278c2108..00000000 Binary files a/tests/pypi/flask/Flask-0.12.2.tar.gz and /dev/null differ diff --git a/tests/pypi/funcsigs/funcsigs-1.0.2-py2.py3-none-any.whl b/tests/pypi/funcsigs/funcsigs-1.0.2-py2.py3-none-any.whl deleted file mode 100644 index ec5973f4..00000000 Binary files a/tests/pypi/funcsigs/funcsigs-1.0.2-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/funcsigs/funcsigs-1.0.2.tar.gz b/tests/pypi/funcsigs/funcsigs-1.0.2.tar.gz deleted file mode 100644 index c53bf6aa..00000000 Binary files a/tests/pypi/funcsigs/funcsigs-1.0.2.tar.gz and /dev/null differ diff --git a/tests/pypi/futures/futures-3.2.0-py2-none-any.whl b/tests/pypi/futures/futures-3.2.0-py2-none-any.whl deleted file mode 100644 index c0659c1a..00000000 Binary files a/tests/pypi/futures/futures-3.2.0-py2-none-any.whl and /dev/null differ diff --git a/tests/pypi/futures/futures-3.2.0.tar.gz b/tests/pypi/futures/futures-3.2.0.tar.gz deleted file mode 100644 index 5e896bfa..00000000 Binary files a/tests/pypi/futures/futures-3.2.0.tar.gz and /dev/null differ diff --git a/tests/pypi/gitdb2/gitdb2-2.0.3-py2.py3-none-any.whl b/tests/pypi/gitdb2/gitdb2-2.0.3-py2.py3-none-any.whl deleted file mode 100644 index 26ffc716..00000000 Binary files a/tests/pypi/gitdb2/gitdb2-2.0.3-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/gitdb2/gitdb2-2.0.3.tar.gz b/tests/pypi/gitdb2/gitdb2-2.0.3.tar.gz deleted file mode 100644 index de6ee8af..00000000 Binary files a/tests/pypi/gitdb2/gitdb2-2.0.3.tar.gz and /dev/null differ diff --git a/tests/pypi/humanize/humanize-0.5.1.tar.gz b/tests/pypi/humanize/humanize-0.5.1.tar.gz deleted file mode 100644 index b293ff05..00000000 Binary files a/tests/pypi/humanize/humanize-0.5.1.tar.gz and /dev/null differ diff --git a/tests/pypi/ibm-db-sa-py3/ibm-db-sa-py3-0.3.0.tar.gz b/tests/pypi/ibm-db-sa-py3/ibm-db-sa-py3-0.3.0.tar.gz deleted file mode 100644 index dd4e4a88..00000000 Binary files a/tests/pypi/ibm-db-sa-py3/ibm-db-sa-py3-0.3.0.tar.gz and /dev/null differ diff --git a/tests/pypi/ibm-db-sa-py3/ibm-db-sa-py3-0.3.1-1.tar.gz b/tests/pypi/ibm-db-sa-py3/ibm-db-sa-py3-0.3.1-1.tar.gz deleted file mode 100644 index 7770d23e..00000000 Binary files a/tests/pypi/ibm-db-sa-py3/ibm-db-sa-py3-0.3.1-1.tar.gz and /dev/null differ diff --git a/tests/pypi/idna/idna-2.6-py2.py3-none-any.whl b/tests/pypi/idna/idna-2.6-py2.py3-none-any.whl deleted file mode 100644 index 11cb4140..00000000 Binary files a/tests/pypi/idna/idna-2.6-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/idna/idna-2.6.tar.gz b/tests/pypi/idna/idna-2.6.tar.gz deleted file mode 100644 index d38e1f29..00000000 Binary files a/tests/pypi/idna/idna-2.6.tar.gz and /dev/null differ diff --git a/tests/pypi/idna/idna-2.7-py2.py3-none-any.whl b/tests/pypi/idna/idna-2.7-py2.py3-none-any.whl deleted file mode 100644 index 9d1a3285..00000000 Binary files a/tests/pypi/idna/idna-2.7-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/idna/idna-2.7.tar.gz b/tests/pypi/idna/idna-2.7.tar.gz deleted file mode 100644 index 8f0c5f27..00000000 Binary files a/tests/pypi/idna/idna-2.7.tar.gz and /dev/null differ diff --git a/tests/pypi/itsdangerous/itsdangerous-0.24.tar.gz b/tests/pypi/itsdangerous/itsdangerous-0.24.tar.gz deleted file mode 100644 index d0416330..00000000 Binary files a/tests/pypi/itsdangerous/itsdangerous-0.24.tar.gz and /dev/null differ diff --git a/tests/pypi/jdcal/jdcal-1.3.tar.gz b/tests/pypi/jdcal/jdcal-1.3.tar.gz deleted file mode 100644 index 2f3ba8ce..00000000 Binary files a/tests/pypi/jdcal/jdcal-1.3.tar.gz and /dev/null differ diff --git a/tests/pypi/jdcal/jdcal-1.4-py2.py3-none-any.whl b/tests/pypi/jdcal/jdcal-1.4-py2.py3-none-any.whl deleted file mode 100644 index bac3f040..00000000 Binary files a/tests/pypi/jdcal/jdcal-1.4-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/jdcal/jdcal-1.4.tar.gz b/tests/pypi/jdcal/jdcal-1.4.tar.gz deleted file mode 100644 index e82fdae7..00000000 Binary files a/tests/pypi/jdcal/jdcal-1.4.tar.gz and /dev/null differ diff --git a/tests/pypi/jinja2/Jinja2-2.10-py2.py3-none-any.whl b/tests/pypi/jinja2/Jinja2-2.10-py2.py3-none-any.whl deleted file mode 100644 index 7bc4e35f..00000000 Binary files a/tests/pypi/jinja2/Jinja2-2.10-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/jinja2/Jinja2-2.10.tar.gz b/tests/pypi/jinja2/Jinja2-2.10.tar.gz deleted file mode 100644 index c311087a..00000000 Binary files a/tests/pypi/jinja2/Jinja2-2.10.tar.gz and /dev/null differ diff --git a/tests/pypi/mako/Mako-1.0.7.tar.gz b/tests/pypi/mako/Mako-1.0.7.tar.gz deleted file mode 100644 index 9e5825f6..00000000 Binary files a/tests/pypi/mako/Mako-1.0.7.tar.gz and /dev/null differ diff --git a/tests/pypi/markupsafe/MarkupSafe-1.0.tar.gz b/tests/pypi/markupsafe/MarkupSafe-1.0.tar.gz deleted file mode 100644 index 606021ae..00000000 Binary files a/tests/pypi/markupsafe/MarkupSafe-1.0.tar.gz and /dev/null differ diff --git a/tests/pypi/maya/maya-0.3.4-py2.py3-none-any.whl b/tests/pypi/maya/maya-0.3.4-py2.py3-none-any.whl deleted file mode 100644 index 7dcd32bd..00000000 Binary files a/tests/pypi/maya/maya-0.3.4-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/maya/maya-0.3.4.tar.gz b/tests/pypi/maya/maya-0.3.4.tar.gz deleted file mode 100644 index 8a20fb58..00000000 Binary files a/tests/pypi/maya/maya-0.3.4.tar.gz and /dev/null differ diff --git a/tests/pypi/maya/maya-0.5.0-py2.py3-none-any.whl b/tests/pypi/maya/maya-0.5.0-py2.py3-none-any.whl deleted file mode 100644 index b36d48e5..00000000 Binary files a/tests/pypi/maya/maya-0.5.0-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/maya/maya-0.5.0.tar.gz b/tests/pypi/maya/maya-0.5.0.tar.gz deleted file mode 100644 index 5f4016f0..00000000 Binary files a/tests/pypi/maya/maya-0.5.0.tar.gz and /dev/null differ diff --git a/tests/pypi/multidict/multidict-4.1.0-cp36-cp36m-win_amd64.whl b/tests/pypi/multidict/multidict-4.1.0-cp36-cp36m-win_amd64.whl deleted file mode 100644 index 6f8c38f3..00000000 Binary files a/tests/pypi/multidict/multidict-4.1.0-cp36-cp36m-win_amd64.whl and /dev/null differ diff --git a/tests/pypi/multidict/multidict-4.1.0.tar.gz b/tests/pypi/multidict/multidict-4.1.0.tar.gz deleted file mode 100644 index 89adbb22..00000000 Binary files a/tests/pypi/multidict/multidict-4.1.0.tar.gz and /dev/null differ diff --git a/tests/pypi/odfpy/odfpy-1.3.6-py2.7.egg b/tests/pypi/odfpy/odfpy-1.3.6-py2.7.egg deleted file mode 100644 index ccb4eeed..00000000 Binary files a/tests/pypi/odfpy/odfpy-1.3.6-py2.7.egg and /dev/null differ diff --git a/tests/pypi/odfpy/odfpy-1.3.6.tar.gz b/tests/pypi/odfpy/odfpy-1.3.6.tar.gz deleted file mode 100644 index c8861245..00000000 Binary files a/tests/pypi/odfpy/odfpy-1.3.6.tar.gz and /dev/null differ diff --git a/tests/pypi/openpyxl/openpyxl-2.5.0.tar.gz b/tests/pypi/openpyxl/openpyxl-2.5.0.tar.gz deleted file mode 100644 index 52f84550..00000000 Binary files a/tests/pypi/openpyxl/openpyxl-2.5.0.tar.gz and /dev/null differ diff --git a/tests/pypi/openpyxl/openpyxl-2.5.4.tar.gz b/tests/pypi/openpyxl/openpyxl-2.5.4.tar.gz deleted file mode 100644 index b257fba5..00000000 Binary files a/tests/pypi/openpyxl/openpyxl-2.5.4.tar.gz and /dev/null differ diff --git a/tests/pypi/pandas/pandas-0.22.0-cp27-cp27m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl b/tests/pypi/pandas/pandas-0.22.0-cp27-cp27m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl deleted file mode 100644 index 6230a732..00000000 Binary files a/tests/pypi/pandas/pandas-0.22.0-cp27-cp27m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl and /dev/null differ diff --git a/tests/pypi/pandas/pandas-0.22.0-cp27-cp27m-win_amd64.whl b/tests/pypi/pandas/pandas-0.22.0-cp27-cp27m-win_amd64.whl deleted file mode 100644 index bf879e90..00000000 Binary files a/tests/pypi/pandas/pandas-0.22.0-cp27-cp27m-win_amd64.whl and /dev/null differ diff --git a/tests/pypi/pandas/pandas-0.22.0-cp27-cp27mu-manylinux1_i686.whl b/tests/pypi/pandas/pandas-0.22.0-cp27-cp27mu-manylinux1_i686.whl deleted file mode 100644 index d01d7706..00000000 Binary files a/tests/pypi/pandas/pandas-0.22.0-cp27-cp27mu-manylinux1_i686.whl and /dev/null differ diff --git a/tests/pypi/pandas/pandas-0.22.0-cp27-cp27mu-manylinux1_x86_64.whl b/tests/pypi/pandas/pandas-0.22.0-cp27-cp27mu-manylinux1_x86_64.whl deleted file mode 100644 index 878bb1bd..00000000 Binary files a/tests/pypi/pandas/pandas-0.22.0-cp27-cp27mu-manylinux1_x86_64.whl and /dev/null differ diff --git a/tests/pypi/pandas/pandas-0.22.0-cp36-cp36m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl b/tests/pypi/pandas/pandas-0.22.0-cp36-cp36m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl deleted file mode 100644 index 19a474d9..00000000 Binary files a/tests/pypi/pandas/pandas-0.22.0-cp36-cp36m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl and /dev/null differ diff --git a/tests/pypi/pandas/pandas-0.22.0-cp36-cp36m-manylinux1_x86_64.whl b/tests/pypi/pandas/pandas-0.22.0-cp36-cp36m-manylinux1_x86_64.whl deleted file mode 100644 index 6ef88537..00000000 Binary files a/tests/pypi/pandas/pandas-0.22.0-cp36-cp36m-manylinux1_x86_64.whl and /dev/null differ diff --git a/tests/pypi/pandas/pandas-0.22.0-cp36-cp36m-win_amd64.whl b/tests/pypi/pandas/pandas-0.22.0-cp36-cp36m-win_amd64.whl deleted file mode 100644 index 871d5094..00000000 Binary files a/tests/pypi/pandas/pandas-0.22.0-cp36-cp36m-win_amd64.whl and /dev/null differ diff --git a/tests/pypi/parse/parse-1.8.2.tar.gz b/tests/pypi/parse/parse-1.8.2.tar.gz deleted file mode 100644 index d24aa0d1..00000000 Binary files a/tests/pypi/parse/parse-1.8.2.tar.gz and /dev/null differ diff --git a/tests/pypi/pendulum/pendulum-1.4.4-cp27-cp27m-manylinux1_x86_64.whl b/tests/pypi/pendulum/pendulum-1.4.4-cp27-cp27m-manylinux1_x86_64.whl deleted file mode 100644 index b38c9d13..00000000 Binary files a/tests/pypi/pendulum/pendulum-1.4.4-cp27-cp27m-manylinux1_x86_64.whl and /dev/null differ diff --git a/tests/pypi/pendulum/pendulum-1.4.4-cp36-cp36m-macosx_10_13_x86_64.whl b/tests/pypi/pendulum/pendulum-1.4.4-cp36-cp36m-macosx_10_13_x86_64.whl deleted file mode 100644 index 72ef97b4..00000000 Binary files a/tests/pypi/pendulum/pendulum-1.4.4-cp36-cp36m-macosx_10_13_x86_64.whl and /dev/null differ diff --git a/tests/pypi/pendulum/pendulum-1.4.4-cp36-cp36m-manylinux1_x86_64.whl b/tests/pypi/pendulum/pendulum-1.4.4-cp36-cp36m-manylinux1_x86_64.whl deleted file mode 100644 index 31d50199..00000000 Binary files a/tests/pypi/pendulum/pendulum-1.4.4-cp36-cp36m-manylinux1_x86_64.whl and /dev/null differ diff --git a/tests/pypi/pendulum/pendulum-1.4.4.tar.gz b/tests/pypi/pendulum/pendulum-1.4.4.tar.gz deleted file mode 100644 index 826bc090..00000000 Binary files a/tests/pypi/pendulum/pendulum-1.4.4.tar.gz and /dev/null differ diff --git a/tests/pypi/pendulum/pendulum-1.5.1-cp27-cp27m-manylinux1_i686.whl b/tests/pypi/pendulum/pendulum-1.5.1-cp27-cp27m-manylinux1_i686.whl deleted file mode 100644 index 0d22ef79..00000000 Binary files a/tests/pypi/pendulum/pendulum-1.5.1-cp27-cp27m-manylinux1_i686.whl and /dev/null differ diff --git a/tests/pypi/pendulum/pendulum-1.5.1-cp27-cp27m-manylinux1_x86_64.whl b/tests/pypi/pendulum/pendulum-1.5.1-cp27-cp27m-manylinux1_x86_64.whl deleted file mode 100644 index d29ce7a8..00000000 Binary files a/tests/pypi/pendulum/pendulum-1.5.1-cp27-cp27m-manylinux1_x86_64.whl and /dev/null differ diff --git a/tests/pypi/pendulum/pendulum-1.5.1-cp36-cp36m-manylinux1_i686.whl b/tests/pypi/pendulum/pendulum-1.5.1-cp36-cp36m-manylinux1_i686.whl deleted file mode 100644 index 53eaa9c3..00000000 Binary files a/tests/pypi/pendulum/pendulum-1.5.1-cp36-cp36m-manylinux1_i686.whl and /dev/null differ diff --git a/tests/pypi/pendulum/pendulum-1.5.1-cp36-cp36m-manylinux1_x86_64.whl b/tests/pypi/pendulum/pendulum-1.5.1-cp36-cp36m-manylinux1_x86_64.whl deleted file mode 100644 index af4c30b9..00000000 Binary files a/tests/pypi/pendulum/pendulum-1.5.1-cp36-cp36m-manylinux1_x86_64.whl and /dev/null differ diff --git a/tests/pypi/pendulum/pendulum-1.5.1.tar.gz b/tests/pypi/pendulum/pendulum-1.5.1.tar.gz deleted file mode 100644 index 77aff89d..00000000 Binary files a/tests/pypi/pendulum/pendulum-1.5.1.tar.gz and /dev/null differ diff --git a/tests/pypi/py/py-1.5.3-py2.py3-none-any.whl b/tests/pypi/py/py-1.5.3-py2.py3-none-any.whl deleted file mode 100644 index 127d886e..00000000 Binary files a/tests/pypi/py/py-1.5.3-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/py/py-1.5.3.tar.gz b/tests/pypi/py/py-1.5.3.tar.gz deleted file mode 100644 index a0592b18..00000000 Binary files a/tests/pypi/py/py-1.5.3.tar.gz and /dev/null differ diff --git a/tests/pypi/pysocks/PySocks-1.6.8.tar.gz b/tests/pypi/pysocks/PySocks-1.6.8.tar.gz deleted file mode 100644 index e6366dcf..00000000 Binary files a/tests/pypi/pysocks/PySocks-1.6.8.tar.gz and /dev/null differ diff --git a/tests/pypi/pytest/pytest-3.1.0-py2.py3-none-any.whl b/tests/pypi/pytest/pytest-3.1.0-py2.py3-none-any.whl deleted file mode 100644 index dfaf980a..00000000 Binary files a/tests/pypi/pytest/pytest-3.1.0-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/pytest/pytest-3.1.0.tar.gz b/tests/pypi/pytest/pytest-3.1.0.tar.gz deleted file mode 100644 index bb1e3573..00000000 Binary files a/tests/pypi/pytest/pytest-3.1.0.tar.gz and /dev/null differ diff --git a/tests/pypi/pytest/pytest-3.1.1-py2.py3-none-any.whl b/tests/pypi/pytest/pytest-3.1.1-py2.py3-none-any.whl deleted file mode 100644 index aaab63cd..00000000 Binary files a/tests/pypi/pytest/pytest-3.1.1-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/pytest/pytest-3.1.1.tar.gz b/tests/pypi/pytest/pytest-3.1.1.tar.gz deleted file mode 100644 index 5b5f6dc4..00000000 Binary files a/tests/pypi/pytest/pytest-3.1.1.tar.gz and /dev/null differ diff --git a/tests/pypi/python-dateutil/python-dateutil-2.6.1.tar.gz b/tests/pypi/python-dateutil/python-dateutil-2.6.1.tar.gz deleted file mode 100644 index 6b86c6d3..00000000 Binary files a/tests/pypi/python-dateutil/python-dateutil-2.6.1.tar.gz and /dev/null differ diff --git a/tests/pypi/python-dateutil/python-dateutil-2.7.3.tar.gz b/tests/pypi/python-dateutil/python-dateutil-2.7.3.tar.gz deleted file mode 100644 index 8f1a68d9..00000000 Binary files a/tests/pypi/python-dateutil/python-dateutil-2.7.3.tar.gz and /dev/null differ diff --git a/tests/pypi/python-dateutil/python_dateutil-2.6.1-py2.py3-none-any.whl b/tests/pypi/python-dateutil/python_dateutil-2.6.1-py2.py3-none-any.whl deleted file mode 100644 index 97b3947a..00000000 Binary files a/tests/pypi/python-dateutil/python_dateutil-2.6.1-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/python-dateutil/python_dateutil-2.7.3-py2.py3-none-any.whl b/tests/pypi/python-dateutil/python_dateutil-2.7.3-py2.py3-none-any.whl deleted file mode 100644 index 55da69c3..00000000 Binary files a/tests/pypi/python-dateutil/python_dateutil-2.7.3-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/python-editor/python-editor-1.0.3.tar.gz b/tests/pypi/python-editor/python-editor-1.0.3.tar.gz deleted file mode 100644 index 8149cccc..00000000 Binary files a/tests/pypi/python-editor/python-editor-1.0.3.tar.gz and /dev/null differ diff --git a/tests/pypi/pytz/pytz-2018.3-py2.py3-none-any.whl b/tests/pypi/pytz/pytz-2018.3-py2.py3-none-any.whl deleted file mode 100644 index d274bd36..00000000 Binary files a/tests/pypi/pytz/pytz-2018.3-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/pytz/pytz-2018.3.tar.gz b/tests/pypi/pytz/pytz-2018.3.tar.gz deleted file mode 100644 index c8045799..00000000 Binary files a/tests/pypi/pytz/pytz-2018.3.tar.gz and /dev/null differ diff --git a/tests/pypi/pytz/pytz-2018.4-py2.py3-none-any.whl b/tests/pypi/pytz/pytz-2018.4-py2.py3-none-any.whl deleted file mode 100644 index 53b5bd48..00000000 Binary files a/tests/pypi/pytz/pytz-2018.4-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/pytz/pytz-2018.4.tar.gz b/tests/pypi/pytz/pytz-2018.4.tar.gz deleted file mode 100644 index 58526bf0..00000000 Binary files a/tests/pypi/pytz/pytz-2018.4.tar.gz and /dev/null differ diff --git a/tests/pypi/pytzdata/pytzdata-2018.3-py2.py3-none-any.whl b/tests/pypi/pytzdata/pytzdata-2018.3-py2.py3-none-any.whl deleted file mode 100644 index af5f8bf7..00000000 Binary files a/tests/pypi/pytzdata/pytzdata-2018.3-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/pytzdata/pytzdata-2018.3.tar.gz b/tests/pypi/pytzdata/pytzdata-2018.3.tar.gz deleted file mode 100644 index fd4e397d..00000000 Binary files a/tests/pypi/pytzdata/pytzdata-2018.3.tar.gz and /dev/null differ diff --git a/tests/pypi/pytzdata/pytzdata-2018.5-py2.py3-none-any.whl b/tests/pypi/pytzdata/pytzdata-2018.5-py2.py3-none-any.whl deleted file mode 100644 index 73740fb2..00000000 Binary files a/tests/pypi/pytzdata/pytzdata-2018.5-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/pytzdata/pytzdata-2018.5.tar.gz b/tests/pypi/pytzdata/pytzdata-2018.5.tar.gz deleted file mode 100644 index 506cb247..00000000 Binary files a/tests/pypi/pytzdata/pytzdata-2018.5.tar.gz and /dev/null differ diff --git a/tests/pypi/pyyaml/PyYAML-3.12-cp27-cp27m-win32.whl b/tests/pypi/pyyaml/PyYAML-3.12-cp27-cp27m-win32.whl deleted file mode 100644 index d9f5e245..00000000 Binary files a/tests/pypi/pyyaml/PyYAML-3.12-cp27-cp27m-win32.whl and /dev/null differ diff --git a/tests/pypi/pyyaml/PyYAML-3.12-cp27-cp27m-win_amd64.whl b/tests/pypi/pyyaml/PyYAML-3.12-cp27-cp27m-win_amd64.whl deleted file mode 100644 index 2eb7094f..00000000 Binary files a/tests/pypi/pyyaml/PyYAML-3.12-cp27-cp27m-win_amd64.whl and /dev/null differ diff --git a/tests/pypi/pyyaml/PyYAML-3.12-cp35-cp35m-win32.whl b/tests/pypi/pyyaml/PyYAML-3.12-cp35-cp35m-win32.whl deleted file mode 100644 index f58c2f69..00000000 Binary files a/tests/pypi/pyyaml/PyYAML-3.12-cp35-cp35m-win32.whl and /dev/null differ diff --git a/tests/pypi/pyyaml/PyYAML-3.12-cp35-cp35m-win_amd64.whl b/tests/pypi/pyyaml/PyYAML-3.12-cp35-cp35m-win_amd64.whl deleted file mode 100644 index c3d36a8b..00000000 Binary files a/tests/pypi/pyyaml/PyYAML-3.12-cp35-cp35m-win_amd64.whl and /dev/null differ diff --git a/tests/pypi/pyyaml/PyYAML-3.12.tar.gz b/tests/pypi/pyyaml/PyYAML-3.12.tar.gz deleted file mode 100644 index aabee39f..00000000 Binary files a/tests/pypi/pyyaml/PyYAML-3.12.tar.gz and /dev/null differ diff --git a/tests/pypi/randomwords/RandomWords-0.2.1-py2.7.egg b/tests/pypi/randomwords/RandomWords-0.2.1-py2.7.egg deleted file mode 100644 index ef607e11..00000000 Binary files a/tests/pypi/randomwords/RandomWords-0.2.1-py2.7.egg and /dev/null differ diff --git a/tests/pypi/randomwords/RandomWords-0.2.1-py3.5.egg b/tests/pypi/randomwords/RandomWords-0.2.1-py3.5.egg deleted file mode 100644 index e8fa3f78..00000000 Binary files a/tests/pypi/randomwords/RandomWords-0.2.1-py3.5.egg and /dev/null differ diff --git a/tests/pypi/randomwords/RandomWords-0.2.1-py3.6.egg b/tests/pypi/randomwords/RandomWords-0.2.1-py3.6.egg deleted file mode 100644 index bd0b78c7..00000000 Binary files a/tests/pypi/randomwords/RandomWords-0.2.1-py3.6.egg and /dev/null differ diff --git a/tests/pypi/randomwords/RandomWords-0.2.1.tar.gz b/tests/pypi/randomwords/RandomWords-0.2.1.tar.gz deleted file mode 100644 index 3791a446..00000000 Binary files a/tests/pypi/randomwords/RandomWords-0.2.1.tar.gz and /dev/null differ diff --git a/tests/pypi/records/records-0.5.2-py2.py3-none-any.whl b/tests/pypi/records/records-0.5.2-py2.py3-none-any.whl deleted file mode 100644 index 0516a923..00000000 Binary files a/tests/pypi/records/records-0.5.2-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/records/records-0.5.2.tar.gz b/tests/pypi/records/records-0.5.2.tar.gz deleted file mode 100644 index f3690c19..00000000 Binary files a/tests/pypi/records/records-0.5.2.tar.gz and /dev/null differ diff --git a/tests/pypi/regex/regex-2018.02.21-cp27-none-win_amd64.whl b/tests/pypi/regex/regex-2018.02.21-cp27-none-win_amd64.whl deleted file mode 100644 index 31d8b2bc..00000000 Binary files a/tests/pypi/regex/regex-2018.02.21-cp27-none-win_amd64.whl and /dev/null differ diff --git a/tests/pypi/regex/regex-2018.02.21-cp36-none-win_amd64.whl b/tests/pypi/regex/regex-2018.02.21-cp36-none-win_amd64.whl deleted file mode 100644 index ce6a00f3..00000000 Binary files a/tests/pypi/regex/regex-2018.02.21-cp36-none-win_amd64.whl and /dev/null differ diff --git a/tests/pypi/regex/regex-2018.02.21.tar.gz b/tests/pypi/regex/regex-2018.02.21.tar.gz deleted file mode 100644 index 92e23a09..00000000 Binary files a/tests/pypi/regex/regex-2018.02.21.tar.gz and /dev/null differ diff --git a/tests/pypi/regex/regex-2018.06.21-cp27-none-win_amd64.whl b/tests/pypi/regex/regex-2018.06.21-cp27-none-win_amd64.whl deleted file mode 100644 index e715e787..00000000 Binary files a/tests/pypi/regex/regex-2018.06.21-cp27-none-win_amd64.whl and /dev/null differ diff --git a/tests/pypi/regex/regex-2018.06.21-cp36-none-win_amd64.whl b/tests/pypi/regex/regex-2018.06.21-cp36-none-win_amd64.whl deleted file mode 100644 index ef5e3a24..00000000 Binary files a/tests/pypi/regex/regex-2018.06.21-cp36-none-win_amd64.whl and /dev/null differ diff --git a/tests/pypi/regex/regex-2018.06.21-cp37-none-win_amd64.whl b/tests/pypi/regex/regex-2018.06.21-cp37-none-win_amd64.whl deleted file mode 100644 index 4b1ccfad..00000000 Binary files a/tests/pypi/regex/regex-2018.06.21-cp37-none-win_amd64.whl and /dev/null differ diff --git a/tests/pypi/regex/regex-2018.06.21.tar.gz b/tests/pypi/regex/regex-2018.06.21.tar.gz deleted file mode 100644 index e4696f18..00000000 Binary files a/tests/pypi/regex/regex-2018.06.21.tar.gz and /dev/null differ diff --git a/tests/pypi/requests/requests-1.0.0.tar.gz b/tests/pypi/requests/requests-1.0.0.tar.gz deleted file mode 100644 index d6184496..00000000 Binary files a/tests/pypi/requests/requests-1.0.0.tar.gz and /dev/null differ diff --git a/tests/pypi/requests/requests-2.14.0-py2.py3-none-any.whl b/tests/pypi/requests/requests-2.14.0-py2.py3-none-any.whl deleted file mode 100644 index 1fb0b95a..00000000 Binary files a/tests/pypi/requests/requests-2.14.0-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/requests/requests-2.14.0.tar.gz b/tests/pypi/requests/requests-2.14.0.tar.gz deleted file mode 100644 index 33631280..00000000 Binary files a/tests/pypi/requests/requests-2.14.0.tar.gz and /dev/null differ diff --git a/tests/pypi/requests/requests-2.18.4-py2.py3-none-any.whl b/tests/pypi/requests/requests-2.18.4-py2.py3-none-any.whl deleted file mode 100644 index cf3bdc07..00000000 Binary files a/tests/pypi/requests/requests-2.18.4-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/requests/requests-2.18.4.tar.gz b/tests/pypi/requests/requests-2.18.4.tar.gz deleted file mode 100644 index fce54795..00000000 Binary files a/tests/pypi/requests/requests-2.18.4.tar.gz and /dev/null differ diff --git a/tests/pypi/requests/requests-2.19.1-py2.py3-none-any.whl b/tests/pypi/requests/requests-2.19.1-py2.py3-none-any.whl deleted file mode 100644 index bc342f1d..00000000 Binary files a/tests/pypi/requests/requests-2.19.1-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/requests/requests-2.19.1.tar.gz b/tests/pypi/requests/requests-2.19.1.tar.gz deleted file mode 100644 index 5d4ad446..00000000 Binary files a/tests/pypi/requests/requests-2.19.1.tar.gz and /dev/null differ diff --git a/tests/pypi/ruamel-ordereddict/ruamel.ordereddict-0.4.9-cp27-cp27m-manylinux1_x86_64.whl b/tests/pypi/ruamel-ordereddict/ruamel.ordereddict-0.4.9-cp27-cp27m-manylinux1_x86_64.whl deleted file mode 100644 index d6919701..00000000 Binary files a/tests/pypi/ruamel-ordereddict/ruamel.ordereddict-0.4.9-cp27-cp27m-manylinux1_x86_64.whl and /dev/null differ diff --git a/tests/pypi/ruamel-ordereddict/ruamel.ordereddict-0.4.9-cp27-cp27mu-manylinux1_x86_64.whl b/tests/pypi/ruamel-ordereddict/ruamel.ordereddict-0.4.9-cp27-cp27mu-manylinux1_x86_64.whl deleted file mode 100644 index 26e63a8e..00000000 Binary files a/tests/pypi/ruamel-ordereddict/ruamel.ordereddict-0.4.9-cp27-cp27mu-manylinux1_x86_64.whl and /dev/null differ diff --git a/tests/pypi/ruamel-ordereddict/ruamel.ordereddict-0.4.9-cp27-none-win_amd64.whl b/tests/pypi/ruamel-ordereddict/ruamel.ordereddict-0.4.9-cp27-none-win_amd64.whl deleted file mode 100644 index b2b6bcd4..00000000 Binary files a/tests/pypi/ruamel-ordereddict/ruamel.ordereddict-0.4.9-cp27-none-win_amd64.whl and /dev/null differ diff --git a/tests/pypi/ruamel-ordereddict/ruamel.ordereddict-0.4.9.tar.gz b/tests/pypi/ruamel-ordereddict/ruamel.ordereddict-0.4.9.tar.gz deleted file mode 100644 index 0a6b0769..00000000 Binary files a/tests/pypi/ruamel-ordereddict/ruamel.ordereddict-0.4.9.tar.gz and /dev/null differ diff --git a/tests/pypi/ruamel-yaml/ruamel.yaml-0.15.9-cp27-cp27m-manylinux1_x86_64.whl b/tests/pypi/ruamel-yaml/ruamel.yaml-0.15.9-cp27-cp27m-manylinux1_x86_64.whl deleted file mode 100644 index 26694f18..00000000 Binary files a/tests/pypi/ruamel-yaml/ruamel.yaml-0.15.9-cp27-cp27m-manylinux1_x86_64.whl and /dev/null differ diff --git a/tests/pypi/ruamel-yaml/ruamel.yaml-0.15.9-cp27-cp27mu-manylinux1_x86_64.whl b/tests/pypi/ruamel-yaml/ruamel.yaml-0.15.9-cp27-cp27mu-manylinux1_x86_64.whl deleted file mode 100644 index 362db9cb..00000000 Binary files a/tests/pypi/ruamel-yaml/ruamel.yaml-0.15.9-cp27-cp27mu-manylinux1_x86_64.whl and /dev/null differ diff --git a/tests/pypi/ruamel-yaml/ruamel.yaml-0.15.9-cp36-cp36m-manylinux1_x86_64.whl b/tests/pypi/ruamel-yaml/ruamel.yaml-0.15.9-cp36-cp36m-manylinux1_x86_64.whl deleted file mode 100644 index 73912720..00000000 Binary files a/tests/pypi/ruamel-yaml/ruamel.yaml-0.15.9-cp36-cp36m-manylinux1_x86_64.whl and /dev/null differ diff --git a/tests/pypi/ruamel-yaml/ruamel.yaml-0.15.9.tar.gz b/tests/pypi/ruamel-yaml/ruamel.yaml-0.15.9.tar.gz deleted file mode 100644 index 9e607d32..00000000 Binary files a/tests/pypi/ruamel-yaml/ruamel.yaml-0.15.9.tar.gz and /dev/null differ diff --git a/tests/pypi/setuptools/setuptools-39.0.1-py2.py3-none-any.whl b/tests/pypi/setuptools/setuptools-39.0.1-py2.py3-none-any.whl deleted file mode 100644 index edc3ca2d..00000000 Binary files a/tests/pypi/setuptools/setuptools-39.0.1-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/six/six-1.11.0-py2.py3-none-any.whl b/tests/pypi/six/six-1.11.0-py2.py3-none-any.whl deleted file mode 100644 index 59960239..00000000 Binary files a/tests/pypi/six/six-1.11.0-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/six/six-1.11.0.tar.gz b/tests/pypi/six/six-1.11.0.tar.gz deleted file mode 100644 index 353f31f9..00000000 Binary files a/tests/pypi/six/six-1.11.0.tar.gz and /dev/null differ diff --git a/tests/pypi/smmap2/smmap2-2.0.3-py2.py3-none-any.whl b/tests/pypi/smmap2/smmap2-2.0.3-py2.py3-none-any.whl deleted file mode 100644 index 72599115..00000000 Binary files a/tests/pypi/smmap2/smmap2-2.0.3-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/smmap2/smmap2-2.0.3.tar.gz b/tests/pypi/smmap2/smmap2-2.0.3.tar.gz deleted file mode 100644 index e959ad02..00000000 Binary files a/tests/pypi/smmap2/smmap2-2.0.3.tar.gz and /dev/null differ diff --git a/tests/pypi/snaptime/snaptime-0.2.4.tar.gz b/tests/pypi/snaptime/snaptime-0.2.4.tar.gz deleted file mode 100644 index df152642..00000000 Binary files a/tests/pypi/snaptime/snaptime-0.2.4.tar.gz and /dev/null differ diff --git a/tests/pypi/sqlalchemy/SQLAlchemy-1.2.0b3.tar.gz b/tests/pypi/sqlalchemy/SQLAlchemy-1.2.0b3.tar.gz deleted file mode 100644 index 40eafa66..00000000 Binary files a/tests/pypi/sqlalchemy/SQLAlchemy-1.2.0b3.tar.gz and /dev/null differ diff --git a/tests/pypi/sqlalchemy/SQLAlchemy-1.2.4.tar.gz b/tests/pypi/sqlalchemy/SQLAlchemy-1.2.4.tar.gz deleted file mode 100644 index e965a280..00000000 Binary files a/tests/pypi/sqlalchemy/SQLAlchemy-1.2.4.tar.gz and /dev/null differ diff --git a/tests/pypi/sqlalchemy/SQLAlchemy-1.2.8.tar.gz b/tests/pypi/sqlalchemy/SQLAlchemy-1.2.8.tar.gz deleted file mode 100644 index d624e540..00000000 Binary files a/tests/pypi/sqlalchemy/SQLAlchemy-1.2.8.tar.gz and /dev/null differ diff --git a/tests/pypi/tablib/tablib-0.11.5.tar.gz b/tests/pypi/tablib/tablib-0.11.5.tar.gz deleted file mode 100644 index 460cd0a7..00000000 Binary files a/tests/pypi/tablib/tablib-0.11.5.tar.gz and /dev/null differ diff --git a/tests/pypi/tablib/tablib-0.12.0.tar.gz b/tests/pypi/tablib/tablib-0.12.0.tar.gz deleted file mode 100644 index 36ab89f2..00000000 Binary files a/tests/pypi/tablib/tablib-0.12.0.tar.gz and /dev/null differ diff --git a/tests/pypi/tablib/tablib-0.12.1.tar.gz b/tests/pypi/tablib/tablib-0.12.1.tar.gz deleted file mode 100644 index 825ee278..00000000 Binary files a/tests/pypi/tablib/tablib-0.12.1.tar.gz and /dev/null differ diff --git a/tests/pypi/tpfd/tpfd-0.2.4-py2.py3-none-any.whl b/tests/pypi/tpfd/tpfd-0.2.4-py2.py3-none-any.whl deleted file mode 100644 index a1e64899..00000000 Binary files a/tests/pypi/tpfd/tpfd-0.2.4-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/tpfd/tpfd-0.2.4.tar.gz b/tests/pypi/tpfd/tpfd-0.2.4.tar.gz deleted file mode 100644 index c13d45c0..00000000 Binary files a/tests/pypi/tpfd/tpfd-0.2.4.tar.gz and /dev/null differ diff --git a/tests/pypi/tzlocal/tzlocal-1.5.1.tar.gz b/tests/pypi/tzlocal/tzlocal-1.5.1.tar.gz deleted file mode 100644 index 3f5704a8..00000000 Binary files a/tests/pypi/tzlocal/tzlocal-1.5.1.tar.gz and /dev/null differ diff --git a/tests/pypi/unicodecsv/unicodecsv-0.14.1.tar.gz b/tests/pypi/unicodecsv/unicodecsv-0.14.1.tar.gz deleted file mode 100644 index a7858b00..00000000 Binary files a/tests/pypi/unicodecsv/unicodecsv-0.14.1.tar.gz and /dev/null differ diff --git a/tests/pypi/urllib3/urllib3-1.22-py2.py3-none-any.whl b/tests/pypi/urllib3/urllib3-1.22-py2.py3-none-any.whl deleted file mode 100644 index d4464993..00000000 Binary files a/tests/pypi/urllib3/urllib3-1.22-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/urllib3/urllib3-1.22.tar.gz b/tests/pypi/urllib3/urllib3-1.22.tar.gz deleted file mode 100644 index 3d9ee9f1..00000000 Binary files a/tests/pypi/urllib3/urllib3-1.22.tar.gz and /dev/null differ diff --git a/tests/pypi/urllib3/urllib3-1.23-py2.py3-none-any.whl b/tests/pypi/urllib3/urllib3-1.23-py2.py3-none-any.whl deleted file mode 100644 index a385e3c9..00000000 Binary files a/tests/pypi/urllib3/urllib3-1.23-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/urllib3/urllib3-1.23.tar.gz b/tests/pypi/urllib3/urllib3-1.23.tar.gz deleted file mode 100644 index 401dc252..00000000 Binary files a/tests/pypi/urllib3/urllib3-1.23.tar.gz and /dev/null differ diff --git a/tests/pypi/vcrpy/vcrpy-1.11.0-py2.py3-none-any.whl b/tests/pypi/vcrpy/vcrpy-1.11.0-py2.py3-none-any.whl deleted file mode 100644 index 2853aafc..00000000 Binary files a/tests/pypi/vcrpy/vcrpy-1.11.0-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/vcrpy/vcrpy-1.11.0.tar.gz b/tests/pypi/vcrpy/vcrpy-1.11.0.tar.gz deleted file mode 100644 index 605bce97..00000000 Binary files a/tests/pypi/vcrpy/vcrpy-1.11.0.tar.gz and /dev/null differ diff --git a/tests/pypi/werkzeug/Werkzeug-0.14.1-py2.py3-none-any.whl b/tests/pypi/werkzeug/Werkzeug-0.14.1-py2.py3-none-any.whl deleted file mode 100644 index 865d5248..00000000 Binary files a/tests/pypi/werkzeug/Werkzeug-0.14.1-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/werkzeug/Werkzeug-0.14.1.tar.gz b/tests/pypi/werkzeug/Werkzeug-0.14.1.tar.gz deleted file mode 100644 index 27e7b2d7..00000000 Binary files a/tests/pypi/werkzeug/Werkzeug-0.14.1.tar.gz and /dev/null differ diff --git a/tests/pypi/wheel/wheel-0.31.0-py2.py3-none-any.whl b/tests/pypi/wheel/wheel-0.31.0-py2.py3-none-any.whl deleted file mode 100644 index 8cfdda05..00000000 Binary files a/tests/pypi/wheel/wheel-0.31.0-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/win-inet-pton/win_inet_pton-1.0.1.tar.gz b/tests/pypi/win-inet-pton/win_inet_pton-1.0.1.tar.gz deleted file mode 100644 index dfb77887..00000000 Binary files a/tests/pypi/win-inet-pton/win_inet_pton-1.0.1.tar.gz and /dev/null differ diff --git a/tests/pypi/wrapt/wrapt-1.10.11.tar.gz b/tests/pypi/wrapt/wrapt-1.10.11.tar.gz deleted file mode 100644 index 7f6870e2..00000000 Binary files a/tests/pypi/wrapt/wrapt-1.10.11.tar.gz and /dev/null differ diff --git a/tests/pypi/xlrd/xlrd-1.1.0-py2.py3-none-any.whl b/tests/pypi/xlrd/xlrd-1.1.0-py2.py3-none-any.whl deleted file mode 100644 index fd326f8a..00000000 Binary files a/tests/pypi/xlrd/xlrd-1.1.0-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/xlrd/xlrd-1.1.0.tar.gz b/tests/pypi/xlrd/xlrd-1.1.0.tar.gz deleted file mode 100644 index 6f8d5666..00000000 Binary files a/tests/pypi/xlrd/xlrd-1.1.0.tar.gz and /dev/null differ diff --git a/tests/pypi/xlwt/xlwt-1.3.0-py2.py3-none-any.whl b/tests/pypi/xlwt/xlwt-1.3.0-py2.py3-none-any.whl deleted file mode 100644 index ab824454..00000000 Binary files a/tests/pypi/xlwt/xlwt-1.3.0-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/xlwt/xlwt-1.3.0.tar.gz b/tests/pypi/xlwt/xlwt-1.3.0.tar.gz deleted file mode 100644 index f9a65e06..00000000 Binary files a/tests/pypi/xlwt/xlwt-1.3.0.tar.gz and /dev/null differ diff --git a/tests/pypi/yarl/yarl-1.1.1-cp36-cp36m-manylinux1_i686.whl b/tests/pypi/yarl/yarl-1.1.1-cp36-cp36m-manylinux1_i686.whl deleted file mode 100644 index ab6c0e0c..00000000 Binary files a/tests/pypi/yarl/yarl-1.1.1-cp36-cp36m-manylinux1_i686.whl and /dev/null differ diff --git a/tests/pypi/yarl/yarl-1.1.1-cp36-cp36m-manylinux1_x86_64.whl b/tests/pypi/yarl/yarl-1.1.1-cp36-cp36m-manylinux1_x86_64.whl deleted file mode 100644 index baeeb2c1..00000000 Binary files a/tests/pypi/yarl/yarl-1.1.1-cp36-cp36m-manylinux1_x86_64.whl and /dev/null differ diff --git a/tests/pypi/yarl/yarl-1.1.1-cp36-cp36m-win32.whl b/tests/pypi/yarl/yarl-1.1.1-cp36-cp36m-win32.whl deleted file mode 100644 index 45fe89ed..00000000 Binary files a/tests/pypi/yarl/yarl-1.1.1-cp36-cp36m-win32.whl and /dev/null differ diff --git a/tests/pypi/yarl/yarl-1.1.1-cp36-cp36m-win_amd64.whl b/tests/pypi/yarl/yarl-1.1.1-cp36-cp36m-win_amd64.whl deleted file mode 100644 index e11d5c62..00000000 Binary files a/tests/pypi/yarl/yarl-1.1.1-cp36-cp36m-win_amd64.whl and /dev/null differ diff --git a/tests/pytest-pypi/DESCRIPTION.rst b/tests/pytest-pypi/DESCRIPTION.rst index 4e0cba87..823e4733 100644 --- a/tests/pytest-pypi/DESCRIPTION.rst +++ b/tests/pytest-pypi/DESCRIPTION.rst @@ -1,30 +1,5 @@ -pytest-httpbin -============== +pytest-pypi +=========== -httpbin is an amazing web service for testing HTTP libraries. It has several -great endpoints that can test pretty much everything you need in a HTTP -library. The only problem is: maybe you don't want to wait for your tests to -travel across the Internet and back to make assertions against a remote web -service. - -Enter pytest-httpbin. Pytest-httpbin creates a pytest "fixture" that is -dependency-injected into your tests. It automatically starts up a HTTP server -in a separate thread running httpbin and provides your test with the URL in the -fixture. Check out this example: - -.. code-block:: python - - def test_that_my_library_works_kinda_ok(httpbin): - assert requests.get(httpbin.url + '/get/').status_code == 200 - -This replaces a test that might have looked like this before: - -.. code-block:: python - - def test_that_my_library_works_kinda_ok(): - assert requests.get('http://httpbin.org/get').status_code == 200 - -pytest-httpbin also supports https and includes its own CA cert you can use. -Check out `the full documentation`_ on the github page. - -.. _the full documentation: https://github.com/kevin1024/pytest-httpbin +Easily test your HTTP library against a local copy of PyPI. +This is an internal pytest plugin of pipenv. diff --git a/tests/pytest-pypi/README.md b/tests/pytest-pypi/README.md index a9ba37c2..31b33807 100644 --- a/tests/pytest-pypi/README.md +++ b/tests/pytest-pypi/README.md @@ -1,194 +1,4 @@ -# pytest-httpbin +# pytest-pypi -[![Build Status](https://travis-ci.org/kevin1024/pytest-httpbin.svg?branch=master)](https://travis-ci.org/kevin1024/pytest-httpbin) - -[httpbin](https://httpbin.org/) is an amazing web service for testing HTTP libraries. It has several great endpoints that can test pretty much everything you need in a HTTP library. The only problem is: maybe you don't want to wait for your tests to travel across the Internet and back to make assertions against a remote web service (speed), and maybe you want to work offline (convenience). - -Enter **pytest-httpbin**. Pytest-httpbin creates a [pytest fixture](http://pytest.org/latest/fixture.html) that is dependency-injected into your tests. It automatically starts up a HTTP server in a separate thread running httpbin and provides your test with the URL in the fixture. Check out this example: - -```python -def test_that_my_library_works_kinda_ok(httpbin): - assert requests.get(httpbin.url + '/get').status_code == 200 -``` - -This replaces a test that might have looked like this before: - -```python -def test_that_my_library_works_kinda_ok(): - assert requests.get('http://httpbin.org/get').status_code == 200 -``` - -If you're making a lot of requests to httpbin, it can radically speed up your tests. - -![demo](http://i.imgur.com/heNOQLP.gif) - - -# HTTPS support - -pytest-httpbin also supports HTTPS: - -```python -def test_that_my_library_works_kinda_ok(httpbin_secure): - assert requests.get(httpbin_secure.url + '/get/').status_code == 200 -``` - -It's actually starting 2 web servers in separate threads in the background: one HTTP and one HTTPS. The servers are started on a random port (see below for fixed port support), on the loopback interface on your machine. Pytest-httpbin includes a self-signed certificate. If your library verifies certificates against a CA (and it should), you'll have to add the CA from pytest-httpbin. The path to the pytest-httpbin CA bundle can by found like this `python -m pytest_httpbin.certs`. - -For example in requests, you can set the `REQUESTS_CA_BUNDLE` python path. You can run your tests like this: - -```bash -REQUESTS_CA_BUNDLE=`python -m pytest_httpbin.certs` py.test tests/ -``` - -# API of the injected object - -The injected object has the following attributes: - - * url - * port - * host - -and the following methods: - - * join(string): Returns the results of calling `urlparse.urljoin` with the url from the injected server automatically applied as the first argument. You supply the second argument - -Also, I defined `__add__` on the object to append to `httpbin.url`. This means you can do stuff like `httpbin + '/get'` instead of `httpbin.url + '/get'`. - -## Testing both HTTP and HTTPS endpoints with one test - -If you ever find yourself needing to test both the http and https version of and endpoint, you can use the `httpbin_both` funcarg like this: - - -```python -def test_that_my_library_works_kinda_ok(httpbin_both): - assert requests.get(httpbin_both.url + '/get/').status_code == 200 -``` - -Through the magic of pytest parametrization, this function will actually execute twice: once with an http url and once with an https url. - -## Using pytest-httpbin with unittest-style test cases - -I have provided 2 additional fixtures to make testing with class-based tests easier. I have also provided a couple decorators that provide some syntactic sugar around the pytest method of adding the fixtures to class-based tests. Just add the `use_class_based_httpbin` and/or `use_class_based_httpbin_secure` class decorators to your class, and then you can access httpbin using self.httpbin and self.httpbin_secure. - -```python -import pytest_httpbin - -@pytest_httpbin.use_class_based_httpbin -@pytest_httpbin.use_class_based_httpbin_secure -class TestClassBassedTests(unittest.TestCase): - def test_http(self): - assert requests.get(self.httpbin.url + '/get').response - - def test_http_secure(self): - assert requests.get(self.httpbin_secure.url + '/get').response -``` - -## Running the server on fixed port - -Sometimes a randomized port can be a problem. Worry not, you can fix the port number to a desired value with the `HTTPBIN_HTTP_PORT` and `HTTPBIN_HTTPS_PORT` environment variables. If those are defined during pytest plugins are loaded, `httbin` and `httpbin_secure` fixtures will run on given ports. You can run your tests like this: - -```bash -HTTPBIN_HTTP_PORT=8080 HTTPBIN_HTTPS_PORT=8443 py.test tests/ -``` - -## Installation - -All you need to do is this: - -```bash -pip install pytest-httpbin -``` - -and your tests executed by pytest all will have access to the `httpbin` and `httpbin_secure` funcargs. Cool right? - -## Support and dependencies - -pytest-httpbin supports Python 2.6, 2.7, 3.4, and pypy. It will automatically install httpbin and flask when you install it from pypi. - -[httpbin](https://github.com/kennethreitz/httpbin) itself does not support python 2.6 as of version 0.6.0, when the Flask-common dependency was added. If you need python 2.6 support pin the httpbin version to 0.5.0 - -## Running the pytest-httpbin test suite - -If you want to run pytest-httpbin's test suite, you'll need to install requests and pytest, and then use the ./runtests.sh script. - -```bash -pip install pytest -/.runtests.sh -``` - -Also, you can use tox to run the tests on all supported python versions: - -```bash -pip install tox -tox -``` - -## Changelog - -* 0.3.0 - * Allow to run httpbin on fixed port using environment variables (thanks @hroncok) - * Allow server to be thread.join()ed (thanks @graingert) - * Add support for Python 3.6 (thanks @graingert) -* 0.2.3: - * Another attempt to fix #32 (Rare bug, only happens on Travis) -* 0.2.2: - * Fix bug with python3 -* 0.2.1: - * Attempt to fix strange, impossible-to-reproduce bug with broken SSL certs - that only happens on Travis (#32) [Bad release, breaks py3] -* 0.2.0: - * Remove threaded HTTP server. I built it for Requests, but they deleted - their threaded test since it didn't really work very well. The threaded - server seems to cause some strange problems with HTTP chunking, so I'll - just remove it since nobody is using it (I hope) -* 0.1.1: - * Fix weird hang with SSL on pypy (again) -* 0.1.0: - * Update server to use multithreaded werkzeug server -* 0.0.7: - * Update the certificates (they expired) -* 0.0.6: - * Fix an issue where pypy was hanging when a request was made with an invalid - certificate -* 0.0.5: - * Fix broken version parsing in 0.0.4 -* 0.0.4: - * **Bad release: Broken version parsing** - * Fix `BadStatusLine` error that occurs when sending multiple requests - in a single session (PR #16). Thanks @msabramo! - * Fix #9 ("Can't be installed at the same time than pytest?") (PR - #14). Thanks @msabramo! - * Add `httpbin_ca_bundle` pytest fixture. With this fixture there is - no need to specify the bundle on every request, as it will - automatically set `REQUESTS_CA_BUNDLE` if using - [requests](http://docs.python-requests.org/). And you don't have to - care about where it is located (PR #8). Thanks @t-8ch! -* 0.0.3: Add a couple test fixtures to make testing old class-based test suites - easier -* 0.0.2: Fixed a couple bugs with the wsgiref server to bring behavior in line - with httpbin.org, thanks @jakubroztocil for the bug reports -* 0.0.1: Initial release - -## License - -The MIT License (MIT) - -Copyright (c) 2014-2015 Kevin McCarthy - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. +Easily test your HTTP library against a local copy of PyPI. +This is an internal pytest plugin of pipenv. diff --git a/tests/pytest-pypi/pytest_pypi/app.py b/tests/pytest-pypi/pytest_pypi/app.py index ba1a5437..95dbd076 100644 --- a/tests/pytest-pypi/pytest_pypi/app.py +++ b/tests/pytest-pypi/pytest_pypi/app.py @@ -1,13 +1,42 @@ -import os +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function +import contextlib +import io import json +import os +import sys + +from tarfile import is_tarfile +from zipfile import is_zipfile import requests +from six.moves import xmlrpc_client + from flask import Flask, redirect, abort, render_template, send_file, jsonify + app = Flask(__name__) session = requests.Session() packages = {} +ARTIFACTS = {} + + +@contextlib.contextmanager +def xml_pypi_server(server): + transport = xmlrpc_client.Transport() + client = xmlrpc_client.ServerProxy(server, transport) + try: + yield client + finally: + transport.close() + + +def get_pypi_package_names(): + pypi_packages = set() + with xml_pypi_server("https://pypi.org/pypi") as client: + pypi_packages = set(client.list_packages()) + return pypi_packages class Package(object): @@ -26,7 +55,26 @@ class Package(object): with open(os.path.join(path, 'api.json')) as f: return json.load(f) except FileNotFoundError: - pass + r = session.get('https://pypi.org/pypi/{0}/json'.format(self.name)) + response = r.json() + releases = response["releases"] + files = { + pkg for pkg_dir in self._package_dirs + for pkg in os.listdir(pkg_dir) + } + for release in list(releases.keys()): + values = ( + r for r in releases[release] if r["filename"] in files + ) + values = list(values) + if values: + releases[release] = values + else: + del releases[release] + response["releases"] = releases + with io.open(os.path.join(path, "api.json"), "w") as fh: + json.dump(response, fh, indent=4) + return response def __repr__(self): return "/') def simple_package(package): - if package in packages: + if package in packages and packages[package].releases: return render_template('package.html', package=packages[package]) + else: + try: + r = requests.get("https://pypi.org/simple/{0}".format(package)) + r.raise_for_status() + except Exception: + abort(404) + else: + return render_template( + 'package_pypi.html', package_contents=r.text + ) + + +@app.route('/artifacts//') +def simple_artifact(artifact): + if artifact in ARTIFACTS: + return render_template('artifact.html', artifact=ARTIFACTS[artifact]) else: abort(404) @@ -83,20 +200,28 @@ def serve_package(package, release): abort(404) +@app.route('/artifacts//') +def serve_artifact(artifact, fn): + if artifact in ARTIFACTS: + artifact = ARTIFACTS[artifact] + if fn in artifact.files: + return send_file(artifact.files[fn]) + abort(404) + + @app.route('/pypi//json') def json_for_package(package): - try: - return jsonify(packages[package].json) - except Exception: - pass - - r = session.get('https://pypi.org/pypi/{0}/json'.format(package)) - return jsonify(r.json()) + return jsonify(packages[package].json) + # try: + # except Exception: + # r = session.get('https://pypi.org/pypi/{0}/json'.format(package)) + # return jsonify(r.json()) if __name__ == '__main__': PYPI_VENDOR_DIR = os.environ.get('PYPI_VENDOR_DIR', './pypi') PYPI_VENDOR_DIR = os.path.abspath(PYPI_VENDOR_DIR) prepare_packages(PYPI_VENDOR_DIR) + prepare_fixtures(os.path.join(PYPI_VENDOR_DIR, "fixtures")) app.run() diff --git a/tests/pytest-pypi/pytest_pypi/templates/artifact.html b/tests/pytest-pypi/pytest_pypi/templates/artifact.html new file mode 100644 index 00000000..5f4199c5 --- /dev/null +++ b/tests/pytest-pypi/pytest_pypi/templates/artifact.html @@ -0,0 +1,14 @@ + + + + + Links for {{ artifact.name }} + + +

Links for {{ artifact.name }}

+ {% for fn in artifact.files %} + {{ fn }} +
+ {% endfor %} + + diff --git a/tests/pytest-pypi/pytest_pypi/templates/artifacts.html b/tests/pytest-pypi/pytest_pypi/templates/artifacts.html new file mode 100644 index 00000000..6bee78da --- /dev/null +++ b/tests/pytest-pypi/pytest_pypi/templates/artifacts.html @@ -0,0 +1,13 @@ + + + + + Artifact Index + + + {% for artifact in artifacts %} + {{ artifact.name }} +
+ {% endfor %} + + diff --git a/tests/pytest-pypi/pytest_pypi/templates/package_pypi.html b/tests/pytest-pypi/pytest_pypi/templates/package_pypi.html new file mode 100644 index 00000000..217d8aa0 --- /dev/null +++ b/tests/pytest-pypi/pytest_pypi/templates/package_pypi.html @@ -0,0 +1,4 @@ + +{% autoescape false %} + {{ package_contents }} +{% endautoescape %} diff --git a/tests/pytest-pypi/setup.py b/tests/pytest-pypi/setup.py index 87f6b752..558dd3f8 100644 --- a/tests/pytest-pypi/setup.py +++ b/tests/pytest-pypi/setup.py @@ -62,7 +62,7 @@ setup( long_description=long_description, # The project URL. - url='https://github.com/kennethreitz/pytest-pypi', + url='https://github.com/pypa/pipenv/tree/master/tests/pytest-pypi', # Author details author='Kenneth Reitz', diff --git a/tests/test_artifacts/django/3.4.x.zip b/tests/test_artifacts/django/3.4.x.zip new file mode 100644 index 00000000..6c2ae5f6 Binary files /dev/null and b/tests/test_artifacts/django/3.4.x.zip differ diff --git a/tests/test_artifacts/git/flask b/tests/test_artifacts/git/flask new file mode 160000 index 00000000..dcc02d6e --- /dev/null +++ b/tests/test_artifacts/git/flask @@ -0,0 +1 @@ +Subproject commit dcc02d6e7d4bf486e64aa5b6e55a75501c2ba2e5 diff --git a/tests/test_artifacts/git/jinja2 b/tests/test_artifacts/git/jinja2 new file mode 160000 index 00000000..a7f1f528 --- /dev/null +++ b/tests/test_artifacts/git/jinja2 @@ -0,0 +1 @@ +Subproject commit a7f1f528f5e77d5401a96aa326885508245f7c6f diff --git a/tests/test_artifacts/git/pyinstaller b/tests/test_artifacts/git/pyinstaller new file mode 160000 index 00000000..19d8a378 --- /dev/null +++ b/tests/test_artifacts/git/pyinstaller @@ -0,0 +1 @@ +Subproject commit 19d8a378987d6115c0acb72ef954bbf3bca3c61b diff --git a/tests/test_artifacts/git/requests b/tests/test_artifacts/git/requests index 57d7284c..4983a9bd 160000 --- a/tests/test_artifacts/git/requests +++ b/tests/test_artifacts/git/requests @@ -1 +1 @@ -Subproject commit 57d7284c1a245cf9fbcecb594f50471d86e879f7 +Subproject commit 4983a9bde39c6320aa4f3e34e50dac6e263dab6f diff --git a/tests/test_artifacts/git/requests-2.18.4 b/tests/test_artifacts/git/requests-2.18.4 new file mode 160000 index 00000000..a3d7cf3f --- /dev/null +++ b/tests/test_artifacts/git/requests-2.18.4 @@ -0,0 +1 @@ +Subproject commit a3d7cf3f27e74c28ef30f01e9f2e483570ab042e diff --git a/tests/test_artifacts/git/six b/tests/test_artifacts/git/six index e114efce..aa4e90bc 160000 --- a/tests/test_artifacts/git/six +++ b/tests/test_artifacts/git/six @@ -1 +1 @@ -Subproject commit e114efceea962fb143c909c904157ca994246fd2 +Subproject commit aa4e90bcd7b7bc13a71dfaebcb2021f4caaa8432 diff --git a/tests/unit/test_cmdparse.py b/tests/unit/test_cmdparse.py index 1b329a53..912031e9 100644 --- a/tests/unit/test_cmdparse.py +++ b/tests/unit/test_cmdparse.py @@ -64,3 +64,12 @@ def test_cmdify_quote_if_paren_in_command(): '-c', "print(123)", ]), script + + +@pytest.mark.run +@pytest.mark.script +def test_cmdify_quote_if_carets(): + """Ensure arguments are quoted if they contain carets. + """ + script = Script('foo^bar', ['baz^rex']) + assert script.cmdify() == '"foo^bar" "baz^rex"', script diff --git a/tests/unit/test_core.py b/tests/unit/test_core.py index 394e1120..61d318c3 100644 --- a/tests/unit/test_core.py +++ b/tests/unit/test_core.py @@ -1,10 +1,10 @@ import os -import pytest import mock +import pytest from pipenv._compat import TemporaryDirectory -from pipenv.core import warn_in_virtualenv, load_dot_env +from pipenv.core import load_dot_env, warn_in_virtualenv from pipenv.utils import temp_environ diff --git a/tests/unit/test_help.py b/tests/unit/test_help.py index 2432e969..d3755601 100644 --- a/tests/unit/test_help.py +++ b/tests/unit/test_help.py @@ -9,3 +9,16 @@ def test_help(): stderr=subprocess.STDOUT, env=os.environ.copy(), ) assert output + + +def test_count_of_description_pre_option(): + test_command = 'pipenv install --help' + test_line = '--pre Allow pre-releases.' + out = subprocess.Popen(test_command.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + stdout, _ = out.communicate() + lines = stdout.decode().split('\n') + count = 0 + for line in lines: + if line.strip().split() == test_line.split(): + count += 1 + assert count == 1 diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 422c1002..18cb94a8 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -1,10 +1,14 @@ # -*- coding: utf-8 -*- import os + import pytest -from mock import patch, Mock + from first import first +from mock import Mock, patch + import pipenv.utils import pythonfinder.utils +from pipenv.exceptions import PipenvUsageError # Pipfile format <-> requirements.txt format. @@ -71,12 +75,20 @@ DEP_PIP_PAIRS = [ ] +def mock_unpack(link, source_dir, download_dir, only_download=False, session=None, + hashes=None, progress_bar="off"): + return + + @pytest.mark.utils @pytest.mark.parametrize("deps, expected", DEP_PIP_PAIRS) -def test_convert_deps_to_pip(deps, expected): - if expected.startswith("Django"): - expected = expected.lower() - assert pipenv.utils.convert_deps_to_pip(deps, r=False) == [expected] +def test_convert_deps_to_pip(monkeypatch, deps, expected): + with monkeypatch.context() as m: + import pip_shims + m.setattr(pip_shims.shims, "unpack_url", mock_unpack) + if expected.startswith("Django"): + expected = expected.lower() + assert pipenv.utils.convert_deps_to_pip(deps, r=False) == [expected] @pytest.mark.utils @@ -117,8 +129,11 @@ def test_convert_deps_to_pip(deps, expected): ), ], ) -def test_convert_deps_to_pip_one_way(deps, expected): - assert pipenv.utils.convert_deps_to_pip(deps, r=False) == [expected.lower()] +def test_convert_deps_to_pip_one_way(monkeypatch, deps, expected): + with monkeypatch.context() as m: + import pip_shims + # m.setattr(pip_shims.shims, "unpack_url", mock_unpack) + assert pipenv.utils.convert_deps_to_pip(deps, r=False) == [expected.lower()] @pytest.mark.skipif(isinstance(u"", str), reason="don't need to test if unicode is str") @@ -222,6 +237,29 @@ class TestUtils: assert os.path.exists(output) os.remove(output) + @pytest.mark.utils + @pytest.mark.parametrize('line, expected', [ + ("python", True), + ("python3.7", True), + ("python2.7", True), + ("python2", True), + ("python3", True), + ("pypy3", True), + ("anaconda3-5.3.0", True), + ("which", False), + ("vim", False), + ("miniconda", True), + ("micropython", True), + ("ironpython", True), + ("jython3.5", True), + ("2", True), + ("2.7", True), + ("3.7", True), + ("3", True) + ]) + def test_is_python_command(self, line, expected): + assert pipenv.utils.is_python_command(line) == expected + @pytest.mark.utils def test_new_line_end_of_toml_file(this): # toml file that needs clean up @@ -294,6 +332,15 @@ twine = "*" "test.example.com", ], ), + ( + [{"url": "https://test.example.com:12345/simple", "verify_ssl": False}], + [ + "-i", + "https://test.example.com:12345/simple", + "--trusted-host", + "test.example.com:12345", + ], + ), ( [ {"url": "https://pypi.org/simple"}, @@ -320,6 +367,20 @@ twine = "*" "custom.example.com", ], ), + ( + [ + {"url": "https://pypi.org/simple"}, + {"url": "https://custom.example.com:12345/simple", "verify_ssl": False}, + ], + [ + "-i", + "https://pypi.org/simple", + "--extra-index-url", + "https://custom.example.com:12345/simple", + "--trusted-host", + "custom.example.com:12345", + ], + ), ( [ {"url": "https://pypi.org/simple"}, @@ -371,6 +432,11 @@ twine = "*" == expected_args ) + def test_invalid_prepare_pip_source_args(self): + sources = [{}] + with pytest.raises(PipenvUsageError): + pipenv.utils.prepare_pip_source_args(sources, pip_args=None) + @pytest.mark.utils def test_parse_python_version(self): ver = pipenv.utils.parse_python_version("Python 3.6.5\n")